Extractive Text Summarization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "9adf9fae-1b9c-440c-bd66-e678a69ea270",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pip in /databricks/python3/lib/python3.8/site-packages (21.0.1)\r\nCollecting pip\r\n  Downloading pip-22.0.3-py3-none-any.whl (2.1 MB)\r\n\u001B[?25l\r\u001B[K     |▏                               | 10 kB 19.3 MB/s eta 0:00:01\r\u001B[K     |▎                               | 20 kB 7.8 MB/s eta 0:00:01\r\u001B[K     |▌                               | 30 kB 6.8 MB/s eta 0:00:01\r\u001B[K     |▋                               | 40 kB 6.3 MB/s eta 0:00:01\r\u001B[K     |▉                               | 51 kB 4.9 MB/s eta 0:00:01\r\u001B[K     |█                               | 61 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█                               | 71 kB 5.1 MB/s eta 0:00:01\r\u001B[K     |█▎                              | 81 kB 5.2 MB/s eta 0:00:01\r\u001B[K     |█▍                              | 92 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█▋                              | 102 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█▊                              | 112 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█▉                              | 122 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██                              | 133 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██▏                             | 143 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██▍                             | 153 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██▌                             | 163 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██▊                             | 174 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██▉                             | 184 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███                             | 194 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███▏                            | 204 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███▎                            | 215 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███▌                            | 225 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███▋                            | 235 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███▊                            | 245 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████                            | 256 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████                            | 266 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████▎                           | 276 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████▍                           | 286 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████▋                           | 296 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████▊                           | 307 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████▉                           | 317 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████                           | 327 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████▏                          | 337 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████▍                          | 348 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████▌                          | 358 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████▋                          | 368 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████▉                          | 378 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████                          | 389 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████▏                         | 399 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████▎                         | 409 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████▌                         | 419 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████▋                         | 430 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████▊                         | 440 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████                         | 450 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████                         | 460 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████▎                        | 471 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████▍                        | 481 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████▌                        | 491 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████▊                        | 501 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████▉                        | 512 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████                        | 522 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████▏                       | 532 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████▎                       | 542 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████▌                       | 552 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████▋                       | 563 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████▉                       | 573 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████                       | 583 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████▏                      | 593 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████▎                      | 604 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████▍                      | 614 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████▋                      | 624 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████▊                      | 634 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████                      | 645 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████                      | 655 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████▏                     | 665 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████▍                     | 675 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████▌                     | 686 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████▊                     | 696 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████▉                     | 706 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████                     | 716 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████▏                    | 727 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████▎                    | 737 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████▌                    | 747 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████▋                    | 757 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████▉                    | 768 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████                    | 778 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████                    | 788 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████▎                   | 798 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████▍                   | 808 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████▋                   | 819 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████▊                   | 829 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████                   | 839 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████                   | 849 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████▏                  | 860 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████▍                  | 870 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████▌                  | 880 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████▊                  | 890 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████▉                  | 901 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████                  | 911 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████▏                 | 921 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████▎                 | 931 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████▌                 | 942 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████▋                 | 952 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████▊                 | 962 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████                 | 972 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████                 | 983 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████▎                | 993 kB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████▍                | 1.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████▋                | 1.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████▊                | 1.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████▉                | 1.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████                | 1.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████▏               | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████▍               | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████▌               | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████▋               | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████▉               | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████               | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████▏              | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████▎              | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████▌              | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████▋              | 1.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████▊              | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████              | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████              | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████▎             | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████▍             | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████▌             | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████▊             | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████▉             | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████             | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████▏            | 1.2 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████▍            | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████▌            | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████▋            | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████▉            | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████            | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████▏           | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████▎           | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████▍           | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████▋           | 1.3 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████▊           | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████           | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████           | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████▏          | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████▍          | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████▌          | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████▊          | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████▉          | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████          | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████▏         | 1.4 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████▎         | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████▌         | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████▋         | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████▉         | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████         | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████         | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████▎        | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████▍        | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████▋        | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████▊        | 1.5 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████        | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████        | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████▏       | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████▍       | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████▌       | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████▊       | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████▉       | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████       | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████▏      | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████▎      | 1.6 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████▌      | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████▋      | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████▉      | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████      | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████      | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████▎     | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████▍     | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████▋     | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████▊     | 1.7 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████▉     | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████     | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████▏    | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████▍    | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████▌    | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████▋    | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████▉    | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████    | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████▏   | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████▎   | 1.8 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████▌   | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████▋   | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████▊   | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████████   | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████████   | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████████▎  | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████████▍  | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████████▌  | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████████▊  | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |█████████████████████████████▉  | 1.9 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████████  | 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████████▏ | 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████████▍ | 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████████▌ | 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████████▋ | 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |██████████████████████████████▉ | 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████████ | 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████████▏| 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████████▎| 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████████▍| 2.0 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████████▋| 2.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |███████████████████████████████▊| 2.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████████| 2.1 MB 5.7 MB/s eta 0:00:01\r\u001B[K     |████████████████████████████████| 2.1 MB 5.7 MB/s \r\n\u001B[?25hInstalling collected packages: pip\r\n  Attempting uninstall: pip\r\n    Found existing installation: pip 21.0.1\r\n    Uninstalling pip-21.0.1:\r\n      Successfully uninstalled pip-21.0.1\r\nSuccessfully installed pip-22.0.3\r\nCollecting rouge\r\n  Downloading rouge-1.0.1-py3-none-any.whl (13 kB)\r\nRequirement already satisfied: six in /databricks/python3/lib/python3.8/site-packages (from rouge) (1.15.0)\r\nInstalling collected packages: rouge\r\nSuccessfully installed rouge-1.0.1\r\nCollecting nltk\r\n  Downloading nltk-3.7-py3-none-any.whl (1.5 MB)\r\n\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/1.5 MB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[91m━━━━━━━━\u001B[0m\u001B[91m╸\u001B[0m\u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.3/1.5 MB\u001B[0m \u001B[31m10.4 MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.5/1.5 MB\u001B[0m \u001B[31m19.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n\u001B[?25hCollecting tqdm\r\n  Downloading tqdm-4.62.3-py2.py3-none-any.whl (76 kB)\r\n\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/76.2 KB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m76.2/76.2 KB\u001B[0m \u001B[31m12.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n\u001B[?25hCollecting click\r\n  Downloading click-8.0.4-py3-none-any.whl (97 kB)\r\n\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/97.5 KB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m97.5/97.5 KB\u001B[0m \u001B[31m15.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n\u001B[?25hCollecting regex>=2021.8.3\r\n  Downloading regex-2022.1.18-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (764 kB)\r\n\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/764.6 KB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m764.6/764.6 KB\u001B[0m \u001B[31m53.0 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n\u001B[?25hRequirement already satisfied: joblib in /databricks/python3/lib/python3.8/site-packages (from nltk) (1.0.1)\r\nInstalling collected packages: regex, tqdm, click, nltk\r\nSuccessfully installed click-8.0.4 nltk-3.7 regex-2022.1.18 tqdm-4.62.3\r\nCollecting rake_nltk\r\n  Downloading rake_nltk-1.0.6-py3-none-any.whl (9.1 kB)\r\nRequirement already satisfied: nltk<4.0.0,>=3.6.2 in /databricks/python3/lib/python3.8/site-packages (from rake_nltk) (3.7)\r\nRequirement already satisfied: tqdm in /databricks/python3/lib/python3.8/site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (4.62.3)\r\nRequirement already satisfied: click in /databricks/python3/lib/python3.8/site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (8.0.4)\r\nRequirement already satisfied: regex>=2021.8.3 in /databricks/python3/lib/python3.8/site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (2022.1.18)\r\nRequirement already satisfied: joblib in /databricks/python3/lib/python3.8/site-packages (from nltk<4.0.0,>=3.6.2->rake_nltk) (1.0.1)\r\nInstalling collected packages: rake_nltk\r\nSuccessfully installed rake_nltk-1.0.6\r\nCollecting autocorrect\r\n  Downloading autocorrect-2.6.1.tar.gz (622 kB)\r\n\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/622.8 KB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[90m╺\u001B[0m \u001B[32m614.4/622.8 KB\u001B[0m \u001B[31m21.6 MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m622.8/622.8 KB\u001B[0m \u001B[31m15.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n\u001B[?25h  Preparing metadata (setup.py) ... \u001B[?25l-\b \bdone\r\n\u001B[?25hBuilding wheels for collected packages: autocorrect\r\n  Building wheel for autocorrect (setup.py) ... \u001B[?25l-\b \b\\\b \bdone\r\n\u001B[?25h  Created wheel for autocorrect: filename=autocorrect-2.6.1-py3-none-any.whl size=622380 sha256=741f56d511394f8551a335b939a54ed5d73a948165cf8396cf263efe524574b6\r\n  Stored in directory: /root/.cache/pip/wheels/72/b8/3b/a90246d13090e85394a8a44b78c8abf577c0766f29d6543c75\r\nSuccessfully built autocorrect\r\nInstalling collected packages: autocorrect\r\nSuccessfully installed autocorrect-2.6.1\r\nCollecting networkx\r\n  Downloading networkx-2.6.3-py3-none-any.whl (1.9 MB)\r\n\u001B[?25l     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.0/1.9 MB\u001B[0m \u001B[31m?\u001B[0m eta \u001B[36m-:--:--\u001B[0m\r\u001B[2K     \u001B[91m━━━━━━━━━━━━━\u001B[0m\u001B[91m╸\u001B[0m\u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m0.7/1.9 MB\u001B[0m \u001B[31m19.8 MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\r\u001B[2K     \u001B[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m\u001B[91m╸\u001B[0m \u001B[32m1.9/1.9 MB\u001B[0m \u001B[31m35.0 MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\r\u001B[2K     \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.9/1.9 MB\u001B[0m \u001B[31m25.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n\u001B[?25hInstalling collected packages: networkx\r\nSuccessfully installed networkx-2.6.3\r\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade pip\n",
    "!pip install rouge\n",
    "!pip install nltk\n",
    "!pip install rake_nltk\n",
    "!pip install autocorrect\n",
    "!pip install networkx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "9f58155a-8838-4e68-abc2-e8f25193ce85",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n[nltk_data]   Unzipping corpora/wordnet.zip.\n[nltk_data] Downloading package averaged_perceptron_tagger to\n[nltk_data]     /root/nltk_data...\n[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n[nltk_data] Downloading package punkt to /root/nltk_data...\n[nltk_data]   Unzipping tokenizers/punkt.zip.\n[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n[nltk_data]   Unzipping corpora/omw-1.4.zip.\n[nltk_data] Downloading package stopwords to /root/nltk_data...\n[nltk_data]   Unzipping corpora/stopwords.zip.\nOut[2]: True"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('punkt')\n",
    "nltk.download('omw-1.4')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "87eba0ff-746e-4def-b002-24c474b1bccb",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out[3]: <Figure size 640x480 with 0 Axes><Figure size 640x480 with 0 Axes>"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import copy\n",
    "import math\n",
    "import time\n",
    "import pickle\n",
    "import string\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "from pyspark import *\n",
    "from rouge import Rouge\n",
    "from nltk import pos_tag\n",
    "from pyspark.sql import *\n",
    "from graphframes import *\n",
    "from rake_nltk import Rake\n",
    "from datetime import datetime\n",
    "from pyspark.sql.types import *\n",
    "from autocorrect import Speller\n",
    "from nltk.corpus import stopwords\n",
    "from matplotlib.pyplot import figure\n",
    "from matplotlib import pyplot as plt\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "figure(num=None, figsize=(8, 6), dpi=80, facecolor='w', edgecolor='k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "c1383795-e65e-4115-b81b-82dcdbe2d446",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Document to process\n",
    "no_documents = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "95248a9a-6801-4550-867e-52babed81f10",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "class misc_functions:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "      \n",
    "    def word_count_dict_generator(self, count_matrix, words):\n",
    "        word_count_dict = {}\n",
    "        word_count_list = count_matrix.sum(axis=0)\n",
    "        for i in range(0, len(words)):\n",
    "            word_count_dict[words[i]] = word_count_list[i]\n",
    "        return word_count_dict\n",
    "        \n",
    "    def jaccard_simlarity(self,sen1_lemma,sen2_lemma):\n",
    "        s1_set = set(sen1_lemma)\n",
    "        s2_set = set(sen2_lemma)\n",
    "        similarity = float(len(s1_set & s2_set))/float(len(s1_set | s2_set))\n",
    "        return round(similarity,3)\n",
    "    \n",
    "    def table_to_sentence_list(self, table_name, low_doc_id = 0, upper_doc_id = no_documents):\n",
    "    #    i/p: csv_name: name of csv file\n",
    "    #             type: string\n",
    "    #    o/p: sentence_lists: list of list of sentences document wise\n",
    "    #             type: list\n",
    "    #             ex: [ [s1,s2,s3,s4], [s1,s2], [s1,s2,s3,s4,s5]]\n",
    "    #                     doc 1         doc 2         doc 3\n",
    "    #         low_doc_id: lower limit to retrive document.\n",
    "    #             type: int\n",
    "    #         upper_doc_id: upper limit to retrive document.\n",
    "    #             type: int\n",
    "        sentence_lists = []\n",
    "        wiki_how_all_sentences = sqlContext.sql(\"SELECT * FROM \" + table_name +  \" WHERE \" + \"doc_id >= \" + str(low_doc_id) + \" AND \" + \"doc_id < \" + str(upper_doc_id))\n",
    "        column_list = wiki_how_all_sentences.collect()\n",
    "        doc_id_list = []\n",
    "        count = 0\n",
    "        for row in column_list:\n",
    "#           print(str(row['doc_id']) + str(count))\n",
    "          try:\n",
    "            doc_id_list.append(int(row['doc_id']))\n",
    "          except TypeError:\n",
    "#             print(str(row['doc_id']) + \" \" + str(count)+ \" \"+ str(row['sen_id']))\n",
    "            continue\n",
    "          count += 1\n",
    "#         df = pd.read_csv(dataset_path + csv_name)\n",
    "        for i in range(low_doc_id, upper_doc_id):\n",
    "            doc = wiki_how_all_sentences.filter(wiki_how_all_sentences.doc_id==i)\n",
    "            doc_column_list = doc.collect()\n",
    "            sentence_lists.append([str(row['sentence']) for row in doc_column_list])\n",
    "        return sentence_lists\n",
    "      \n",
    "    def get_contractions(self):\n",
    "        contractions = {\"'cause\": 'because', \"ain't\": 'am not', \"aren't\": 'are not', \"can't\": 'cannot', \"can't've\": 'cannot have', \"could've\": 'could have', \"couldn't\": 'could not', \"couldn't've\": 'could not have', \"didn't\": 'did not', \"doesn't\": 'does not', \"don't\": 'do not', \"hadn't\": 'had not', \"hadn't've\": 'had not have', \"hasn't\": 'has not', \"haven't\": 'have not', \"he'd\": 'he had', \"he'd've\": 'he would have', \"he'll\": 'he will', \"he'll've\": 'he will have', \"he's\": 'he is', \"how'd\": 'how did', \"how'd'y\": 'how do you', \"how'll\": 'how will', \"how's\": 'how is', \"i'd\": 'I had', \"i'd've\": 'I would have', \"i'll\": 'I will', \"i'll've\": 'I will have', \"i'm\": 'I am', \"i've\": 'I have', \"isn't\": 'is not', \"it'd\": 'it had', \"it'd've\": 'it would have', \"it'll\": 'it will', \"it'll've\": 'it will have', \"it's\": 'it is', \"let's\": 'let us', \"ma'am\": 'madam', \"mayn't\": 'may not', \"might've\": 'might have', \"mightn't\": 'might not', \"mightn't've\": 'might not have', \"must've\": 'must have', \"mustn't\": 'must not', \"mustn't've\": 'must not have', \"needn't\": 'need not', \"needn't've\": 'need not have', \"o'clock\": 'of the clock', \"oughtn't\": 'ought not', \"oughtn't've\": 'ought not have', \"sha'n't\": 'shall not', \"shan't\": 'shall not', \"shan't've\": 'shall not have', \"she'd\": 'she had', \"she'd've\": 'she would have', \"she'll\": 'she will', \"she'll've\": 'she will have', \"she's\": 'she is', \"should've\": 'should have', \"shouldn't\": 'should not', \"shouldn't've\": 'should not have', \"so's\": 'so is', \"so've\": 'so have', \"that'd\": 'that had', \"that'd've\": 'that would have', \"that's\": 'that is', \"there'd\": 'there had', \"there'd've\": 'there would have', \"there's\": 'there is', \"they'd\": 'they had',\n",
    "\"they'd've\": 'they would have', \"they'll\": 'they will', \"they'll've\": 'they will have', \"they're\": 'they are', \"they've\": 'they have', \"to've\": 'to have', \"wasn't\": 'was not', \"we'd\": 'we had', \"we'd've\": 'we would have', \"we'll\": 'we will', \"we'll've\": 'we will have', \"we're\": 'we are', \"we've\": 'we have', \"weren't\": 'were not', \"what'll\": 'what will', \"what'll've\": 'what will have', \"what're\": 'what are', \"what's\": 'what is', \"what've\": 'what have', \"when's\": 'when is', \"when've\": 'when have', \"where'd\": 'where did', \"where's\": 'where is', \"where've\": 'where have', \"who'll\": 'who will', \"who'll've\": 'who will have', \"who's\": 'who is', \"who've\": 'who have', \"why's\": 'why is', \"why've\": 'why have', \"will've\": 'will have', \"won't\": 'will not', \"won't've\": 'will not have', \"would've\": 'would have', \"wouldn't\": 'would not', \"wouldn't've\": 'would not have', \"y'all\": 'you all', \"y'all'd\": 'you all would', \"y'all'd've\": 'you all would have', \"y'all're\": 'you all are', \"y'all've\": 'you all have',\n",
    "\"you'd\": 'you had', \"you'd've\": 'you would have', \"you'll\": 'you will', \"you'll've\": 'you will have',\"you're\": 'you are',\n",
    "\"you've\": 'you have'}\n",
    "        return contractions\n",
    "      \n",
    "    def plot_graph(self, graph):\n",
    "        edge_list= graph.edges\n",
    "        Gplot=nx.Graph()\n",
    "        for row in edge_list.select('src','dst').take(1000):\n",
    "            Gplot.add_edge(row['src'],row['dst'])\n",
    "        plt.subplot(121)\n",
    "        nx.draw(Gplot, with_labels=True, font_weight='bold')\n",
    "        return\n",
    "      \n",
    "    def checkpoint(self, gcs,last_finished):\n",
    "        file =  open('gcs0', 'wb')\n",
    "        pickle.dump(gcs, file)\n",
    "        file =  open('lastcp0', 'wb')\n",
    "        pickle.dump(last_finished, file)\n",
    "        file.close()\n",
    "        return\n",
    "      \n",
    "    def calculate_rank_cluster(self, vertex_id, graph_obj):\n",
    "        rank = 0.0\n",
    "        inter_graph = graph_obj.filterEdges(\"dst == \" + str(vertex_id))\n",
    "        edges_df = inter_graph.edges.collect()\n",
    "        for row in edges_df:\n",
    "          rank+=float(row['relationship'])\n",
    "      \n",
    "        inter_graph = graph_obj.filterEdges(\"src == \" + str(vertex_id))\n",
    "        edges_df = inter_graph.edges.collect()\n",
    "        for row in edges_df:\n",
    "          rank+=float(row['relationship'])\n",
    "      \n",
    "        return round(rank,4)  \n",
    "      \n",
    "    def rankwise_sort(self, sentence_list, sentence_ranks):\n",
    "        sorted_list = []\n",
    "        ranks = []\n",
    "        \n",
    "        for i in sentence_list:\n",
    "            ranks.append(sentence_ranks[i])\n",
    "            \n",
    "        for i in range(len(ranks)):\n",
    "            max_index = ranks.index(max(ranks))\n",
    "            ranks[max_index] = float('-inf')\n",
    "            sorted_list.append(sentence_list[max_index])\n",
    "            \n",
    "        return sorted_list\n",
    "      \n",
    "    def print_rouge_statistics(self, rouge_statistics):\n",
    "        print(\"------------------------------------------------------------------------------------\")\n",
    "        print(\"|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\")\n",
    "        print(\"------------------------------------------------------------------------------------\")\n",
    "        print(\"|  AVG  |   \"+str(rouge_statistics['rouge-1']).ljust(6, '0')+\"  |     NA     |   \"+str(rouge_statistics['rouge-2']).ljust(6, '0')+\"  |     NA     |  \"+str(rouge_statistics['rouge-l']).ljust(6, '0')+\"   |     NA     |\")\n",
    "        print(\"------------------------------------------------------------------------------------\")\n",
    "        print(\"|  MIN  |   \"+str(rouge_statistics['min_rouge_1']).ljust(6, '0')+\"  |     \"+str(rouge_statistics['min_rouge_1_doc_id']).rjust(2, '0')+\"     |   \"+str(rouge_statistics['min_rouge_2']).ljust(6, '0').ljust(6, '0')+\"  |     \"+str(rouge_statistics['min_rouge_2_doc_id']).rjust(2, '0')+\"     |  \"+str(rouge_statistics['min_rouge_l']).ljust(6, '0')+\"   |     \"+str(rouge_statistics['min_rouge_l_doc_id']).rjust(2, '0')+\"     |\")\n",
    "        print(\"------------------------------------------------------------------------------------\")\n",
    "        print(\"|  MAX  |   \"+str(rouge_statistics['max_rouge_1']).ljust(6, '0')+\"  |     \"+str(rouge_statistics['max_rouge_1_doc_id']).rjust(2, '0')+\"     |   \"+str(rouge_statistics['max_rouge_2']).ljust(6, '0')+\"  |     \"+str(rouge_statistics['max_rouge_2_doc_id']).rjust(2, '0')+\"     |  \"+str(rouge_statistics['max_rouge_l']).ljust(6, '0')+\"   |     \"+str(rouge_statistics['max_rouge_l_doc_id']).rjust(2, '0')+\"     |\")\n",
    "        print(\"------------------------------------------------------------------------------------\")\n",
    "        return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "231ad126-d420-4179-8b7c-6b89d4eaf937",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "1d42f48b-f955-4091-9987-7c9b5b23134a",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "976ac453-39f6-4df3-a84f-5b34e17e2680",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "class data_preprocessing:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    def early_preprocessing(self, doc_sentence_list):\n",
    "    #    i/p: doc_sentence_list: list of all senteneces in single document\n",
    "    #             type: list\n",
    "    #    o/p: early_porced_doc_sentence_list: list of all processed sentences of a single document\n",
    "    #             type: list\n",
    "    #    This methods performs lower casing, contractions removal, puctuation removal, \n",
    "    #    white noise removal\n",
    "        mf = misc_functions()\n",
    "        contractions = mf.get_contractions()\n",
    "        \n",
    "        early_porced_doc_sentence_list = []\n",
    "        \n",
    "        for sentence in doc_sentence_list:\n",
    "            #     Lower Casing\n",
    "            sentence = sentence.lower()   \n",
    "            #     Convert he'll -> he will\n",
    "            for word in sentence.split():\n",
    "                if word in contractions:\n",
    "                    sentence = sentence.replace(word, contractions[word])\n",
    "                    \n",
    "            if re.search(\"\\\\w*'s\", sentence ):\n",
    "                temp = sentence\n",
    "                sentence = \"\"\n",
    "                for i in temp.split():\n",
    "                    if re.search(\"\\\\w*'s\", i ):\n",
    "                        i = i.replace(\"'s\",\"\")\n",
    "                    sentence += \" \" + i \n",
    "                    \n",
    "            #    REMOVE PUNCTUATION\n",
    "            translator = str.maketrans('', '', string.punctuation)\n",
    "            sentence = sentence.translate(translator)\n",
    "             \n",
    "            #     REMOVE WHITE NOISE\n",
    "            sentence = \" \".join(sentence.split())\n",
    "            early_porced_doc_sentence_list.append(sentence)\n",
    "        return early_porced_doc_sentence_list\n",
    "        \n",
    "    def word_tokenization(self, early_porced_doc_sentence_list):\n",
    "    #    i/p: early_porced_doc_sentence_list: list of all processed senteteces in single document using 'early_preprocessing_wikihow' method\n",
    "    #             type: list\n",
    "    #    o/p: sentence_tokens_list: list of list containing tokens of sentences\n",
    "    #             type: list\n",
    "    #    example: i/p: ['this example','this test']\n",
    "    #             o/p: [['this','example'],['this','test']]\n",
    "        no_sentences = len(early_porced_doc_sentence_list)\n",
    "        sentence_tokens_list = []\n",
    "        for sentence in early_porced_doc_sentence_list:\n",
    "            sentence_tokens_list.append(word_tokenize(sentence))\n",
    "        return sentence_tokens_list    \n",
    "    \n",
    "    def lemmatization(self, sentence_tokens_list):\n",
    "    #    i/p: sentence_tokens_list: list of list containing tokens of sentences\n",
    "    #             type: list\n",
    "    #    o/p: sentence_lemma_list: list of list containing lemmas of individual sentence\n",
    "    #             type: list\n",
    "    #    example: i/p: [['i', 'am', 'playimg'], ['they', 'have', 'studied']]\n",
    "    #             o/p: [['i', 'be', 'play'], ['they', 'have', 'study']]\n",
    "        lemmatizer = WordNetLemmatizer()\n",
    "        sentence_lemma_list = []\n",
    "        for token_list in sentence_tokens_list:\n",
    "            lemmas = [lemmatizer.lemmatize(word, pos ='v') for word in token_list]\n",
    "            lemmas = [e for e in lemmas if e.isalnum()]\n",
    "            if lemmas != []:\n",
    "                sentence_lemma_list.append(lemmas)\n",
    "        return sentence_lemma_list\n",
    "    \n",
    "    def stop_word_removal(self, sentence_lemma_list):\n",
    "    #    i/p: sentence_lemma_list: list of list containing lemmas of individual sentence\n",
    "    #             type: list\n",
    "    #    o/p: sentence_lemma_list: list of list containing lemmas of individual sentence\n",
    "    #         but without stop words.\n",
    "    #             type: list\n",
    "    #    example: i/p: [['i','am','playing'],['they','have','study','hard']]\n",
    "    #             o/p: [['play'], ['hard','study']]\n",
    "        stop_words = set(stopwords.words('english'))\n",
    "        for index in range(0, len(sentence_lemma_list)):\n",
    "            if sentence_lemma_list[index] == []:\n",
    "              continue\n",
    "            temp = list(set(sentence_lemma_list[index])-stop_words)\n",
    "            if temp != []:\n",
    "                sentence_lemma_list[index] = temp\n",
    "        return sentence_lemma_list\n",
    "    \n",
    "    def pos_tagging(self, sentence_lemma_list):\n",
    "    #    i/p: sentence_lemma_list: list of list containing lemmas of individual sentence\n",
    "    #         after stopword removal\n",
    "    #             type: list\n",
    "    #    o/p: nouns: list of nouns in a document\n",
    "    #             type: list\n",
    "    #         words: list of unique words in a document\n",
    "    #             type: list\n",
    "    #    example: i/p: [['i', 'be', 'play','extra','playground'], \n",
    "    #                   ['they', 'have', 'study','hard','extra','exam','college']]\n",
    "    #             o/p: Nouns: ['play', 'college', 'playground'], \n",
    "    #                  Words: ['play', 'college', 'extra', 'be', 'study', 'exam', 'i', 'have', 'they', 'hard', 'playground']\n",
    "        nouns = []\n",
    "        words = []\n",
    "        for lemma_list in sentence_lemma_list:\n",
    "            words = list(set(words).union(set(lemma_list)))\n",
    "        for i in pos_tag(words):\n",
    "            if (i[1] == 'NN' or i[1] == 'NNP') and i[0] not in nouns:\n",
    "                nouns.append(i[0])\n",
    "        return nouns, words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "e762a73f-0599-4dc3-8da6-e431fd164ab9",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "### Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "7e91c093-0bb4-4282-a430-5c685e4191cd",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "class feature_extraction:\n",
    "    mf = None\n",
    "    def __init__(self):\n",
    "        self.mf = misc_functions()\n",
    "        pass\n",
    "      \n",
    "    def word_freq_counter(self, sentence_lemma_list, words):\n",
    "    #    i/p: sentence_lemma_list: list of list containing lemmas of individual sentence\n",
    "    #         after stopword removal\n",
    "    #             type: list\n",
    "    #         words: list of unique words in a document\n",
    "    #             type: list\n",
    "    #    o/p: count_matrix: count of each word per sentence\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array\n",
    "    #    example: i/p: [['i', 'i' , 'be', 'play','extra','playground'], \n",
    "    #                  ['they', 'have', 'study','have','college']], \n",
    "    #                  ['play', 'college', 'extra', 'be', 'study', 'i', 'have', 'they', 'playground']\n",
    "    #             o/p: [[1. 0. 1. 1. 0. 2. 0. 0. 1.]\n",
    "    #                  [0. 1. 0. 0. 1. 0. 2. 1. 0.]]\n",
    "        no_sentences = len(sentence_lemma_list)\n",
    "        no_words = len(words)\n",
    "        count_matrix = np.empty([no_sentences, no_words])\n",
    "        for i in range(0,no_sentences):\n",
    "            lemmas = sentence_lemma_list[i]\n",
    "            for j in range(0,no_words):\n",
    "                lemma = words[j]\n",
    "                count_matrix[i][j] = lemmas.count(lemma)\n",
    "        return count_matrix\n",
    "    \n",
    "    def tf_idf_score(self,count_matrix):\n",
    "    #    i/p: count_matrix: count of each word per sentence\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array\n",
    "    #    o/p: tf_idf_scores: tf-idf score of each word\n",
    "    #             type: list\n",
    "    #    Total number of words in the document\n",
    "        total_words_in_document = count_matrix.sum()\n",
    "    #    Number of times word appears in a sentence\n",
    "        word_count = count_matrix.sum(axis=0)\n",
    "    #    TF(word) = (Number of times word appears in a sentence) / (Total number of words in the document).\n",
    "        tf_scores = [ i/float(total_words_in_document) for i in word_count]\n",
    "    #    Total number of sentences\n",
    "        total_no_sentences = count_matrix.shape[0]\n",
    "    #    Number of sentences with word in it\n",
    "        no_sentences = np.count_nonzero(count_matrix,axis=0)\n",
    "    #    IDF(word) = log_e(Total number of sentences / Number of sentences with word in it)\n",
    "        idf_scores_wo_log = [float(total_no_sentences)/i for i in no_sentences]\n",
    "        idf_scores = [math.log(i) for i in idf_scores_wo_log]\n",
    "    #    tf-idf = tf*idf\n",
    "        tf_idf_scores = [i*j for i,j in zip(tf_scores,idf_scores)]\n",
    "        return tf_idf_scores\n",
    "    \n",
    "    def keyword_extraction(self, method, input_parameter, words = []):\n",
    "    #    i/p: method: method name to extract keyword TF-TDF or RAKE\n",
    "    #             type: stirng\n",
    "    #         input_parameter: if TFIDF -> list of tf-idf score of each words\n",
    "    #                          if RAKE -> list of sentence lemmas in a single document   \n",
    "    #         words: list of unique words in a document (only for tf-idf)\n",
    "    #             type: list\n",
    "    #    o/p: keywords: keywords/phrases list for given document.\n",
    "    #             type: list\n",
    "        if method == \"tfidf\":\n",
    "            tf_keywords = []\n",
    "            tf_idf_scores = copy.deepcopy(input_parameter)\n",
    "            for i in range(0, len(tf_idf_scores)):\n",
    "                # Getting max score from list                \n",
    "                max_score = max(tf_idf_scores)\n",
    "                # Getting index of max score\n",
    "                max_score_index = tf_idf_scores.index(max_score)\n",
    "                # Getting word with tthe max score\n",
    "                word = words[max_score_index]\n",
    "                # Append word in final list\n",
    "                tf_keywords.append(word)\n",
    "                # Assigning samallest value to already appended word\n",
    "                # To prevent the selection of the same word in the next iteration\n",
    "                tf_idf_scores[max_score_index] = float('-inf')\n",
    "            return tf_keywords\n",
    "        elif method == \"rake\":\n",
    "            kw_sentence_lemma_list = copy.deepcopy(input_parameter)\n",
    "            text = \"\"\n",
    "            for lemma_list in kw_sentence_lemma_list:\n",
    "                for lemma in lemma_list:\n",
    "                    text += \" \" + lemma\n",
    "                text += \". \"\n",
    "            r = Rake()\n",
    "            # Getting score\n",
    "            r.extract_keywords_from_text(text)\n",
    "            # Sorting words based on the scores\n",
    "            rake_keyphrases = r.get_ranked_phrases()\n",
    "            return rake_keyphrases\n",
    "    \n",
    "    def sentenece_similarity_calculator(self,sentence_lemma_list):\n",
    "    #    i/p: sentence_lemma_list: method name to extract keyword TF-TDF or RAKE\n",
    "    #             type: list\n",
    "    #    o/p: similarity_matrix: numpy array of similarity score between two sentences.\n",
    "    #             type: numpy array\n",
    "        no_sentences = len(sentence_lemma_list)\n",
    "        similarity_matrix = np.empty([no_sentences, no_sentences])\n",
    "        for i in range(0, no_sentences):\n",
    "            for j in range(i, no_sentences):\n",
    "                if i == j:\n",
    "                    similarity_matrix[i][j] = 1\n",
    "                    continue\n",
    "                similarity_score = self.mf.jaccard_simlarity(sentence_lemma_list[i],sentence_lemma_list[j])\n",
    "                similarity_matrix[i][j] = similarity_score\n",
    "                similarity_matrix[j][i] = similarity_score\n",
    "        return similarity_matrix\n",
    "    \n",
    "    def word_weight_calculation(self, nouns, keywords, count_matrix, words):\n",
    "    #    i/p: nouns: list of nouns in a document\n",
    "    #             type: list\n",
    "    #         keywords: list of keywords/phrases in in a document\n",
    "    #             type: list\n",
    "    #         count_matrix: count of each word per sentence\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array \n",
    "    #         keywords: list of words in a document\n",
    "    #             type: list\n",
    "    #    o/p: keywords: keywords list for given document.\n",
    "    #             type: list\n",
    "        word_weight_dict = {}\n",
    "        noun_count = []\n",
    "        keyword_count = []\n",
    "        total_noun_count = 0\n",
    "        total_keyword_count = 0\n",
    "        word_count_dict = self.mf.word_count_dict_generator(count_matrix, words)\n",
    "        temp_index = 0\n",
    "        #    Getting frequency of each noun in a document     \n",
    "        for noun in nouns:\n",
    "            total_noun_count += word_count_dict[noun]\n",
    "        #    Getting frequency of each keyword in a document     \n",
    "        for keyphrase in keywords:\n",
    "            for keyword in keyphrase.split(\" \"):\n",
    "                if keyword.isalnum():\n",
    "                    total_keyword_count += word_count_dict[keyword]\n",
    "       #    Calculation of each noun in a document          \n",
    "        for noun in nouns:\n",
    "            word_weight_dict[noun] = word_count_dict[noun]/float(total_noun_count)\n",
    "            \n",
    "        for keyphrase in keywords:\n",
    "            for keyword in keyphrase.split(\" \"):\n",
    "                if keyword.isalnum():\n",
    "                    if keyword in word_weight_dict and keyword in nouns:\n",
    "                        word_weight_dict[keyword] += word_count_dict[keyword]/float(total_keyword_count)\n",
    "                        continue\n",
    "                    if keyword in word_weight_dict and keyword not in nouns:\n",
    "                        continue\n",
    "                    word_weight_dict[keyword] = word_count_dict[keyword]/float(total_keyword_count)\n",
    "                \n",
    "        return word_weight_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "cd8dfa55-4caf-4f61-913b-7d6906b3a2b7",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "# Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "06d0b08c-10c3-4c05-af41-a2b066d2703b",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "class rank_calculation:\n",
    "    def __init__(self):\n",
    "        return\n",
    "    \n",
    "    def graph_generation(self, similarity_matrix):\n",
    "    #  1. ADDED REMOVAL OF EDGE WITH 0 WEIGHT  \n",
    "    #  2. ONLY ONE SIDED EDGE IS USED      \n",
    "    #  3. THRESOLD IS SET TO 75% Q3\n",
    "    \n",
    "    \n",
    "    #    i/p: similarity_matrix: numpy array of similarity score between two sentences.\n",
    "    #             type: numpy array\n",
    "    #    o/p: sentence_graph: sentence based graph\n",
    "    #             type: graphframe\n",
    "    #             vertces: sentence id\n",
    "    #             edges: jaccard similarity\n",
    "    \n",
    "    \n",
    "        no_sentences = similarity_matrix.shape[0]\n",
    "        \n",
    "        # Checking for identity matrix\n",
    "        row_ele_sum = np.sum(similarity_matrix, axis = 1)\n",
    "        \n",
    "        if no_sentences == np.sum(row_ele_sum):\n",
    "          return None\n",
    "        \n",
    "        # Generating list [0, 1, 2, 3, 4, 5, ...., no_sentences-1]\n",
    "        vertices_list = [i for i in range(0, no_sentences)]\n",
    "#         print(vertices_list)\n",
    "        # Generating Spark SQL DataFrame from list\n",
    "        vertices = spark.createDataFrame(vertices_list, IntegerType())\n",
    "        # Changing column name value->id. 'value' is default one.\n",
    "        vertices = vertices.withColumnRenamed(\"value\",\"id\")\n",
    "        # Defining structure of Edge Spark DataFrame -> (src: source (sentence_id), dst: destination (sentence_id), relationship: similarity score)\n",
    "        edges_schema = StructType([StructField(\"src\", IntegerType()),StructField(\"dst\", IntegerType()),StructField(\"relationship\", FloatType())])\n",
    "        edges_list = []\n",
    "        inter_edges_list = []\n",
    "        weights = []\n",
    "        # Generating edge_list to convert it into Spark SQL DataFrame:\n",
    "        # Where each element (child list) in list (parent) represents single row of Edge Spark DataFrame \n",
    "        # Ex: [[0, 1, 0.2], [0, 2, 0.6], [1, 0, 0.2], [1, 2, 0.5], [2, 0, 0.6],[2, 1, 0.5]]\n",
    "        for i in range(0, no_sentences):\n",
    "          for j in range(i+1, no_sentences):\n",
    "            if i==j or float(similarity_matrix[i][j]) == 0.0:\n",
    "              continue\n",
    "            inter_edges_list.append([i,j,float(similarity_matrix[i][j])])\n",
    "            weights.append(float(similarity_matrix[i][j]))\n",
    "        \n",
    "        \n",
    "        # Uncomment This Part TO Remove edges WRT thresold\n",
    "        weights.sort()\n",
    "        thresold = np.percentile(weights, 75, interpolation='midpoint')\n",
    "        edges_list = [i for i in inter_edges_list if i[2] > thresold]\n",
    "        \n",
    "#         # Uncomment This Part TO not have thresold\n",
    "#         edges_list = inter_edges_list\n",
    "        \n",
    "        # Generating Spark SQL DataFrame from edges_list  \n",
    "        edges = spark.createDataFrame(edges_list,schema=edges_schema)\n",
    "        # Generating Graph\n",
    "        sentence_graph = GraphFrame(vertices, edges)\n",
    "        \n",
    "        return sentence_graph\n",
    "        \n",
    "      \n",
    "    def eq_2(self, A, P, d):\n",
    "    #    i/p: A: Similarity matrix between sentences\n",
    "    #             type: numpy array\n",
    "    #             row: sentences    column: senteces\n",
    "    #         P: List of ranks of sentences\n",
    "    #             type: list\n",
    "    #         d: Damping Factor\n",
    "    #             type: 0<d<1\n",
    "    #    o/p: new_P: List of calculated ranks of sentences\n",
    "    #             type: list\n",
    "        new_P = copy.deepcopy(P)\n",
    "        for i in range(0, len(P)):          #no_sentences == len(P)\n",
    "            rank = 1 - d\n",
    "            for j in range(0, len(P)):\n",
    "                if j == i or A[i][j] == 0:\n",
    "                    continue\n",
    "        #        a_j_k = A.sum(axis = 0)[j]\n",
    "                a_j_k = A[j].sum()\n",
    "                rank += (d)*((float(A[i][j])/float(a_j_k))*new_P[j])\n",
    "            new_P[i] = rank\n",
    "        return new_P\n",
    "\n",
    "    def eq_3(self, B, P, words, word_weight_dict):\n",
    "    #    i/p: B: Normalized count matrix of words per sentences\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array\n",
    "    #         P: List of ranks of sentences\n",
    "    #             type: list\n",
    "    #    o/p: Q: List of word weightage\n",
    "    #             type: list\n",
    "        Q = []\n",
    "        for j in range(0,B.shape[1]):      #B.shape[1] == number of words in document\n",
    "            num = 0\n",
    "            dino = B.sum(axis = 0)[j]\n",
    "            word = words[j]\n",
    "            for i in range(0,len(P)):      #no_sentences == len(P)\n",
    "                num += B[i][j]*P[i]\n",
    "            sentence_based_weight = float(num)/float(dino)\n",
    "            word_dict_weight = 0\n",
    "#             if word in word_weight_dict:\n",
    "#                 word_dict_weight = word_weight_dict[word]\n",
    "            Q.append(sentence_based_weight + word_dict_weight)\n",
    "        return Q\n",
    "    \n",
    "    def eq_4(self, B,Q):\n",
    "    #    i/p: B: Normalized count matrix of words per sentences\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array\n",
    "    #         Q: List of word weightage\n",
    "    #             type: list\n",
    "    #    o/p: P_star: List of calculated ranks of sentences based on word weight\n",
    "    #             type: list\n",
    "        P_star = []\n",
    "        no_sentences = B.shape[0]\n",
    "        for i in range(0,no_sentences):\n",
    "            num = 0\n",
    "            dino = B[i].sum()\n",
    "            for j in range(0,len(Q)):      #number of words in document == len(Q)\n",
    "                num += B[i][j]*Q[j]\n",
    "            P_star.append(float(num)/float(dino))\n",
    "        return P_star\n",
    "    \n",
    "    def eq_5(self, P,P_star,alpha = 0.6):\n",
    "    #    i/p: P: List of ranks of sentences\n",
    "    #             type: list\n",
    "    #         P_star: List of calculated ranks of sentences based on word weight\n",
    "    #             type: list\n",
    "    #         alpha: Ratio between page rank value from eq_2 and eq_4\n",
    "    #             type: 0<alpha<1\n",
    "    #    o/p: P: List of calculated ranks of sentences or linear combination eq_2 and eq_4\n",
    "    #             type: list\n",
    "        for i in range(0,len(P)):\n",
    "            P[i] = alpha*P[i] + (1-alpha)*P_star[i]\n",
    "        return P\n",
    "    \n",
    "    def get_diff(self, P, P_tminus1):\n",
    "    #    i/p: P: List of ranks of sentences from current iteration\n",
    "    #             type: list\n",
    "    #         P_tminus1: List of ranks of sentences from previous iteration\n",
    "    #             type: list\n",
    "    #    o/p: diff: Linear difference between P and P_tminus1\n",
    "    #             type: float\n",
    "        P_array = np.array(P)\n",
    "        P_tminus1_array = np.array(P_tminus1)\n",
    "        diff = (np.linalg.norm(P_array - P_tminus1_array))**2\n",
    "        return diff\n",
    "    \n",
    "    def get_A(self, similarity_matrix):\n",
    "    #    i/p: similarity_matrix: matrix of simalarity scores between sentences\n",
    "    #             type: numpy array\n",
    "    #    o/p: A: Copy of similarity matrix\n",
    "    #             row: sentences    column: sentences\n",
    "    #             type: numpy array\n",
    "        A = copy.deepcopy(similarity_matrix)\n",
    "        return A\n",
    "    \n",
    "    def get_B(self, C):\n",
    "    #    i/p: C: matrix of count of words per sentence\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array\n",
    "    #    o/p: B: Normalized count matrix of words per sentences\n",
    "    #             type: numpy array\n",
    "        B = np.empty([C.shape[0], C.shape[1]])\n",
    "        for i in range(0, B.shape[0]):\n",
    "            row_sum = C[i].sum()\n",
    "            for j in range(0, B.shape[1]):\n",
    "                B[i][j] = float(C[i][j])/float(row_sum)\n",
    "        return B\n",
    "    \n",
    "    def get_C(self, count_matrix):\n",
    "    #    i/p: count_matrix: matrix of count of words per sentence\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array\n",
    "    #    o/p: C: Copy of count matrix\n",
    "    #             type: numpy array\n",
    "        C = copy.deepcopy(count_matrix)\n",
    "        return C\n",
    "    \n",
    "    def get_ranks(self, A, B, words, word_weight_dict):\n",
    "    #    i/p: A: matrix of simalarity scores between sentences\n",
    "    #             row: sentences    column: sentences\n",
    "    #             type: numpy array\n",
    "    #         B: Normalized count matrix of words per sentences\n",
    "    #             row: sentences    column: words\n",
    "    #             type: numpy array\n",
    "    #    o/p: P_t: List of calculated final ranks of sentences\n",
    "    #             type: list\n",
    "        no_sentences = A.shape[0]\n",
    "        alpha = 0.6\n",
    "        d=0.8\n",
    "        P_tminus1 = [1]*no_sentences\n",
    "        while True:\n",
    "            P = self.eq_2(A,P_tminus1,d)\n",
    "            Q = self.eq_3(B,P, words, word_weight_dict)\n",
    "            P_star = self.eq_4(B,Q)\n",
    "            P_t = self.eq_5(P,P_star,alpha)\n",
    "            if self.get_diff(P, P_tminus1) < 10**-5:\n",
    "                break\n",
    "            P_tminus1 = copy.deepcopy(P_t)\n",
    "        return P_t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "0ed4bc34-c54d-4d4a-963a-93d06087689c",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "# Post Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "c374a416-dc03-4c27-b37b-91caaee6a556",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "class summary_post_processing:\n",
    "    def __init__(self):\n",
    "        mf = misc_functions()\n",
    "        return\n",
    "    \n",
    "    def get_indexes(self, sentence_ranks, no_summary_sentences):\n",
    "    #    i/p: sentence_ranks = List of ranks of sentences.\n",
    "    #             type: list\n",
    "    #         no_summary_sentences: No. of sentences in reference summary.\n",
    "    #             type: int\n",
    "    #    o/p: sentence_index: Indexes of top-N ranked sentences. Where N =  no_summary_sentences\n",
    "    #             type: list \n",
    "    \n",
    "        sentence_index = []\n",
    "        temp_list = copy.deepcopy(sentence_ranks)\n",
    "        \n",
    "        # No. of sentences in a document.\n",
    "        no_sentences = len(sentence_ranks)\n",
    "        if no_summary_sentences > no_sentences:\n",
    "            no_summary_sentences = no_sentences\n",
    "        for j in range(0, no_summary_sentences):\n",
    "            index = temp_list.index(max(temp_list))\n",
    "            sentence_index.append(index)\n",
    "            temp_list[index] = 0\n",
    "            \n",
    "        # Arranging sentences indexes\n",
    "        sentence_index.sort()\n",
    "        \n",
    "        return sentence_index\n",
    "    \n",
    "    def summary_generation(self, sentence_list, sentence_index):\n",
    "    #    i/p: sentence_list = List of ranks of sentences.\n",
    "    #             type: list\n",
    "    #         sentence_index: Indexes of top-N ranked sentences. Where N =  no. of senteces in a\n",
    "    #                         ref. summary\n",
    "    #             type: list\n",
    "    #    o/p: generated_summary: Model generated summary.\n",
    "    #             type: string\n",
    "    \n",
    "        generated_summary = \"\"\n",
    "      \n",
    "        for index in sentence_index:\n",
    "            generated_summary += sentence_list[index]\n",
    "            generated_summary += \" \"\n",
    "            \n",
    "        return generated_summary\n",
    "    \n",
    "    def ref_summary(self, unprocessed_ref_summary):\n",
    "    #    i/p: unprocessed_ref_summary = Reference summary which requires mionr processing.\n",
    "    #             type: string\n",
    "    #    o/p: reference_summary: Procesed reference summary.\n",
    "    #             type: string\n",
    "    \n",
    "        reference_summary = copy.deepcopy(unprocessed_ref_summary)\n",
    "        reference_summary.replace('.,','.')\n",
    "        \n",
    "        return reference_summary\n",
    "    \n",
    "    def graph_cluster_generation(self, graph_obj):\n",
    "    #    i/p: graph_obj = Sentence-based graph of a sentence.\n",
    "    #             type: GraphFrame object\n",
    "    #    o/p: clusters: Map between clusters and sentence ids.\n",
    "    #             type: Dictionary\n",
    "    #             ex: {1: [2,8,3], 2: [1,7,9], 3: [4,5,6]}\n",
    "    \n",
    "      if graph_obj == None:\n",
    "        return None\n",
    "      \n",
    "      clusters = {}\n",
    "      cluster_df = graph_obj.connectedComponents()\n",
    "      cluster_df_rows = cluster_df.collect()\n",
    "      \n",
    "      keys = []\n",
    "      vertex_ids = []\n",
    "      \n",
    "      for row in cluster_df_rows:\n",
    "        keys.append(int(row['component']))\n",
    "        vertex_ids.append(int(row['id']))\n",
    "      \n",
    "      for key,vertex in zip(keys,vertex_ids):\n",
    "        if key not in clusters:\n",
    "          clusters[key] = [vertex]\n",
    "          continue  \n",
    "        clusters[key].append(vertex)\n",
    "        \n",
    "      return clusters\n",
    "    \n",
    "    def rearrange_cluster(self, cluster):\n",
    "      arranged_cluster = {}\n",
    "      new_cluster = []\n",
    "      index = 0\n",
    "    \n",
    "      for cid in list(cluster.keys()):\n",
    "      \n",
    "        if len(cluster[cid]) < 2:\n",
    "          new_cluster.append(cluster[cid][0])\n",
    "          continue\n",
    "        \n",
    "        arranged_cluster[index] = copy.deepcopy(cluster[cid])\n",
    "        index += 1\n",
    "      \n",
    "      if new_cluster != []:\n",
    "        arranged_cluster[index] = new_cluster\n",
    "        \n",
    "      return arranged_cluster\n",
    "    \n",
    "    def post_sentence_rank(self, graph_obj, word_sentence_ranks = []):\n",
    "      mf = misc_functions()\n",
    "      cluster_sentence_ranks = []\n",
    "      sentence_ranks = []\n",
    "      vertices = []\n",
    "      \n",
    "      vertices_df = graph_obj.vertices.collect()\n",
    "      for row in vertices_df:\n",
    "        vertices.append(int(row['id']))\n",
    "        \n",
    "      vertices.sort()\n",
    "      if word_sentence_ranks == []:\n",
    "        print(\"I'm here!!!!\")\n",
    "        word_sentence_ranks = [0] * len(vertices)\n",
    "        \n",
    "      #Calculating Rank of vertex based on the cluster\n",
    "      for vertex_id in vertices:\n",
    "        cluster_sentence_ranks.append(mf.calculate_rank_cluster(vertex_id, graph_obj))\n",
    "      \n",
    "      #Combining clustering rank with the word-influced rank\n",
    "      for index in range(len(vertices)):\n",
    "        sentence_ranks.append(cluster_sentence_ranks[index] + word_sentence_ranks[index])\n",
    "        \n",
    "      return sentence_ranks\n",
    "    \n",
    "    def sentence_selection_cluster(self, cluster, no_summary_sentences, graph_obj, word_sentence_ranks = []):\n",
    "      mf = misc_functions()\n",
    "      sentence_index = []\n",
    "      cluster_sentence_ranks = []\n",
    "      \n",
    "      no_sentence_doc = len(word_sentence_ranks)\n",
    "      \n",
    "      if no_summary_sentences > no_sentence_doc:\n",
    "        no_summary_sentences = no_sentence_doc\n",
    "        \n",
    "      try:\n",
    "        #Rearranging clusters\n",
    "        cluster = self.rearrange_cluster(cluster)\n",
    "\n",
    "        #Getting final rank of sentence by combinig cluster rank and word influnced rank\n",
    "        sentence_ranks = self.post_sentence_rank(graph_obj, word_sentence_ranks)\n",
    "\n",
    "        #Sorting nodes in lists of cluster\n",
    "        for cid, sentence_list in zip(list(cluster.keys()),list(cluster.values())):\n",
    "          cluster[cid] = mf.rankwise_sort(sentence_list, sentence_ranks)\n",
    "\n",
    "        no_clusters = len(list(cluster.keys()))\n",
    "        remaining_sentences = (no_summary_sentences)%(no_clusters)\n",
    "        sentence_per_cluster = math.floor(no_summary_sentences/no_clusters)\n",
    "        tempp = remaining_sentences\n",
    "        temp_cluster = copy.deepcopy(cluster)\n",
    "\n",
    "        for cid in list(temp_cluster.keys()):\n",
    "\n",
    "          if sentence_per_cluster > len(temp_cluster[cid]):\n",
    "\n",
    "            diff = sentence_per_cluster - len(temp_cluster[cid])\n",
    "            remaining_sentences += diff\n",
    "            sentence_index += temp_cluster[cid]\n",
    "            temp_cluster[cid] = []\n",
    "            continue\n",
    "\n",
    "          for i in range(sentence_per_cluster):\n",
    "            sentence_index.append(temp_cluster[cid].pop(0))\n",
    "\n",
    "        reverse_index = -1\n",
    "        while remaining_sentences != 0:\n",
    "          index = list(temp_cluster.keys())[reverse_index]\n",
    "\n",
    "          if temp_cluster[index] == []:\n",
    "            reverse_index -= 1\n",
    "            continue\n",
    "\n",
    "          if remaining_sentences >= len(temp_cluster[index]):\n",
    "            sentence_index += temp_cluster[index]\n",
    "            remaining_sentences -= len(temp_cluster[index])\n",
    "            temp_cluster[index] = []\n",
    "            reverse_index -= 1\n",
    "            continue\n",
    "\n",
    "          if remaining_sentences < len(temp_cluster[index]):\n",
    "            for i in range(remaining_sentences):\n",
    "              sentence_index.append(temp_cluster[index].pop(0))\n",
    "            remaining_sentences = 0\n",
    "            reverse_index -= 1\n",
    "            continue\n",
    "\n",
    "      except AttributeError:\n",
    "        sentence_index = self.get_indexes(word_sentence_ranks, no_summary_sentences)\n",
    "          \n",
    "      sentence_index.sort()\n",
    "      \n",
    "      return sentence_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "e9a2b549-9422-4bfc-8ad4-a6b1130809cd",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "e2ae90bf-90dc-4644-ab32-94ebc6abbbc5",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "class summary_evaluation:\n",
    "    def __init__(self):\n",
    "        return\n",
    "    \n",
    "    def rouge_measure(self, generated_summary, reference_summary, sub_cat = 'r'):\n",
    "    #    i/p: sentence_ranks = List of ranks of sentences.\n",
    "    #             type: list\n",
    "    #         reference_summary: Procesed reference summary.\n",
    "    #             type: string\n",
    "    #         sub_cat: Sub-category for rouge.\n",
    "    #                  'r' -> Recall, 'p' -> Precision, 'f' -> F1-score\n",
    "    #             type: charcter\n",
    "    #    o/p: rouge_scores_dict: Contains all three rouge scores: rouge-1, rouge-2, rouge-l\n",
    "    #             type: dictionary\n",
    "        rouge = Rouge()\n",
    "        rouge_scores_dict = {}\n",
    "        scores = rouge.get_scores(generated_summary, reference_summary, avg=True)\n",
    "        rouge_scores_dict['rouge-1'] = scores['rouge-1'][sub_cat]\n",
    "        rouge_scores_dict['rouge-2'] = scores['rouge-2'][sub_cat]\n",
    "        rouge_scores_dict['rouge-l'] = scores['rouge-l'][sub_cat]\n",
    "        return rouge_scores_dict\n",
    "      \n",
    "    def rouge_statistics(self,m_rouge_scores, need_list = False):\n",
    "    #    i/p: rouge_scores_dict: Contains all three rouge scores: rouge-1, rouge-2, rouge-l\n",
    "    #             type: dictionary\n",
    "    #         need_list: Set true if want tje list of all  rouge scores\n",
    "    #             type: dictionary\n",
    "    #    o/p: rouge_stats_dict: Min, Max, Average of all three rouge scores: rouge-1, rouge-2, rouge-l for all documents\n",
    "    #             type: dictionary\n",
    "    #         rouge_list: Contains list of score lists for rouge-1, rouge-2, rouge-l\n",
    "    #             type: list\n",
    "    #    If need_list is true then o/p will be a list contains rouge_stats_dict and rouge_list\n",
    "    \n",
    "      rouge_stats_dict = {}\n",
    "      rouge_1 = []\n",
    "      rouge_2 = []\n",
    "      rouge_l = []\n",
    "      \n",
    "      no_documents = len(m_rouge_scores)\n",
    "      \n",
    "      for dicti in m_rouge_scores:\n",
    "        rouge_1.append(dicti[\"rouge-1\"])\n",
    "        rouge_2.append(dicti['rouge-2'])\n",
    "        rouge_l.append(dicti[\"rouge-l\"])\n",
    "        \n",
    "      rouge_stats_dict[\"rouge-1\"] = round((sum(rouge_1)/no_documents),4)\n",
    "      rouge_stats_dict[\"rouge-2\"] = round((sum(rouge_2)/no_documents),4)\n",
    "      rouge_stats_dict[\"rouge-l\"] = round((sum(rouge_l)/no_documents),4)\n",
    "      \n",
    "      rouge_stats_dict[\"max_rouge_1\"] = round(max(rouge_1),4)\n",
    "      rouge_stats_dict[\"max_rouge_2\"] = round(max(rouge_2),4)\n",
    "      rouge_stats_dict[\"max_rouge_l\"] = round(max(rouge_l),4)\n",
    "      \n",
    "      rouge_stats_dict[\"min_rouge_1\"] = round(min(rouge_1),4)\n",
    "      rouge_stats_dict[\"min_rouge_2\"] = round(min(rouge_2),4)\n",
    "      rouge_stats_dict[\"min_rouge_l\"] = round(min(rouge_l),4)\n",
    "      \n",
    "      rouge_stats_dict[\"max_rouge_1_doc_id\"] = round(rouge_1.index(max(rouge_1)),4)\n",
    "      rouge_stats_dict[\"max_rouge_2_doc_id\"] = round(rouge_2.index(max(rouge_2)),4)\n",
    "      rouge_stats_dict[\"max_rouge_l_doc_id\"] = round(rouge_l.index(max(rouge_l)),4)\n",
    "      \n",
    "      rouge_stats_dict[\"min_rouge_1_doc_id\"] = round(rouge_1.index(min(rouge_1)),4)\n",
    "      rouge_stats_dict[\"min_rouge_2_doc_id\"] = round(rouge_2.index(min(rouge_2)),4)\n",
    "      rouge_stats_dict[\"min_rouge_l_doc_id\"] = round(rouge_l.index(min(rouge_l)),4)\n",
    "      \n",
    "      if need_list:\n",
    "        return [rouge_stats_dict, [rouge_1, rouge_2, rouge_l]]\n",
    "      \n",
    "      return rouge_stats_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "507dcfbf-6abf-4136-9328-7f0374fdab78",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "# Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "cb64d59a-4c20-45fa-b11f-e9e55ee2ade4",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "mf = misc_functions()\n",
    "dp = data_preprocessing()\n",
    "fe = feature_extraction()\n",
    "rc = rank_calculation()\n",
    "spp = summary_post_processing()\n",
    "se = summary_evaluation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "2815dfa7-bbde-42cd-9848-6e4bab366405",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "sentence_table_name = 'wiki_how_all_sentences'\n",
    "summary_table_name = 'wiki_how_all_summary'\n",
    "m_doc_id = []\n",
    "m_sentence_lemma_list = []\n",
    "m_kw_sentence_lemma_list = []\n",
    "m_nouns = []\n",
    "m_words = []\n",
    "m_count_matrix = []\n",
    "m_keywords = []\n",
    "m_word_weight_dict = []\n",
    "m_similarity_matrix = []\n",
    "mul_doc_ranks = []\n",
    "m_sentence_index = []\n",
    "m_rouge_scores = []\n",
    "m_graph = []\n",
    "m_tf_idf_score = []\n",
    "rejected_sentences = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "5393f143-bebb-4bce-a322-d9be3bf9ab20",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "m_generated_summary = []\n",
    "m_reference_summary = []\n",
    "df_summary = sqlContext.sql(\"SELECT * FROM \"+summary_table_name)\n",
    "summary_collection = df_summary.collect()\n",
    "m_unprocessed_ref_summary = [str(row['summary']) for row in summary_collection]\n",
    "no_summary_sentences = [int(row['sentence_count']) for row in summary_collection]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "bc55beff-b052-4297-be8d-b432571251b0",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "sentence_lists = mf.table_to_sentence_list(sentence_table_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "c96c301e-a61b-411b-8b50-236bd6d0aca5",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "# DATA PRE-PROCESSING\n",
    "i = 0\n",
    "for doc_sentence_list in sentence_lists:\n",
    "#     print(i)\n",
    "    \n",
    "    early_porced_doc_sentence_list = dp.early_preprocessing(doc_sentence_list)\n",
    "    sentence_tokens_list = dp.word_tokenization(early_porced_doc_sentence_list)\n",
    "    sentence_lemma_list = dp.lemmatization(sentence_tokens_list)\n",
    "    \n",
    "    # sentence lemma list for keyword extraction\n",
    "    m_kw_sentence_lemma_list.append(copy.deepcopy(sentence_lemma_list))\n",
    "    \n",
    "    sentence_lemma_list = dp.stop_word_removal(sentence_lemma_list)\n",
    "    # sentence lemma list\n",
    "    \n",
    "    m_sentence_lemma_list.append(sentence_lemma_list)\n",
    "    \n",
    "    similarity_matrix = fe.sentenece_similarity_calculator(sentence_lemma_list)\n",
    "    # similarity matrix\n",
    "    m_similarity_matrix.append(similarity_matrix)\n",
    "\n",
    "    # POS Tagging\n",
    "    nouns, words = dp.pos_tagging(sentence_lemma_list)\n",
    "\n",
    "    # noun list\n",
    "    m_nouns.append(nouns)\n",
    "    \n",
    "    # word list\n",
    "    m_words.append(words)\n",
    "\n",
    "    i += 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "0d94cddf-64d4-4db1-9af7-be45043141cb",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# count matrix\n",
    "for sentence_lemma_list in m_sentence_lemma_list:\n",
    "  count_matrix = fe.word_freq_counter(sentence_lemma_list, words)\n",
    "  m_count_matrix.append(count_matrix)\n",
    "# KEYWORD GENERATION\n",
    "for sentence_list in m_kw_sentence_lemma_list:\n",
    "    # Finding keywords for each document\n",
    "    keywords = fe.keyword_extraction('rake', sentence_list)\n",
    "    m_keywords.append(keywords)\n",
    "i = 0\n",
    "for nn, kw, cm, wr in zip(m_nouns, m_keywords, m_count_matrix, m_words):\n",
    "    # Calculating word weightage\n",
    "    word_weight_dict = fe.word_weight_calculation(nn, kw, cm, wr)\n",
    "    m_word_weight_dict.append(word_weight_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "889acd9f-f461-4fb0-a5df-e8eb93810910",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "#Graph Generation\n",
    "for similarity_matrix in m_similarity_matrix:\n",
    "  graph = rc.graph_generation(similarity_matrix)\n",
    "  m_graph.append(graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "d9d13017-2703-4108-9e94-bb9b2e8b28ae",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# PROCESSING (RANK CALCULATION)\n",
    "i = 0\n",
    "for doc_id in range(0, len(sentence_lists)):\n",
    "#     print(i)\n",
    "#     i += 1 \n",
    "    A = rc.get_A(m_similarity_matrix[doc_id])   #dimensions = number of sentences X number of sentences\n",
    "    C = rc.get_C(m_count_matrix[doc_id])   #dimensions = number of sentences X number of words\n",
    "    B = rc.get_B(C)   #dimensions = number of sentences X number of words\n",
    "    P_t = rc.get_ranks(A, B, m_words[doc_id], m_word_weight_dict[doc_id])\n",
    "    mul_doc_ranks.append(P_t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "fc58f39c-cea8-414e-bb7e-7105133bf03b",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "### W/O Graph clstering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "24bc9313-2584-4f30-8ecd-d0902424ec8f",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# POST-PROCESSING\n",
    "i = 0\n",
    "for index in range(0, len(sentence_lists)):\n",
    "#     print(i)\n",
    "    i += 1  \n",
    "    sentence_ranks = mul_doc_ranks[index]\n",
    "    sentence_list = sentence_lists[index]\n",
    "    sentence_index = spp.get_indexes(sentence_ranks, no_summary_sentences[index])\n",
    "    m_sentence_index.append(sentence_index)\n",
    "    generated_summary = spp.summary_generation(sentence_list, sentence_index)\n",
    "    m_generated_summary.append(generated_summary)\n",
    "    unprocessed_ref_summary = m_unprocessed_ref_summary[index]\n",
    "    reference_summary = spp.ref_summary(unprocessed_ref_summary)\n",
    "    m_reference_summary.append(reference_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "129daddb-a103-4542-8474-0f82b46143ab",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# EVALUATION\n",
    "for generated_summary,reference_summary in zip(m_generated_summary, m_reference_summary):\n",
    "    rouge_scores_dict = se.rouge_measure(generated_summary, reference_summary, sub_cat = 'r')\n",
    "    m_rouge_scores.append(rouge_scores_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "c358deb7-5880-4ec6-8230-3c7737005f23",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4121  |     NA     |   0.1116  |     NA     |  0.3523   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.0000  |     88     |   0.0000  |     06     |  0.0000   |     88     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.7778  |     93     |   0.5294  |     93     |  0.7692   |     93     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4121  |     NA     |   0.1116  |     NA     |  0.3523   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.0000  |     88     |   0.0000  |     06     |  0.0000   |     88     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.7778  |     93     |   0.5294  |     93     |  0.7692   |     93     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(m_rouge_scores)) # 10-50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "80c61372-b5b2-4c8d-a80a-3952477df7a0",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4492  |     NA     |   0.1194  |     NA     |  0.3742   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.0000  |     24     |   0.0000  |     20     |  0.0000   |     24     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.6591  |     57     |   0.3056  |     16     |  0.5714   |     63     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4492  |     NA     |   0.1194  |     NA     |  0.3742   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.0000  |     24     |   0.0000  |     20     |  0.0000   |     24     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.6591  |     57     |   0.3056  |     16     |  0.5714   |     63     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(m_rouge_scores)) # 50-100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "a06f1853-c6fc-489f-9874-771643644d11",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4850  |     NA     |   0.1304  |     NA     |  0.378   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.2566  |     95     |   0.0179  |     95     |  0.2065   |     95     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.6414  |     67     |   0.3158  |     37     |  0.5435   |     38     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4850  |     NA     |   0.1304  |     NA     |  0.378   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.2566  |     95     |   0.0179  |     95     |  0.2065   |     95     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.6414  |     67     |   0.3158  |     37     |  0.5435   |     38     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(m_rouge_scores)) # 100-150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "e423f78f-e078-4545-bb4f-493832fe79f4",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4761  |     NA     |   0.1171  |     NA     |  0.3654   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.2105  |     58     |   0.0000  |     58     |  0.1224   |     58     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.6868  |     65     |   0.3259  |     62     |  0.5616   |     36     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4761  |     NA     |   0.1171  |     NA     |  0.3654   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.2105  |     58     |   0.0000  |     58     |  0.1224   |     58     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.6868  |     65     |   0.3259  |     62     |  0.5616   |     36     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(m_rouge_scores)) # 150-200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "59d512e1-7dd2-4e33-b6c4-bdde3b39b8a2",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4487  |     NA     |   0.1004  |     NA     |  0.3536   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.0849  |     30     |   0.0000  |     20     |  0.0270   |     30     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.7053  |     05     |   0.3016  |     05     |  0.5785   |     05     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4487  |     NA     |   0.1004  |     NA     |  0.3536   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.0849  |     30     |   0.0000  |     20     |  0.0270   |     30     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.7053  |     05     |   0.3016  |     05     |  0.5785   |     05     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(m_rouge_scores)) # 200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "f23f236e-839a-4afb-a35c-c7d6459b2bf7",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "### Graph Cluster based results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "c1e14079-5ed5-43c0-af2b-bd637ca967f1",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Graph Cluster Generation\n",
    "for graph in m_graph:\n",
    "  m_cluster.append(spp.graph_cluster_generation(graph))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "f3c9ce56-00f0-4540-b6b3-15789550c96c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "cl_m_sentence_index = []\n",
    "cl_m_generated_summary = []\n",
    "cl_m_rouge_scores = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "582cea49-1001-4933-af43-ec1e7c4eb630",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "for i in range(100):\n",
    "  val = []\n",
    "  try:\n",
    "    if type(m_cluster[i]) == None:\n",
    "      print(str(i)+\": NoneType\")\n",
    "      continue\n",
    "    temp = list(m_cluster[i].values())\n",
    "    for j in temp:\n",
    "      val += j\n",
    "    print(str(i)+\": \"+str(len(val)))\n",
    "    if i % 10 == 9:\n",
    "      print(\"=========================================\")\n",
    "      print(str(i+1) + \" to \" + str(i+10))\n",
    "      print(\"=========================================\")\n",
    "  except KeyError: \n",
    "    print(str(i)+\": Missing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "89db8e67-a912-4586-a3bc-4796011cd925",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# POST-PROCESSING\n",
    "i = 0\n",
    "for index in range(0, len(sentence_lists)):\n",
    "    start=datetime.now()\n",
    "    print(i)\n",
    "    i += 1  \n",
    "    graph_obj = m_graph[index]\n",
    "    cluster = m_cluster[index]\n",
    "    cl_sentence_ranks = mul_doc_ranks[index]\n",
    "    cl_sentence_list = sentence_lists[index]\n",
    "    cl_sentence_index = spp.sentence_selection_cluster(cluster, no_summary_sentences[index], graph_obj, cl_sentence_ranks)\n",
    "    cl_m_sentence_index.append(cl_sentence_index)\n",
    "    cl_generated_summary = spp.summary_generation(cl_sentence_list, cl_sentence_index)\n",
    "    cl_m_generated_summary.append(cl_generated_summary)\n",
    "    cmin = time.localtime(time.time())[4]\n",
    "    csec = time.localtime(time.time())[5]\n",
    "    print(\"Process Time: \"+str(datetime.now()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "518de390-74ff-418c-a2aa-9734e397a406",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# EVALUATION\n",
    "for cl_generated_summary,reference_summary in zip(cl_m_generated_summary, m_reference_summary):\n",
    "    cl_rouge_scores_dict = se.rouge_measure(cl_generated_summary, reference_summary, sub_cat = 'r')\n",
    "    cl_m_rouge_scores.append(cl_rouge_scores_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "460e2ec4-5354-4ebe-b55a-a717c8b4972a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "se.rouge_statistics(cl_m_rouge_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "f72910d0-979e-429e-83f1-475ba4616754",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(cl_m_rouge_scores)) #200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "db3dbb34-e1be-4fd3-b82c-c8ba9cb897c5",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4296  |     NA     |   0.0828  |     NA     |  0.3333   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.1587  |     76     |   0.0000  |     16     |  0.1020   |     58     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.6103  |     62     |   0.2320  |     65     |  0.5205   |     36     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4296  |     NA     |   0.0828  |     NA     |  0.3333   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.1587  |     76     |   0.0000  |     16     |  0.1020   |     58     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.6103  |     62     |   0.2320  |     65     |  0.5205   |     36     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(cl_m_rouge_scores)) #150-200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "d94f5d88-572c-4af0-b529-ad4165b60471",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4397  |     NA     |   0.0915  |     NA     |  0.3505   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.1351  |     68     |   0.0135  |     42     |  0.1250   |     68     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.6167  |     76     |   0.2526  |     37     |  0.5714   |     13     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4397  |     NA     |   0.0915  |     NA     |  0.3505   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.1351  |     68     |   0.0135  |     42     |  0.1250   |     68     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.6167  |     76     |   0.2526  |     37     |  0.5714   |     13     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(cl_m_rouge_scores)) #100-150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "b9baacfa-33c7-4237-945b-9a47111e3090",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.4219  |     NA     |   0.0946  |     NA     |  0.3572   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.1000  |     23     |   0.0000  |     20     |  0.1111   |     23     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.6591  |     57     |   0.3125  |     39     |  0.5231   |     98     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.4219  |     NA     |   0.0946  |     NA     |  0.3572   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.1000  |     23     |   0.0000  |     20     |  0.1111   |     23     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.6591  |     57     |   0.3125  |     39     |  0.5231   |     98     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(cl_m_rouge_scores)) #50-100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "4db7ce2f-8959-401d-b75b-b6dfc9580c2a",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">------------------------------------------------------------------------------------\n",
       "       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n",
       "------------------------------------------------------------------------------------\n",
       "  AVG  |   0.3942  |     NA     |   0.0952  |     NA     |  0.3436   |     NA     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MIN  |   0.0833  |     88     |   0.0000  |     06     |  0.0909   |     88     |\n",
       "------------------------------------------------------------------------------------\n",
       "  MAX  |   0.7222  |     93     |   0.4118  |     93     |  0.7692   |     93     |\n",
       "------------------------------------------------------------------------------------\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">------------------------------------------------------------------------------------\n|       |  ROUGE-1  |  Document  |  ROUGE-2  |  Document  |  ROUGE-L  |  Document  |\n------------------------------------------------------------------------------------\n|  AVG  |   0.3942  |     NA     |   0.0952  |     NA     |  0.3436   |     NA     |\n------------------------------------------------------------------------------------\n|  MIN  |   0.0833  |     88     |   0.0000  |     06     |  0.0909   |     88     |\n------------------------------------------------------------------------------------\n|  MAX  |   0.7222  |     93     |   0.4118  |     93     |  0.7692   |     93     |\n------------------------------------------------------------------------------------\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mf.print_rouge_statistics(se.rouge_statistics(cl_m_rouge_scores)) #10-50"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 2
   },
   "notebookName": "GitHub Extractive Text Summarization",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}