Extractive Text Summarization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Roshan\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.cluster.util import cosine_distance\n",
    "import numpy as np\n",
    "import networkx as nx\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def read_article(file_name):\n",
    "    file = open(file_name, \"r\")\n",
    "    filedata = file.readlines()\n",
    "    article = filedata[0].split(\". \")\n",
    "    sentences = []\n",
    "\n",
    "    for sentence in article:\n",
    "        print(sentence)\n",
    "        sentences.append(sentence.replace(\"[^a-zA-Z]\", \" \").split(\" \"))\n",
    "    sentences.pop() \n",
    "    \n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_similarity(sent1, sent2, stopwords=None):\n",
    "    if stopwords is None:\n",
    "        stopwords = []\n",
    " \n",
    "    sent1 = [w.lower() for w in sent1]\n",
    "    sent2 = [w.lower() for w in sent2]\n",
    " \n",
    "    all_words = list(set(sent1 + sent2))\n",
    " \n",
    "    vector1 = [0] * len(all_words)\n",
    "    vector2 = [0] * len(all_words)\n",
    " \n",
    "    # build the vector for the first sentence\n",
    "    for w in sent1:\n",
    "        if w in stopwords:\n",
    "            continue\n",
    "        vector1[all_words.index(w)] += 1\n",
    " \n",
    "    # build the vector for the second sentence\n",
    "    for w in sent2:\n",
    "        if w in stopwords:\n",
    "            continue\n",
    "        vector2[all_words.index(w)] += 1\n",
    " \n",
    "    return 1 - cosine_distance(vector1, vector2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_similarity_matrix(sentences, stop_words):\n",
    "    # Create an empty similarity matrix\n",
    "    similarity_matrix = np.zeros((len(sentences), len(sentences)))\n",
    " \n",
    "    for idx1 in range(len(sentences)):\n",
    "        for idx2 in range(len(sentences)):\n",
    "            if idx1 == idx2: #ignore if both are same sentences\n",
    "                continue \n",
    "            similarity_matrix[idx1][idx2] = sentence_similarity(sentences[idx1], sentences[idx2], stop_words)\n",
    "\n",
    "    return similarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ranked_sentence(file_name):\n",
    "    stop_words = stopwords.words('english')\n",
    "    summarize_text = []\n",
    "\n",
    "    file = open(file_name, \"r\")\n",
    "    filedata = file.readlines()\n",
    "    article = filedata[0].split(\". \")\n",
    "    sentences = []\n",
    "\n",
    "    for sentence in article:\n",
    "        sentences.append(sentence.replace(\"[^a-zA-Z]\", \" \").split(\" \"))\n",
    "    sentences.pop() \n",
    "\n",
    "    # Step 2 - Generate Similary Martix across sentences\n",
    "    sentence_similarity_martix = build_similarity_matrix(sentences, stop_words)\n",
    "\n",
    "    # Step 3 - Rank sentences in similarity martix\n",
    "    sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_martix)\n",
    "    scores = nx.pagerank(sentence_similarity_graph)\n",
    "    \n",
    "    # Step 4 - Sort the rank and pick top sentences\n",
    "    ranked_sentence = sorted(((scores[i],s) for i,s in enumerate(sentences)), reverse=True)\n",
    "    print(\"\\n\\n Indexes of top ranked_sentence order are \\n\\n\")\n",
    "    pprint(ranked_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_summary(file_name, top_n=5):\n",
    "    stop_words = stopwords.words('english')\n",
    "    summarize_text = []\n",
    "\n",
    "    # Step 1 - Read text anc split it\n",
    "    sentences =  read_article(file_name)\n",
    "\n",
    "    # Step 2 - Generate Similary Martix across sentences\n",
    "    sentence_similarity_martix = build_similarity_matrix(sentences, stop_words)\n",
    "\n",
    "    # Step 3 - Rank sentences in similarity martix\n",
    "    sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_martix)\n",
    "    scores = nx.pagerank(sentence_similarity_graph)\n",
    "\n",
    "    # Step 4 - Sort the rank and pick top sentences\n",
    "    ranked_sentence = sorted(((scores[i],s) for i,s in enumerate(sentences)), reverse=True)\n",
    "\n",
    "    for i in range(top_n):\n",
    "        summarize_text.append(\" \".join(ranked_sentence[i][1]))\n",
    "\n",
    "    # Step 5 - Offcourse, output the summarize texr\n",
    "    print(\"\\n\\n Summarize Text: \\n\\n\", \". \".join(summarize_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " Indexes of top ranked_sentence order are \n",
      "\n",
      "\n",
      "[(0.15083257041122708,\n",
      "  ['Envisioned',\n",
      "   'as',\n",
      "   'a',\n",
      "   'three-year',\n",
      "   'collaborative',\n",
      "   'program,',\n",
      "   'Intelligent',\n",
      "   'Cloud',\n",
      "   'Hub',\n",
      "   'will',\n",
      "   'support',\n",
      "   'around',\n",
      "   '100',\n",
      "   'institutions',\n",
      "   'with',\n",
      "   'AI',\n",
      "   'infrastructure,',\n",
      "   'course',\n",
      "   'content',\n",
      "   'and',\n",
      "   'curriculum,',\n",
      "   'developer',\n",
      "   'support,',\n",
      "   'development',\n",
      "   'tools',\n",
      "   'and',\n",
      "   'give',\n",
      "   'students',\n",
      "   'access',\n",
      "   'to',\n",
      "   'cloud',\n",
      "   'and',\n",
      "   'AI',\n",
      "   'services']),\n",
      " (0.13161201335715553,\n",
      "  ['The',\n",
      "   'company',\n",
      "   'will',\n",
      "   'provide',\n",
      "   'AI',\n",
      "   'development',\n",
      "   'tools',\n",
      "   'and',\n",
      "   'Azure',\n",
      "   'AI',\n",
      "   'services',\n",
      "   'such',\n",
      "   'as',\n",
      "   'Microsoft',\n",
      "   'Cognitive',\n",
      "   'Services,',\n",
      "   'Bot',\n",
      "   'Services',\n",
      "   'and',\n",
      "   'Azure',\n",
      "   'Machine',\n",
      "   'Learning.According',\n",
      "   'to',\n",
      "   'Manish',\n",
      "   'Prakash,',\n",
      "   'Country',\n",
      "   'General',\n",
      "   'Manager-PS,',\n",
      "   'Health',\n",
      "   'and',\n",
      "   'Education,',\n",
      "   'Microsoft',\n",
      "   'India,',\n",
      "   'said,',\n",
      "   '\"With',\n",
      "   'AI',\n",
      "   'being',\n",
      "   'the',\n",
      "   'defining',\n",
      "   'technology',\n",
      "   'of',\n",
      "   'our',\n",
      "   'time,',\n",
      "   'it',\n",
      "   'is',\n",
      "   'transforming',\n",
      "   'lives',\n",
      "   'and',\n",
      "   'industry',\n",
      "   'and',\n",
      "   'the',\n",
      "   'jobs',\n",
      "   'of',\n",
      "   'tomorrow',\n",
      "   'will',\n",
      "   'require',\n",
      "   'a',\n",
      "   'different',\n",
      "   'skillset']),\n",
      " (0.11403047674961146,\n",
      "  ['Earlier',\n",
      "   'in',\n",
      "   'April',\n",
      "   'this',\n",
      "   'year,',\n",
      "   'the',\n",
      "   'company',\n",
      "   'announced',\n",
      "   'Microsoft',\n",
      "   'Professional',\n",
      "   'Program',\n",
      "   'In',\n",
      "   'AI',\n",
      "   'as',\n",
      "   'a',\n",
      "   'learning',\n",
      "   'track',\n",
      "   'open',\n",
      "   'to',\n",
      "   'the',\n",
      "   'public']),\n",
      " (0.10721749759953528,\n",
      "  ['In',\n",
      "   'an',\n",
      "   'attempt',\n",
      "   'to',\n",
      "   'build',\n",
      "   'an',\n",
      "   'AI-ready',\n",
      "   'workforce,',\n",
      "   'Microsoft',\n",
      "   'announced',\n",
      "   'Intelligent',\n",
      "   'Cloud',\n",
      "   'Hub',\n",
      "   'which',\n",
      "   'has',\n",
      "   'been',\n",
      "   'launched',\n",
      "   'to',\n",
      "   'empower',\n",
      "   'the',\n",
      "   'next',\n",
      "   'generation',\n",
      "   'of',\n",
      "   'students',\n",
      "   'with',\n",
      "   'AI-ready',\n",
      "   'skills']),\n",
      " (0.10404298514456578,\n",
      "  ['As',\n",
      "   'part',\n",
      "   'of',\n",
      "   'the',\n",
      "   'program,',\n",
      "   'the',\n",
      "   'Redmond',\n",
      "   'giant',\n",
      "   'which',\n",
      "   'wants',\n",
      "   'to',\n",
      "   'expand',\n",
      "   'its',\n",
      "   'reach',\n",
      "   'and',\n",
      "   'is',\n",
      "   'planning',\n",
      "   'to',\n",
      "   'build',\n",
      "   'a',\n",
      "   'strong',\n",
      "   'developer',\n",
      "   'ecosystem',\n",
      "   'in',\n",
      "   'India',\n",
      "   'with',\n",
      "   'the',\n",
      "   'program',\n",
      "   'will',\n",
      "   'set',\n",
      "   'up',\n",
      "   'the',\n",
      "   'core',\n",
      "   'AI',\n",
      "   'infrastructure',\n",
      "   'and',\n",
      "   'IoT',\n",
      "   'Hub',\n",
      "   'for',\n",
      "   'the',\n",
      "   'selected',\n",
      "   'campuses']),\n",
      " (0.10031366655994461,\n",
      "  ['Thatâ€™s',\n",
      "   'why',\n",
      "   'it',\n",
      "   'has',\n",
      "   'become',\n",
      "   'more',\n",
      "   'critical',\n",
      "   'than',\n",
      "   'ever',\n",
      "   'for',\n",
      "   'educational',\n",
      "   'institutions',\n",
      "   'to',\n",
      "   'integrate',\n",
      "   'new',\n",
      "   'cloud',\n",
      "   'and',\n",
      "   'AI',\n",
      "   'technologies']),\n",
      " (0.10001137283486655,\n",
      "  ['The',\n",
      "   'program',\n",
      "   'is',\n",
      "   'an',\n",
      "   'attempt',\n",
      "   'to',\n",
      "   'ramp',\n",
      "   'up',\n",
      "   'the',\n",
      "   'institutional',\n",
      "   'set-up',\n",
      "   'and',\n",
      "   'build',\n",
      "   'capabilities',\n",
      "   'among',\n",
      "   'the',\n",
      "   'educators',\n",
      "   'to',\n",
      "   'educate',\n",
      "   'the',\n",
      "   'workforce',\n",
      "   'of',\n",
      "   'tomorrow.\"',\n",
      "   'The',\n",
      "   'program',\n",
      "   'aims',\n",
      "   'to',\n",
      "   'build',\n",
      "   'up',\n",
      "   'the',\n",
      "   'cognitive',\n",
      "   'skills',\n",
      "   'and',\n",
      "   'in-depth',\n",
      "   'understanding',\n",
      "   'of',\n",
      "   'developing',\n",
      "   'intelligent',\n",
      "   'cloud',\n",
      "   'connected',\n",
      "   'solutions',\n",
      "   'for',\n",
      "   'applications',\n",
      "   'across',\n",
      "   'industry']),\n",
      " (0.09916750119894317,\n",
      "  ['This',\n",
      "   'will',\n",
      "   'require',\n",
      "   'more',\n",
      "   'collaborations',\n",
      "   'and',\n",
      "   'training',\n",
      "   'and',\n",
      "   'working',\n",
      "   'with',\n",
      "   'AI']),\n",
      " (0.09277191614415067,\n",
      "  ['The',\n",
      "   'program',\n",
      "   'was',\n",
      "   'developed',\n",
      "   'to',\n",
      "   'provide',\n",
      "   'job',\n",
      "   'ready',\n",
      "   'skills',\n",
      "   'to',\n",
      "   'programmers',\n",
      "   'who',\n",
      "   'wanted',\n",
      "   'to',\n",
      "   'hone',\n",
      "   'their',\n",
      "   'skills',\n",
      "   'in',\n",
      "   'AI',\n",
      "   'and',\n",
      "   'data',\n",
      "   'science',\n",
      "   'with',\n",
      "   'a',\n",
      "   'series',\n",
      "   'of',\n",
      "   'online',\n",
      "   'courses',\n",
      "   'which',\n",
      "   'featured',\n",
      "   'hands-on',\n",
      "   'labs',\n",
      "   'and',\n",
      "   'expert',\n",
      "   'instructors',\n",
      "   'as',\n",
      "   'well'])]\n"
     ]
    }
   ],
   "source": [
    "ranked_sentence('text.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In an attempt to build an AI-ready workforce, Microsoft announced Intelligent Cloud Hub which has been launched to empower the next generation of students with AI-ready skills\n",
      "Envisioned as a three-year collaborative program, Intelligent Cloud Hub will support around 100 institutions with AI infrastructure, course content and curriculum, developer support, development tools and give students access to cloud and AI services\n",
      "As part of the program, the Redmond giant which wants to expand its reach and is planning to build a strong developer ecosystem in India with the program will set up the core AI infrastructure and IoT Hub for the selected campuses\n",
      "The company will provide AI development tools and Azure AI services such as Microsoft Cognitive Services, Bot Services and Azure Machine Learning.According to Manish Prakash, Country General Manager-PS, Health and Education, Microsoft India, said, \"With AI being the defining technology of our time, it is transforming lives and industry and the jobs of tomorrow will require a different skillset\n",
      "This will require more collaborations and training and working with AI\n",
      "Thatâ€™s why it has become more critical than ever for educational institutions to integrate new cloud and AI technologies\n",
      "The program is an attempt to ramp up the institutional set-up and build capabilities among the educators to educate the workforce of tomorrow.\" The program aims to build up the cognitive skills and in-depth understanding of developing intelligent cloud connected solutions for applications across industry\n",
      "Earlier in April this year, the company announced Microsoft Professional Program In AI as a learning track open to the public\n",
      "The program was developed to provide job ready skills to programmers who wanted to hone their skills in AI and data science with a series of online courses which featured hands-on labs and expert instructors as well\n",
      "This program also included developer-focused AI school that provided a bunch of assets to help build AI skills.\n",
      "\n",
      "\n",
      " Summarize Text: \n",
      "\n",
      " Envisioned as a three-year collaborative program, Intelligent Cloud Hub will support around 100 institutions with AI infrastructure, course content and curriculum, developer support, development tools and give students access to cloud and AI services\n"
     ]
    }
   ],
   "source": [
    "generate_summary('text.txt', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In an attempt to build an AI-ready workforce, Microsoft announced Intelligent Cloud Hub which has been launched to empower the next generation of students with AI-ready skills\n",
      "Envisioned as a three-year collaborative program, Intelligent Cloud Hub will support around 100 institutions with AI infrastructure, course content and curriculum, developer support, development tools and give students access to cloud and AI services\n",
      "As part of the program, the Redmond giant which wants to expand its reach and is planning to build a strong developer ecosystem in India with the program will set up the core AI infrastructure and IoT Hub for the selected campuses\n",
      "The company will provide AI development tools and Azure AI services such as Microsoft Cognitive Services, Bot Services and Azure Machine Learning.According to Manish Prakash, Country General Manager-PS, Health and Education, Microsoft India, said, \"With AI being the defining technology of our time, it is transforming lives and industry and the jobs of tomorrow will require a different skillset\n",
      "This will require more collaborations and training and working with AI\n",
      "Thatâ€™s why it has become more critical than ever for educational institutions to integrate new cloud and AI technologies\n",
      "The program is an attempt to ramp up the institutional set-up and build capabilities among the educators to educate the workforce of tomorrow.\" The program aims to build up the cognitive skills and in-depth understanding of developing intelligent cloud connected solutions for applications across industry\n",
      "Earlier in April this year, the company announced Microsoft Professional Program In AI as a learning track open to the public\n",
      "The program was developed to provide job ready skills to programmers who wanted to hone their skills in AI and data science with a series of online courses which featured hands-on labs and expert instructors as well\n",
      "This program also included developer-focused AI school that provided a bunch of assets to help build AI skills.\n",
      "\n",
      "\n",
      " Summarize Text: \n",
      "\n",
      " Envisioned as a three-year collaborative program, Intelligent Cloud Hub will support around 100 institutions with AI infrastructure, course content and curriculum, developer support, development tools and give students access to cloud and AI services. The company will provide AI development tools and Azure AI services such as Microsoft Cognitive Services, Bot Services and Azure Machine Learning.According to Manish Prakash, Country General Manager-PS, Health and Education, Microsoft India, said, \"With AI being the defining technology of our time, it is transforming lives and industry and the jobs of tomorrow will require a different skillset\n"
     ]
    }
   ],
   "source": [
    "generate_summary('text.txt', 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In an attempt to build an AI-ready workforce, Microsoft announced Intelligent Cloud Hub which has been launched to empower the next generation of students with AI-ready skills\n",
      "Envisioned as a three-year collaborative program, Intelligent Cloud Hub will support around 100 institutions with AI infrastructure, course content and curriculum, developer support, development tools and give students access to cloud and AI services\n",
      "As part of the program, the Redmond giant which wants to expand its reach and is planning to build a strong developer ecosystem in India with the program will set up the core AI infrastructure and IoT Hub for the selected campuses\n",
      "The company will provide AI development tools and Azure AI services such as Microsoft Cognitive Services, Bot Services and Azure Machine Learning.According to Manish Prakash, Country General Manager-PS, Health and Education, Microsoft India, said, \"With AI being the defining technology of our time, it is transforming lives and industry and the jobs of tomorrow will require a different skillset\n",
      "This will require more collaborations and training and working with AI\n",
      "Thatâ€™s why it has become more critical than ever for educational institutions to integrate new cloud and AI technologies\n",
      "The program is an attempt to ramp up the institutional set-up and build capabilities among the educators to educate the workforce of tomorrow.\" The program aims to build up the cognitive skills and in-depth understanding of developing intelligent cloud connected solutions for applications across industry\n",
      "Earlier in April this year, the company announced Microsoft Professional Program In AI as a learning track open to the public\n",
      "The program was developed to provide job ready skills to programmers who wanted to hone their skills in AI and data science with a series of online courses which featured hands-on labs and expert instructors as well\n",
      "This program also included developer-focused AI school that provided a bunch of assets to help build AI skills.\n",
      "\n",
      "\n",
      " Summarize Text: \n",
      "\n",
      " Envisioned as a three-year collaborative program, Intelligent Cloud Hub will support around 100 institutions with AI infrastructure, course content and curriculum, developer support, development tools and give students access to cloud and AI services. The company will provide AI development tools and Azure AI services such as Microsoft Cognitive Services, Bot Services and Azure Machine Learning.According to Manish Prakash, Country General Manager-PS, Health and Education, Microsoft India, said, \"With AI being the defining technology of our time, it is transforming lives and industry and the jobs of tomorrow will require a different skillset. Earlier in April this year, the company announced Microsoft Professional Program In AI as a learning track open to the public\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "x = generate_summary('text.txt', 3)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = 'For some, emoji have caused frustration for users (how the heck are you \\\n",
    "        supposed to use the 🙃 emoji?). Yet for many others, emoji has opened up \\\n",
    "        a fascinating new medium of communication. There are even emoji charade-esque \\\n",
    "        “games” where users can guess a movie title based on a series of emoji.\\\n",
    "        (try these: 💉💎 or 👦🏻👓⚡). But what happens when you push emoji a step further?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'value': ['🙃', '💉', '💎', '👦', '🏻', '👓', '⚡'],\n",
       " 'mean': [':upside-down_face:',\n",
       "  ':syringe:',\n",
       "  ':gem_stone:',\n",
       "  ':boy:',\n",
       "  ':light_skin_tone:',\n",
       "  ':glasses:',\n",
       "  ':high_voltage:'],\n",
       " 'location': [[100, 100],\n",
       "  [337, 337],\n",
       "  [338, 338],\n",
       "  [343, 343],\n",
       "  [344, 344],\n",
       "  [345, 345],\n",
       "  [346, 346]],\n",
       " 'flag': True}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lets import emot\n",
    "import emot\n",
    "\n",
    "emot.emoji(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">For some, emoji have caused frustration for users (how the heck are you         supposed to use the 🙃 emoji?). Yet for many others, \n",
       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    emoji\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
       "</mark>\n",
       " has opened up         a fascinating new medium of communication. There are even \n",
       "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    emoji charade-esque\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
       "</mark>\n",
       "         “games” where users can guess a movie title based on a series of emoji.        (try these: 💉💎 or 👦🏻👓⚡). But what happens when you push emoji a step further?</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# lets do named entity recognition\n",
    "import spacy\n",
    "\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "def explain_text_entities(text):\n",
    "    doc = nlp(text)\n",
    "    for ent in doc.ents:\n",
    "        print(f'Entity: {ent}, Label: {ent.label_}, {spacy.explain(ent.label_)}')\n",
    "        \n",
    "for i in range(1, 2):\n",
    "    one_sentence = text\n",
    "    doc = nlp(one_sentence)\n",
    "    spacy.displacy.render(doc, style='ent',jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Entity: emoji, Label: PERSON, People, including fictional\n",
      "Entity: emoji charade-esque, Label: PERSON, People, including fictional\n"
     ]
    }
   ],
   "source": [
    "explain_text_entities(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Abstractive Summarization using Gensim\n",
    "\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on package gensim:\n",
      "\n",
      "NAME\n",
      "    gensim\n",
      "\n",
      "DESCRIPTION\n",
      "    This package contains interfaces and functionality to compute pair-wise document similarities within a corpus\n",
      "    of documents.\n",
      "\n",
      "PACKAGE CONTENTS\n",
      "    _matutils\n",
      "    corpora (package)\n",
      "    downloader\n",
      "    interfaces\n",
      "    matutils\n",
      "    models (package)\n",
      "    nosy\n",
      "    parsing (package)\n",
      "    scripts (package)\n",
      "    similarities (package)\n",
      "    sklearn_api (package)\n",
      "    summarization (package)\n",
      "    test (package)\n",
      "    topic_coherence (package)\n",
      "    utils\n",
      "    viz (package)\n",
      "\n",
      "DATA\n",
      "    logger = <Logger gensim (WARNING)>\n",
      "\n",
      "VERSION\n",
      "    3.8.1\n",
      "\n",
      "FILE\n",
      "    c:\\users\\roshan\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\gensim\\__init__.py\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# lets check the documentation for gensim\n",
    "help(gensim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on package gensim.summarization in gensim:\n",
      "\n",
      "NAME\n",
      "    gensim.summarization - # bring model classes directly into package namespace, to save some typing\n",
      "\n",
      "PACKAGE CONTENTS\n",
      "    bm25\n",
      "    commons\n",
      "    graph\n",
      "    keywords\n",
      "    mz_entropy\n",
      "    pagerank_weighted\n",
      "    summarizer\n",
      "    syntactic_unit\n",
      "    textcleaner\n",
      "\n",
      "FILE\n",
      "    c:\\users\\roshan\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\gensim\\summarization\\__init__.py\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# lets take some help and check the documentation of summarization module\n",
    "help(gensim.summarization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function summarize in module gensim.summarization.summarizer:\n",
      "\n",
      "summarize(text, ratio=0.2, word_count=None, split=False)\n",
      "    Get a summarized version of the given text.\n",
      "    \n",
      "    The output summary will consist of the most representative sentences\n",
      "    and will be returned as a string, divided by newlines.\n",
      "    \n",
      "    Note\n",
      "    ----\n",
      "    The input should be a string, and must be longer than :const:`~gensim.summarization.summarizer.INPUT_MIN_LENGTH`\n",
      "    sentences for the summary to make sense.\n",
      "    The text will be split into sentences using the split_sentences method in the :mod:`gensim.summarization.texcleaner`\n",
      "    module. Note that newlines divide sentences.\n",
      "    \n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    text : str\n",
      "        Given text.\n",
      "    ratio : float, optional\n",
      "        Number between 0 and 1 that determines the proportion of the number of\n",
      "        sentences of the original text to be chosen for the summary.\n",
      "    word_count : int or None, optional\n",
      "        Determines how many words will the output contain.\n",
      "        If both parameters are provided, the ratio will be ignored.\n",
      "    split : bool, optional\n",
      "        If True, list of sentences will be returned. Otherwise joined\n",
      "        strings will bwe returned.\n",
      "    \n",
      "    Returns\n",
      "    -------\n",
      "    list of str\n",
      "        If `split` **OR**\n",
      "    str\n",
      "        Most representative sentences of given the text.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# lets check the documentation of summarizer\n",
    "help(gensim.summarization.summarize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.summarization.summarizer import summarize\n",
    "from gensim.summarization import keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are                 even emoji charade-esque PERSON “games” where users can guess a movie title based on a series of emoji.\n"
     ]
    }
   ],
   "source": [
    "print(summarize(text = 'For some, emoji have caused frustration for users (how the heck are you supposed to use the 🙃 emoji?).\\\n",
    "                Yet for many others, emoji PERSON has opened up a fascinating new medium of communication. There are \\\n",
    "                even emoji charade-esque PERSON “games” where users can guess a movie title based on a series of emoji. \\\n",
    "                (try these: 💉💎 or 👦🏻👓⚡). But what happens when you push emoji a step further?',\n",
    "                ratio = 0.5,\n",
    "                word_count = 15))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing\n",
      "intelligent\n",
      "kunpeng\n",
      "data\n",
      "huawei\n"
     ]
    }
   ],
   "source": [
    "print(keywords('Computing has constantly been at the heart of technology advancement, and is also the pivotal \\\n",
    "force to propel the intelligent world. Huawei Intelligent Computing has been strategically invested in general \\\n",
    "and AI computing, and builds the innovative Kunpeng, Ascend, and x86 computing platforms to unlock the ultimate \\\n",
    "computing power. Huawei Intelligent Computing provides full-stack, all-scenario solutions for the cloud-edge-device \\\n",
    "to catalyze the intelligent transformation of traditional data centers and industries, leading the way forward \\\n",
    "to a fully connected, intelligent world.'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing Text Preprocessing\n",
    "import spacy\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from string import punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a List of Stopwords\n",
    "stopwords = list(STOP_WORDS)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets get some documents for summarization\n",
    "\n",
    "# first document\n",
    "document1 =\"\"\"Machine learning (ML) is the scientific study of algorithms and statistical models that computer \n",
    "systems use to progressively improve their performance on a specific task. Machine learning algorithms build a \n",
    "mathematical model of sample data, known as \"training data\", in order to make predictions or decisions without \n",
    "being explicitly programmed to perform the task. Machine learning algorithms are used in the applications of \n",
    "email filtering, detection of network intruders, and computer vision, where it is infeasible to develop an \n",
    "algorithm of specific instructions for performing the task. Machine learning is closely related to computational\n",
    "statistics, which focuses on making predictions using computers. The study of mathematical optimization deliver\n",
    "methods, theory and application domains to the field of machine learning. Data mining is a field of study within\n",
    "machine learning, and focuses on exploratory data analysis through unsupervised learning.In its application across \n",
    "business problems, machine learning is also referred to as predictive analytics.\"\"\"\n",
    "\n",
    "# second document\n",
    "document2 = \"\"\"Our Father who art in heaven, hallowed be thy name. Thy kingdom come. Thy will be done, on \n",
    "earth as it is in heaven. Give us this day our daily bread; and forgive us our trespasses, as we forgive\n",
    "those who trespass against us; and lead us not into temptation, but deliver us from evil\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets load the english language model for processing the words and embeddings\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build an NLP Object\n",
    "docx = nlp(document1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenization of Text\n",
    "mytokens = [token.text for token in docx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Machine': 4, 'learning': 8, '(': 1, 'ML': 1, ')': 1, 'scientific': 1, 'study': 3, 'algorithms': 3, 'statistical': 1, 'models': 1, 'computer': 2, '\\n': 9, 'systems': 1, 'use': 1, 'progressively': 1, 'improve': 1, 'performance': 1, 'specific': 2, 'task': 3, '.': 7, 'build': 1, 'mathematical': 2, 'model': 1, 'sample': 1, 'data': 3, ',': 9, 'known': 1, '\"': 2, 'training': 1, 'order': 1, 'predictions': 2, 'decisions': 1, 'explicitly': 1, 'programmed': 1, 'perform': 1, 'applications': 1, 'email': 1, 'filtering': 1, 'detection': 1, 'network': 1, 'intruders': 1, 'vision': 1, 'infeasible': 1, 'develop': 1, 'algorithm': 1, 'instructions': 1, 'performing': 1, 'closely': 1, 'related': 1, 'computational': 1, 'statistics': 1, 'focuses': 2, 'making': 1, 'computers': 1, 'The': 1, 'optimization': 1, 'deliver': 1, 'methods': 1, 'theory': 1, 'application': 2, 'domains': 1, 'field': 2, 'machine': 3, 'Data': 1, 'mining': 1, 'exploratory': 1, 'analysis': 1, 'unsupervised': 1, 'In': 1, 'business': 1, 'problems': 1, 'referred': 1, 'predictive': 1, 'analytics': 1}\n"
     ]
    }
   ],
   "source": [
    "# Build Word Frequency\n",
    "\n",
    "# word.text is tokenization in spacy\n",
    "word_frequencies = {}\n",
    "for word in docx:\n",
    "    if word.text not in stopwords:\n",
    "            if word.text not in word_frequencies.keys():\n",
    "                word_frequencies[word.text] = 1\n",
    "            else:\n",
    "                word_frequencies[word.text] += 1\n",
    "                \n",
    "# lets print these word frequencies\n",
    "print(word_frequencies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    }
   ],
   "source": [
    "# Maximum Word Frequency\n",
    "maximum_frequency = max(word_frequencies.values())\n",
    "print(maximum_frequency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Machine': 0.4444444444444444,\n",
       " 'learning': 0.8888888888888888,\n",
       " '(': 0.1111111111111111,\n",
       " 'ML': 0.1111111111111111,\n",
       " ')': 0.1111111111111111,\n",
       " 'scientific': 0.1111111111111111,\n",
       " 'study': 0.3333333333333333,\n",
       " 'algorithms': 0.3333333333333333,\n",
       " 'statistical': 0.1111111111111111,\n",
       " 'models': 0.1111111111111111,\n",
       " 'computer': 0.2222222222222222,\n",
       " '\\n': 1.0,\n",
       " 'systems': 0.1111111111111111,\n",
       " 'use': 0.1111111111111111,\n",
       " 'progressively': 0.1111111111111111,\n",
       " 'improve': 0.1111111111111111,\n",
       " 'performance': 0.1111111111111111,\n",
       " 'specific': 0.2222222222222222,\n",
       " 'task': 0.3333333333333333,\n",
       " '.': 0.7777777777777778,\n",
       " 'build': 0.1111111111111111,\n",
       " 'mathematical': 0.2222222222222222,\n",
       " 'model': 0.1111111111111111,\n",
       " 'sample': 0.1111111111111111,\n",
       " 'data': 0.3333333333333333,\n",
       " ',': 1.0,\n",
       " 'known': 0.1111111111111111,\n",
       " '\"': 0.2222222222222222,\n",
       " 'training': 0.1111111111111111,\n",
       " 'order': 0.1111111111111111,\n",
       " 'predictions': 0.2222222222222222,\n",
       " 'decisions': 0.1111111111111111,\n",
       " 'explicitly': 0.1111111111111111,\n",
       " 'programmed': 0.1111111111111111,\n",
       " 'perform': 0.1111111111111111,\n",
       " 'applications': 0.1111111111111111,\n",
       " 'email': 0.1111111111111111,\n",
       " 'filtering': 0.1111111111111111,\n",
       " 'detection': 0.1111111111111111,\n",
       " 'network': 0.1111111111111111,\n",
       " 'intruders': 0.1111111111111111,\n",
       " 'vision': 0.1111111111111111,\n",
       " 'infeasible': 0.1111111111111111,\n",
       " 'develop': 0.1111111111111111,\n",
       " 'algorithm': 0.1111111111111111,\n",
       " 'instructions': 0.1111111111111111,\n",
       " 'performing': 0.1111111111111111,\n",
       " 'closely': 0.1111111111111111,\n",
       " 'related': 0.1111111111111111,\n",
       " 'computational': 0.1111111111111111,\n",
       " 'statistics': 0.1111111111111111,\n",
       " 'focuses': 0.2222222222222222,\n",
       " 'making': 0.1111111111111111,\n",
       " 'computers': 0.1111111111111111,\n",
       " 'The': 0.1111111111111111,\n",
       " 'optimization': 0.1111111111111111,\n",
       " 'deliver': 0.1111111111111111,\n",
       " 'methods': 0.1111111111111111,\n",
       " 'theory': 0.1111111111111111,\n",
       " 'application': 0.2222222222222222,\n",
       " 'domains': 0.1111111111111111,\n",
       " 'field': 0.2222222222222222,\n",
       " 'machine': 0.3333333333333333,\n",
       " 'Data': 0.1111111111111111,\n",
       " 'mining': 0.1111111111111111,\n",
       " 'exploratory': 0.1111111111111111,\n",
       " 'analysis': 0.1111111111111111,\n",
       " 'unsupervised': 0.1111111111111111,\n",
       " 'In': 0.1111111111111111,\n",
       " 'business': 0.1111111111111111,\n",
       " 'problems': 0.1111111111111111,\n",
       " 'referred': 0.1111111111111111,\n",
       " 'predictive': 0.1111111111111111,\n",
       " 'analytics': 0.1111111111111111}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculating the frquency for each of the words\n",
    "for word in word_frequencies.keys():  \n",
    "        word_frequencies[word] = (word_frequencies[word]/maximum_frequency)\n",
    "\n",
    "# Frequency Table\n",
    "word_frequencies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Sentence Tokens\n",
    "* scoring every sentence based on number of words\n",
    "* non stopwords in our word frequency table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Machine learning (ML) is the scientific study of algorithms and statistical models that computer \n",
      "systems use to progressively improve their performance on a specific task.: 5.5555555555555545, Machine learning algorithms build a \n",
      "mathematical model of sample data, known as \"training data\", in order to make predictions or decisions without \n",
      "being explicitly programmed to perform the task.: 9.333333333333332, Machine learning is closely related to computational\n",
      "statistics, which focuses on making predictions using computers.: 5.111111111111111, The study of mathematical optimization deliver\n",
      "methods, theory and application domains to the field of machine learning.: 5.555555555555556, Data mining is a field of study within\n",
      "machine learning, and focuses on exploratory data analysis through unsupervised learning.: 6.777777777777776, In its application across \n",
      "business problems, machine learning is also referred to as predictive analytics.: 4.777777777777778}\n"
     ]
    }
   ],
   "source": [
    "# Sentence Tokens\n",
    "sentence_list = [ sentence for sentence in docx.sents ]\n",
    "\n",
    "# Sentence Score via comparrng each word with sentence\n",
    "sentence_scores = {}  \n",
    "for sent in sentence_list:  \n",
    "        for word in sent:\n",
    "            if word.text.lower() in word_frequencies.keys():\n",
    "                if len(sent.text.split(' ')) < 30:\n",
    "                    if sent not in sentence_scores.keys():\n",
    "                        sentence_scores[sent] = word_frequencies[word.text.lower()]\n",
    "                    else:\n",
    "                        sentence_scores[sent] += word_frequencies[word.text.lower()]\n",
    "                        \n",
    "# printing the scores of importance for the sentences\n",
    "print(sentence_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Machine learning algorithms build a \n",
      "mathematical model of sample data, known as \"training data\", in order to make predictions or decisions without \n",
      "being explicitly programmed to perform the task., Data mining is a field of study within\n",
      "machine learning, and focuses on exploratory data analysis through unsupervised learning., The study of mathematical optimization deliver\n",
      "methods, theory and application domains to the field of machine learning., Machine learning (ML) is the scientific study of algorithms and statistical models that computer \n",
      "systems use to progressively improve their performance on a specific task., Machine learning is closely related to computational\n",
      "statistics, which focuses on making predictions using computers., In its application across \n",
      "business problems, machine learning is also referred to as predictive analytics.]\n"
     ]
    }
   ],
   "source": [
    "# finding the top N Sentences from the Paragraph\n",
    "\n",
    "# Import Heapq \n",
    "from heapq import nlargest\n",
    "\n",
    "summarized_sentences = nlargest(7, sentence_scores, key=sentence_scores.get)\n",
    "print(summarized_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Machine learning algorithms build a \n",
      "mathematical model of sample data, known as \"training data\", in order to make predictions or decisions without \n",
      "being explicitly programmed to perform the task.\n",
      "Data mining is a field of study within\n",
      "machine learning, and focuses on exploratory data analysis through unsupervised learning.\n",
      "The study of mathematical optimization deliver\n",
      "methods, theory and application domains to the field of machine learning.\n",
      "Machine learning (ML) is the scientific study of algorithms and statistical models that computer \n",
      "systems use to progressively improve their performance on a specific task.\n",
      "Machine learning is closely related to computational\n",
      "statistics, which focuses on making predictions using computers.\n",
      "In its application across \n",
      "business problems, machine learning is also referred to as predictive analytics.\n",
      "\n",
      "\n",
      " Summarized Text: Machine learning algorithms build a \n",
      "mathematical model of sample data, known as \"training data\", in order to make predictions or decisions without \n",
      "being explicitly programmed to perform the task. Data mining is a field of study within\n",
      "machine learning, and focuses on exploratory data analysis through unsupervised learning. The study of mathematical optimization deliver\n",
      "methods, theory and application domains to the field of machine learning. Machine learning (ML) is the scientific study of algorithms and statistical models that computer \n",
      "systems use to progressively improve their performance on a specific task. Machine learning is closely related to computational\n",
      "statistics, which focuses on making predictions using computers. In its application across \n",
      "business problems, machine learning is also referred to as predictive analytics.\n"
     ]
    }
   ],
   "source": [
    "# Convert Sentences from Spacy Span to Strings for joining entire sentence\n",
    "for w in summarized_sentences:\n",
    "    print(w.text)\n",
    "\n",
    "# List Comprehension of Sentences Converted From Spacy.span to strings\n",
    "final_sentences = [ w.text for w in summarized_sentences ]\n",
    "\n",
    "# lets join these sentences\n",
    "summary = ' '.join(final_sentences)\n",
    "\n",
    "# lets print the summary\n",
    "print(\"\\n\\n Summarized Text:\", summary)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}