notebooks/summarize_news_articles_hugging_face.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "summraize-news-articles-hugging-face.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "CTAy7jRsTrL0"
      },
      "source": [
        "!pip install newspaper3k transformers gradio --quiet "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "olfs_aGNXJnx"
      },
      "source": [
        "## Load libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4MXxiFLyXKmJ",
        "outputId": "cc19f177-129e-4cb4-ae32-15caf474f07a"
      },
      "source": [
        "from newspaper import Article\n",
        "from newspaper import Config\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "\n",
        "from transformers import pipeline\n",
        "import gradio as gr\n",
        "from gradio.mix import Parallel, Series"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1p4LpNOmVdoR"
      },
      "source": [
        "USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'\n",
        "\n",
        "config = Config()\n",
        "config.browser_user_agent = USER_AGENT\n",
        "config.request_timeout = 10\n",
        "\n",
        "url = 'https://www.technologyreview.com/2021/07/09/1028140/ai-voice-actors-sound-human/'\n",
        "article = Article(url, config=config)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LsDGs17PXsl7"
      },
      "source": [
        "## Download the article"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-byYnIVWJfj"
      },
      "source": [
        "article.download() "
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vK60sy3LWV9F"
      },
      "source": [
        "article.html"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ypAKBsZOXqfX"
      },
      "source": [
        "## Parse information from article"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ObHg-d3EWYLI"
      },
      "source": [
        "article.parse() \n",
        "\n",
        "authors = \", \".join(author for author in article.authors)\n",
        "title = article.title\n",
        "date = article.publish_date\n",
        "text = article.text\n",
        "image = article.top_image\n",
        "videos = article.movies\n",
        "url = article.url"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FQDY_ZmlWk0_",
        "outputId": "b4310166-a1a9-40ab-f63f-1493c8cff4f8"
      },
      "source": [
        "print(\"Information about the article\")\n",
        "print(\"=\" * 30)\n",
        "print(f\"Title: {title}\")\n",
        "print(f\"Author(s): {authors}\")\n",
        "print(f\"Publish date: {date}\")\n",
        "print(f\"Image: {image}\")\n",
        "print(f\"Videos: {videos}\")\n",
        "print(f\"Article link: {url}\")\n",
        "print(f\"Content: {text[:100] + '...'}\")"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Information about the article\n",
            "==============================\n",
            "Title: AI voice actors sound more human than ever—and they’re ready to hire\n",
            "Author(s): Karen Hao\n",
            "Publish date: 2021-07-09 00:00:00\n",
            "Image: https://wp.technologyreview.com/wp-content/uploads/2021/07/AIAudioActor-2.jpg?resize=1200,600\n",
            "Videos: []\n",
            "Article link: https://www.technologyreview.com/2021/07/09/1028140/ai-voice-actors-sound-human/\n",
            "Content: The company blog post drips with the enthusiasm of a ’90s US infomercial. WellSaid Labs describes wh...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iLBHcE3lXxHG"
      },
      "source": [
        "## NLP from article"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Maf0OQROWUi0"
      },
      "source": [
        "article.nlp()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LgaAM_e0Yk1L",
        "outputId": "4a8df6d8-3257-4e6d-f634-27c797ec8598"
      },
      "source": [
        "keywords = article.keywords\n",
        "keywords.sort()\n",
        "print(keywords)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['actors', 'ai', 'audio', 'certainly', 'clients', 'companies', 'different', 'everand', 'hire', 'human', 'ready', 'sound', 'theyre', 'voice', 'voices']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TkuS75W5YERO"
      },
      "source": [
        "keywords = \"\\n\".join(keyw for keyw in keywords)"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W00yQu7dX46d",
        "outputId": "84e30ee4-f6a5-40e0-c818-3f3b54326042"
      },
      "source": [
        "print(f\"Article Keywords: \\n{keywords}\")"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Article Keywords: \n",
            "actors\n",
            "ai\n",
            "audio\n",
            "certainly\n",
            "clients\n",
            "companies\n",
            "different\n",
            "everand\n",
            "hire\n",
            "human\n",
            "ready\n",
            "sound\n",
            "theyre\n",
            "voice\n",
            "voices\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zd2wIFIkWq8A"
      },
      "source": [
        "### Newspaper library summary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sl5_-D1DYAlx",
        "outputId": "a29662dc-ca90-4fec-d7a1-c03cc9d822df"
      },
      "source": [
        "print(f\"Summary: \\n{article.summary}\")"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Summary: \n",
            "Unlike a recording of a human voice actor, synthetic voices can also update their script in real time, opening up new opportunities to personalize advertising.\n",
            "Human voice actors, in particular, have been left to wonder what this means for their livelihoods.\n",
            "Part of what makes a human voice so human is its inconsistency, expressiveness, and ability to deliver the same lines in completely different styles, depending on the context.\n",
            "Capturing these nuances involves finding the right voice actors to supply the appropriate training data and fine-tune the deep-learning models.\n",
            "Many of its clients use the synthesized voices only in pre-production and switch to real voice actors for the final production.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0Nhx8VW-1Nn"
      },
      "source": [
        "text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gKU15cTPWn8O"
      },
      "source": [
        "## summarize with Hugging Face and Gradio"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uGiJlLvfT9P-"
      },
      "source": [
        "io1 = gr.Interface.load('huggingface/sshleifer/distilbart-cnn-12-6')\n",
        "io2 = gr.Interface.load(\"huggingface/facebook/bart-large-cnn\")\n",
        "io3 = gr.Interface.load(\"huggingface/google/pegasus-xsum\")  \n",
        "io4 = gr.Interface.load(\"huggingface/sshleifer/distilbart-cnn-6-6\")                   \n",
        "\n",
        "iface = Parallel(io1, io2, io3, io4,\n",
        "                 theme='huggingface', \n",
        "                 inputs = gr.inputs.Textbox(lines = 10, label=\"Text\"))\n",
        "\n",
        "iface.launch()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3JEx1OCkIf8d"
      },
      "source": [
        "def extract_article_text(url):\n",
        "  USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0'\n",
        "  config = Config()\n",
        "  config.browser_user_agent = USER_AGENT\n",
        "  config.request_timeout = 10\n",
        "\n",
        "  article = Article(url, config=config)\n",
        "  article.download()\n",
        "  article.parse()\n",
        "  text = article.text\n",
        "  return text"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oz3rYZzgIV_"
      },
      "source": [
        "extractor = gr.Interface(extract_article_text, 'text', 'text')\n",
        "summarizer = gr.Interface.load(\"huggingface/facebook/bart-large-cnn\")\n",
        "\n",
        "sample_url = [['https://www.technologyreview.com/2021/07/22/1029973/deepmind-alphafold-protein-folding-biology-disease-drugs-proteome/'],\n",
        "              ['https://www.technologyreview.com/2021/07/21/1029860/disability-rights-employment-discrimination-ai-hiring/'],\n",
        "              ['https://www.technologyreview.com/2021/07/09/1028140/ai-voice-actors-sound-human/']]\n",
        "\n",
        "desc =  '''\n",
        "        Let Hugging Face models summarize articles for you. \n",
        "        Note: Shorter articles generate faster summaries.\n",
        "        This summarizer uses bart-large-cnn model by Facebook\n",
        "        '''\n",
        "\n",
        "iface = Series(extractor, summarizer, \n",
        "  inputs = gr.inputs.Textbox(\n",
        "      lines = 2,\n",
        "      label = 'URL'\n",
        "  ),\n",
        "  outputs = 'text',\n",
        "  title = 'News Summarizer',\n",
        "  theme = 'huggingface',\n",
        "  description = desc,\n",
        "  examples=sample_url)\n",
        "\n",
        "iface.launch()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iG5YuJgOIs0O"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}