true_news_scraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "true_news_scraper.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Interpause/pseudo-text/blob/master/true_news_scraper.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z_N2QQ7-Nt1k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install newspaper3k\n",
        "import newspaper as news\n",
        "from newspaper import Article, Source, Config\n",
        "from newspaper.mthreading import NewsPool"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BMMNCycqTZ3b",
        "colab_type": "code",
        "outputId": "06e23a85-1f2c-4c55-f183-52f3f9bf5155",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive #drive.flush_and_unmount()\n",
        "import os\n",
        "drive.mount('/content/drive')\n",
        "os.chdir('/content/drive/My Drive/CSIT Internship/code')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tMfMWvS-o0ud",
        "colab_type": "text"
      },
      "source": [
        "# Data cleaning 2.0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yIOd1dSZ_iJ2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "df = pd.read_pickle('./raw/myNews4.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4H7_41ckWKg6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import unicodedata\n",
        "#normally also helps remove foreign languages\n",
        "removables = ['AD']\n",
        "def clean(x):\n",
        "    x.title = ' '.join(w for w in unicodedata.normalize('NFKD',x.title.strip()).encode('ascii','ignore').decode('utf8','ignore').split() if w not in removables)\n",
        "    x.text = ' '.join(w for w in unicodedata.normalize('NFKD',x.text.strip()).encode('ascii','ignore').decode('utf8','ignore').split() if w not in removables)\n",
        "    if len(x.title) < 1 or len(x.text.split()) < config.MIN_WORD_COUNT:\n",
        "        x.title = 'None'\n",
        "        x.text = 'None'\n",
        "    return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9uJyIVtitNPe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = df.dropna()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zhf5fVkcby7i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = df.apply(clean,axis=1)[df.text != 'None']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R9a813peIEkT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = df.drop_duplicates(subset=('text',))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liPbuMayJSxH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if not 'uuid' in df:\n",
        "    import uuid\n",
        "    df = df.assign(uuid=tuple(uuid.uuid4() for x in range(len(df))))\n",
        "    print('done')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rsGpGNGGd8vG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.to_pickle('./raw/myNews4.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0ik7kHvCjDM8",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "df1 = pd.read_pickle('./raw/myNews.pkl')\n",
        "df2 = pd.read_pickle('./raw/myNews2.pkl')\n",
        "df3 = pd.read_pickle('./raw/myNews3.pkl')\n",
        "df4 = pd.read_pickle('./raw/myNews4.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mNKWlpAxjDNC",
        "colab": {}
      },
      "source": [
        "dff = pd.concat((df1,df2,df3,df4),ignore_index=True,sort=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVFRTooJ1L3U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dff.text = dff.text.str.lower()\n",
        "dff.title = dff.title.str.lower()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8GcJrUwojMX3",
        "colab_type": "code",
        "outputId": "821b8df2-3d03-497d-dea8-a00d2ad56b1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "subset_impt = ('text',)\n",
        "dff = dff.drop_duplicates(subset=subset_impt).apply(clean,axis=1)[dff.text != 'None'].dropna(subset=subset_impt).drop_duplicates(subset=subset_impt).apply(clean,axis=1)[dff.text != 'None'].dropna(subset=subset_impt).reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hVkcFDPTeur2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tags_all = {}\n",
        "def clean_tags(x):\n",
        "    tags = set()\n",
        "    if str(x) == 'nan': return tags\n",
        "    for tag in x:\n",
        "        if tag[-2:] == ' |': \n",
        "            tags.add(tag[:-2].lower())\n",
        "        if ';' in tag:\n",
        "            for y in tag.split(';'): tags.add(y.strip().lower())\n",
        "    tags.discard('')\n",
        "    for z in tags:\n",
        "        tags_all[z] = tags_all.get(z,0) + 1\n",
        "    return tags"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rjDqyUXqb7IF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dff.meta_tags = dff.meta_tags.apply(clean_tags)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LG2iLk8ChrYy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#only about 700 articles have meta (human) keywords... use NER?\n",
        "#keywords are very inconsistent, will take too long to standardize. either split further by spaces or embed them\n",
        "dff.to_pickle('./raw/myNewsFinal.pkl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SMk3gmdDeMrZ",
        "colab_type": "code",
        "outputId": "63ff6f53-e968-4ac5-a97e-b7c79f3d7601",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "from collections import OrderedDict\n",
        "sorted(OrderedDict(tags_all).items(),key=lambda x: x[1],reverse=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('donald trump', 33),\n",
              " ('general election', 30),\n",
              " ('global warming', 29),\n",
              " ('boris johnson', 28),\n",
              " ('christmas', 27),\n",
              " ('indybest', 27),\n",
              " ('nature', 21),\n",
              " ('climate', 21),\n",
              " ('spintronics', 20),\n",
              " ('neuroscience', 19),\n",
              " ('climate change', 18),\n",
              " ('physics', 17),\n",
              " ('black friday', 17),\n",
              " ('intelligence', 17),\n",
              " ('psychology', 16),\n",
              " (\"alzheimer's\", 16),\n",
              " ('pharmacology', 16),\n",
              " ('ecology', 16),\n",
              " ('materials science', 15),\n",
              " ('chemistry', 15),\n",
              " ('brexit', 14),\n",
              " ('cyber monday', 14),\n",
              " ('dementia', 14),\n",
              " ('champions league', 13),\n",
              " ('anthony joshua', 13),\n",
              " ('mental health', 13),\n",
              " ('black friday 2019', 13),\n",
              " ('computers and internet', 13),\n",
              " ('astronomy', 13),\n",
              " ('brain injury', 13),\n",
              " ('brain-computer interfaces', 13),\n",
              " ('sun', 13),\n",
              " ('inorganic chemistry', 13),\n",
              " ('nanotechnology', 13),\n",
              " ('graphene', 13),\n",
              " ('diseases and conditions', 13),\n",
              " ('jeremy corbyn', 12),\n",
              " ('biodiversity', 12),\n",
              " ('bird flu', 12),\n",
              " ('air pollution', 12),\n",
              " ('cancer', 12),\n",
              " ('biology', 12),\n",
              " ('environmental issues', 12),\n",
              " ('air quality', 12),\n",
              " ('evolution', 12),\n",
              " ('research', 12),\n",
              " ('impeachment', 11),\n",
              " ('electronics', 11),\n",
              " ('environmental policies', 11),\n",
              " ('organic chemistry', 11),\n",
              " ('oceanography', 11),\n",
              " ('new species', 11),\n",
              " ('origin of life', 11),\n",
              " ('breast cancer', 11),\n",
              " ('colon cancer', 11),\n",
              " ('obesity', 11),\n",
              " ('stress', 11),\n",
              " ('fish', 11),\n",
              " ('computer science', 11),\n",
              " ('brain', 11),\n",
              " ('environmental awareness', 11),\n",
              " ('labour', 10),\n",
              " ('liverpool fc', 10),\n",
              " ('lifestyle news', 10),\n",
              " ('food', 10),\n",
              " ('cyber monday 2019', 10),\n",
              " ('biotechnology', 10),\n",
              " ('brain tumor', 10),\n",
              " ('depression', 10),\n",
              " ('early climate', 10),\n",
              " ('language acquisition', 10),\n",
              " ('child development', 10),\n",
              " ('behavior', 10),\n",
              " ('earth science', 10),\n",
              " ('fitness', 10),\n",
              " ('influenza', 10),\n",
              " ('adhd', 10),\n",
              " ('diabetes', 10),\n",
              " ('dna', 10),\n",
              " ('sustainability', 10),\n",
              " ('civil engineering', 10),\n",
              " ('russia', 9),\n",
              " ('conservatives', 9),\n",
              " ('nhs', 9),\n",
              " ('nasa', 9),\n",
              " ('liberal democrats', 9),\n",
              " ('labour party', 9),\n",
              " ('artificial intelligence', 9),\n",
              " ('weather', 9),\n",
              " ('environmental policy', 9),\n",
              " ('exotic species', 9),\n",
              " ('hacking', 9),\n",
              " ('human biology', 9),\n",
              " ('perception', 9),\n",
              " ('alternative medicine', 9),\n",
              " ('healthy aging', 9),\n",
              " ('solar flare', 9),\n",
              " ('relationships', 9),\n",
              " ('social psychology', 9),\n",
              " ('geography', 9),\n",
              " ('university', 9),\n",
              " ('enzyme', 9),\n",
              " ('skin cancer', 9),\n",
              " ('addiction', 9),\n",
              " ('learning', 9),\n",
              " ('fossils', 9),\n",
              " ('genes', 9),\n",
              " ('distributed computing', 9),\n",
              " ('information technology', 9),\n",
              " (\"today's healthcare\", 9),\n",
              " ('water', 9),\n",
              " ('personalized medicine', 9),\n",
              " ('endangered plants', 9),\n",
              " ('ukraine', 8),\n",
              " ('2019 general election', 8),\n",
              " ('andy ruiz', 8),\n",
              " ('pregnancy', 8),\n",
              " ('food & drink', 8),\n",
              " ('gifts', 8),\n",
              " ('endangered animals', 8),\n",
              " ('encryption', 8),\n",
              " ('mice', 8),\n",
              " ('extrasolar planets', 8),\n",
              " ('engineering', 8),\n",
              " ('disorders and syndromes', 8),\n",
              " ('learning disorders', 8),\n",
              " ('study', 8),\n",
              " ('insomnia', 8),\n",
              " ('headaches', 8),\n",
              " ('infant', 8),\n",
              " ('birth control', 8),\n",
              " ('stem cells', 8),\n",
              " ('memory', 8),\n",
              " ('nuclear energy', 8),\n",
              " ('invasive species', 8),\n",
              " ('spinal cord', 8),\n",
              " ('atmosphere', 8),\n",
              " ('spintronics research', 8),\n",
              " ('forest', 8),\n",
              " ('nancy pelosi', 7),\n",
              " ('volodymyr zelensky', 7),\n",
              " ('new zealand', 7),\n",
              " ('google', 7),\n",
              " ('anxiety', 7),\n",
              " ('jo swinson', 7),\n",
              " ('hiv', 7),\n",
              " ('world development', 7),\n",
              " ('solar system', 7),\n",
              " ('public health', 7),\n",
              " ('nervous system', 7),\n",
              " ('ocean policy', 7),\n",
              " ('hearing loss', 7),\n",
              " ('energy technology', 7),\n",
              " ('ice ages', 7),\n",
              " ('lung cancer', 7),\n",
              " ('epilepsy', 7),\n",
              " ('patients', 7),\n",
              " ('weight loss', 7),\n",
              " ('treatment', 7),\n",
              " ('nutrition', 7),\n",
              " ('children', 7),\n",
              " ('people', 7),\n",
              " ('doctors', 7),\n",
              " ('diet', 7),\n",
              " ('immune system', 7),\n",
              " ('cholesterol', 7),\n",
              " ('heart disease', 7),\n",
              " ('prostate cancer', 7),\n",
              " ('bully', 7),\n",
              " ('child psychology', 7),\n",
              " ('brain damage', 7),\n",
              " ('diet and weight loss', 7),\n",
              " ('alternative fuels', 7),\n",
              " ('virtual reality', 7),\n",
              " ('petroleum', 7),\n",
              " ('quantum physics', 7),\n",
              " ('astrophysics', 7),\n",
              " ('computer modeling', 7),\n",
              " ('genetics', 7),\n",
              " ('wild animals', 7),\n",
              " ('rna', 7),\n",
              " ('bacteria', 7),\n",
              " ('ocean', 7),\n",
              " ('rainforests', 7),\n",
              " ('medical technology', 7),\n",
              " ('engineering and construction', 7),\n",
              " ('joe biden', 6),\n",
              " ('kids', 6),\n",
              " ('saturn', 6),\n",
              " ('andy ruiz jr vs anthony joshua ii', 6),\n",
              " ('school', 6),\n",
              " ('technology', 6),\n",
              " ('the homeless fund', 6),\n",
              " ('fossil fuels', 6),\n",
              " ('resource shortage', 6),\n",
              " ('space exploration', 6),\n",
              " ('pollution', 6),\n",
              " ('behavioral science', 6),\n",
              " ('pregnancy and childbirth', 6),\n",
              " ('stars', 6),\n",
              " ('developmental biology', 6),\n",
              " ('thermodynamics', 6),\n",
              " ('anthropology', 6),\n",
              " ('nature of water', 6),\n",
              " ('mammals', 6),\n",
              " ('animals', 6),\n",
              " ('energy and resources', 6),\n",
              " ('health', 6),\n",
              " ('calorie', 6),\n",
              " ('researchers', 6),\n",
              " ('sexual health', 6),\n",
              " ('drugs', 6),\n",
              " ('doctor', 6),\n",
              " ('birth', 6),\n",
              " ('infectious diseases', 6),\n",
              " ('gene', 6),\n",
              " ('multiple sclerosis', 6),\n",
              " ('ovarian cancer', 6),\n",
              " ('stroke', 6),\n",
              " ('gene therapy', 6),\n",
              " ('cardiovascular', 6),\n",
              " ('education', 6),\n",
              " ('love', 6),\n",
              " ('social', 6),\n",
              " (\"alzheimer's research\", 6),\n",
              " ('brains', 6),\n",
              " ('baby', 6),\n",
              " ('biochemistry', 6),\n",
              " ('scientists', 6),\n",
              " ('decomposition', 6),\n",
              " ('polymer', 6),\n",
              " ('ozone', 6),\n",
              " ('quantum computing', 6),\n",
              " ('microbiology', 6),\n",
              " ('geology', 6),\n",
              " ('rainforest', 6),\n",
              " ('acid rain', 6),\n",
              " ('traumatic brain', 6),\n",
              " ('microbes and more', 6),\n",
              " ('ecology research', 6),\n",
              " ('natural disasters', 6),\n",
              " ('homelessness', 5),\n",
              " ('conservative party', 5),\n",
              " ('tottenham hotspur', 5),\n",
              " ('jurgen klopp', 5),\n",
              " ('manchester city fc', 5),\n",
              " ('apple', 5),\n",
              " ('andy ruiz jr', 5),\n",
              " ('formula one', 5),\n",
              " ('alcohol', 5),\n",
              " ('marriage', 5),\n",
              " ('amazon', 5),\n",
              " ('aids', 5),\n",
              " ('marine biology', 5),\n",
              " ('agriculture and food', 5),\n",
              " ('biochemistry research', 5),\n",
              " ('moon', 5),\n",
              " ('health policy', 5),\n",
              " ('geochemistry', 5),\n",
              " ('cells', 5),\n",
              " ('eye', 5),\n",
              " ('skin care', 5),\n",
              " ('exercise', 5),\n",
              " ('risk', 5),\n",
              " ('infection', 5),\n",
              " ('allergy', 5),\n",
              " ('virus', 5),\n",
              " ('illness', 5),\n",
              " ('gynecology', 5),\n",
              " ('hiv and aids', 5),\n",
              " ('hypertension', 5),\n",
              " ('iq', 5),\n",
              " ('parents', 5),\n",
              " ('autism', 5),\n",
              " ('illegal drugs', 5),\n",
              " ('memory loss', 5),\n",
              " (\"causes of alzheimer's disease\", 5),\n",
              " ('neurodegeneration', 5),\n",
              " ('mathematics', 5),\n",
              " ('prejudice', 5),\n",
              " (\"brain's response\", 5),\n",
              " ('social networks', 5),\n",
              " ('communication', 5),\n",
              " ('babies', 5),\n",
              " (\"women's health\", 5),\n",
              " ('robot', 5),\n",
              " ('energy', 5),\n",
              " ('polypropylene', 5),\n",
              " ('polyethylene', 5),\n",
              " ('buckyballs', 5),\n",
              " ('gps', 5),\n",
              " ('nebulae', 5),\n",
              " ('asteroids', 5),\n",
              " ('genetically modified', 5),\n",
              " ('species', 5),\n",
              " ('botany', 5),\n",
              " ('protein-dna interactions', 5),\n",
              " ('endangered', 5),\n",
              " ('renewable energy', 5),\n",
              " ('air contamination', 5),\n",
              " ('landslides', 5),\n",
              " ('bill', 5),\n",
              " ('conservation measures', 5),\n",
              " ('privacy issues', 5),\n",
              " ('pandemic', 5),\n",
              " ('pharmaceuticals', 5),\n",
              " ('encephalization', 5),\n",
              " ('medulla oblongata', 5),\n",
              " ('lobe', 5),\n",
              " ('eeg', 5),\n",
              " ('sensory', 5),\n",
              " ('cerebral cortex', 5),\n",
              " ('cerebellum', 5),\n",
              " ('motor cortex', 5),\n",
              " ('olfactory bulb', 5),\n",
              " ('white matter', 5),\n",
              " ('optic chiasm', 5),\n",
              " ('corpus callosum', 5),\n",
              " ('cranial nerves', 5),\n",
              " ('hypothalamus', 5),\n",
              " ('ganglia', 5),\n",
              " ('thalamus', 5),\n",
              " ('grey matter', 5),\n",
              " ('pituitary gland', 5),\n",
              " ('pons', 5),\n",
              " ('prefrontal cortex', 5),\n",
              " ('neocortex', 5),\n",
              " ('frontal lobe', 5),\n",
              " ('telencephalon', 5),\n",
              " ('hippocampus', 5),\n",
              " ('biochemical', 5),\n",
              " ('mobile computing', 5),\n",
              " ('comets and meteors', 5),\n",
              " ('ecosystems', 5),\n",
              " ('evolutionary biology', 5),\n",
              " ('controlled substances', 5),\n",
              " ('land management', 5),\n",
              " ('jersey city', 4),\n",
              " ('new jersey', 4),\n",
              " ('meditation', 4),\n",
              " ('red bull salzburg', 4),\n",
              " ('facebook', 4),\n",
              " ('students', 4),\n",
              " ('cycling', 4),\n",
              " ('vegan', 4),\n",
              " ('premier league', 4),\n",
              " ('deontay wilder', 4),\n",
              " ('f1', 4),\n",
              " ('sustainable travel', 4),\n",
              " ('best hotels', 4),\n",
              " ('twitter', 4),\n",
              " ('disability', 4),\n",
              " ('london', 4),\n",
              " ('climate crisis', 4),\n",
              " ('aviation', 4),\n",
              " ('arctic', 4),\n",
              " ('green party', 4),\n",
              " ('election', 4),\n",
              " ('final say', 4),\n",
              " ('tactical voting', 4),\n",
              " ('ai', 4),\n",
              " ('biometric', 4),\n",
              " ('pluto', 4),\n",
              " ('psychology research', 4),\n",
              " ('jupiter', 4),\n",
              " ('workplace health', 4),\n",
              " ('soil types', 4),\n",
              " ('mental health research', 4),\n",
              " ('extreme survival', 4),\n",
              " ('electricity', 4),\n",
              " ('tumor', 4),\n",
              " ('hospital', 4),\n",
              " ('medical', 4),\n",
              " ('disease', 4),\n",
              " ('cell', 4),\n",
              " ('sexuality', 4),\n",
              " ('emotional', 4),\n",
              " ('drug', 4),\n",
              " ('stop bleeding', 4),\n",
              " ('leukemia', 4),\n",
              " ('osteoporosis', 4),\n",
              " ('vioxx', 4),\n",
              " ('sleep disorders', 4),\n",
              " ('lymphoma', 4),\n",
              " ('fertility', 4),\n",
              " ('asthma', 4),\n",
              " ('erectile dysfunction', 4),\n",
              " ('metabolism', 4),\n",
              " ('friendship', 4),\n",
              " ('linguistics', 4),\n",
              " ('anger', 4),\n",
              " ('learning disability', 4),\n",
              " ('human', 4),\n",
              " ('marijuana', 4),\n",
              " ('beta amyloid', 4),\n",
              " ('thc', 4),\n",
              " ('shy', 4),\n",
              " ('selfish', 4),\n",
              " ('cynical shyness', 4),\n",
              " ('cognition', 4),\n",
              " ('gullible', 4),\n",
              " ('parenting', 4),\n",
              " ('cooperators', 4),\n",
              " ('hesitates', 4),\n",
              " ('society', 4),\n",
              " ('family', 4),\n",
              " ('dopamine', 4),\n",
              " ('breastfeeding', 4),\n",
              " ('infants', 4),\n",
              " ('mother', 4),\n",
              " ('childhood', 4),\n",
              " ('parent', 4),\n",
              " ('father', 4),\n",
              " ('menopause', 4),\n",
              " ('carbohydrates', 4),\n",
              " ('aging', 4),\n",
              " ('cyropreservation', 4),\n",
              " ('physical sciences', 4),\n",
              " ('physical science', 4),\n",
              " ('solar energy', 4),\n",
              " ('propulsion', 4),\n",
              " ('science', 4),\n",
              " ('crystals', 4),\n",
              " ('organic', 4),\n",
              " ('plasma', 4),\n",
              " ('compound', 4),\n",
              " ('extraordinary properties', 4),\n",
              " ('device', 4),\n",
              " ('gasoline', 4),\n",
              " ('natural gas', 4),\n",
              " ('nano', 4),\n",
              " ('buckyball', 4),\n",
              " ('quantum effects', 4),\n",
              " ('mars', 4),\n",
              " ('quantum computers', 4),\n",
              " ('computer graphics', 4),\n",
              " ('robotics', 4),\n",
              " ('robotic', 4),\n",
              " ('speech', 4),\n",
              " ('viruses', 4),\n",
              " ('confidentiality', 4),\n",
              " ('horse', 4),\n",
              " ('agriculture', 4),\n",
              " ('dolphins', 4),\n",
              " ('pet', 4),\n",
              " ('horses', 4),\n",
              " ('animal', 4),\n",
              " ('sea', 4),\n",
              " ('tiger', 4),\n",
              " ('bat', 4),\n",
              " ('bird', 4),\n",
              " ('dinosaur', 4),\n",
              " ('genome', 4),\n",
              " ('dna to rna', 4),\n",
              " ('loss of biodiversity', 4),\n",
              " ('deforestation', 4),\n",
              " ('dead zone', 4),\n",
              " ('survey', 4),\n",
              " ('water pollution', 4),\n",
              " ('coral reef', 4),\n",
              " ('glacier', 4),\n",
              " ('carbon dioxide', 4),\n",
              " ('rain forest', 4),\n",
              " ('hurricane', 4),\n",
              " ('bioenergy', 4),\n",
              " ('urban air quality', 4),\n",
              " ('car pollutants', 4),\n",
              " ('industrial air pollution', 4),\n",
              " ('airborne pollutants', 4),\n",
              " ('photochemical smog', 4),\n",
              " ('smog', 4),\n",
              " ('recycle', 4),\n",
              " ('air pollution from cars', 4),\n",
              " ('extinct', 4),\n",
              " ('paleontology', 4),\n",
              " ('dinosaurs', 4),\n",
              " ('religion', 4),\n",
              " ('lobby', 4),\n",
              " ('global infrastructure', 4),\n",
              " ('global policy', 4),\n",
              " ('government', 4),\n",
              " ('federal', 4),\n",
              " ('molecular biology', 4),\n",
              " ('internet', 4),\n",
              " ('weapons technology', 4),\n",
              " ('construction', 4),\n",
              " ('medical imaging', 4),\n",
              " ('drug-resistant bacteria', 4),\n",
              " ('teacher', 4),\n",
              " ('dyslexia', 4),\n",
              " ('snow and avalanches', 4),\n",
              " ('nucleus', 4),\n",
              " ('mitochondria', 4),\n",
              " ('neuropathic', 4),\n",
              " ('fisheries', 4),\n",
              " ('sad', 4),\n",
              " ('infant and preschool learning', 4),\n",
              " ('gastrointestinal problems', 4),\n",
              " ('medical topics', 4),\n",
              " (\"huntington's disease\", 4),\n",
              " ('computers', 4),\n",
              " ('educational technology', 4),\n",
              " ('neural interfaces', 4),\n",
              " ('logging', 4),\n",
              " ('floods', 4),\n",
              " ('environmental science', 4),\n",
              " ('charles darwin', 4),\n",
              " ('energy policy', 4),\n",
              " ('energy issues', 4),\n",
              " ('tree', 3),\n",
              " ('plant', 3),\n",
              " ('democrats', 3),\n",
              " ('vladimir putin', 3),\n",
              " ('yougov', 3),\n",
              " ('elizabeth warren', 3),\n",
              " ('saudi arabia', 3),\n",
              " ('australia', 3),\n",
              " ('general election 2019', 3),\n",
              " ('skincare', 3),\n",
              " ('shopping', 3),\n",
              " ('bayern munich', 3),\n",
              " ('frank lampard', 3),\n",
              " ('chelsea', 3),\n",
              " ('microsoft', 3),\n",
              " ('bbc', 3),\n",
              " ('china', 3),\n",
              " ('biomass', 3),\n",
              " ('teenagers', 3),\n",
              " ('gender', 3),\n",
              " ('tv', 3),\n",
              " ('vegetarian', 3),\n",
              " ('boxing', 3),\n",
              " ('arsenal', 3),\n",
              " ('andy ruiz vs anthony joshua 2', 3),\n",
              " ('concussion', 3),\n",
              " ('england cricket team', 3),\n",
              " ('eddie hearn', 3),\n",
              " ('channel 4', 3),\n",
              " ('miscarriage', 3),\n",
              " ('sex', 3),\n",
              " ('women', 3),\n",
              " ('money', 3),\n",
              " ('lifestyle features', 3),\n",
              " ('decorations', 3),\n",
              " ('christmas food', 3),\n",
              " ('wine', 3),\n",
              " ('fashion & beauty', 3),\n",
              " ('poultry', 3),\n",
              " ('toys', 3),\n",
              " ('grief', 3),\n",
              " ('discounts', 3),\n",
              " ('cop25', 3),\n",
              " ('laptop', 3),\n",
              " ('extinction rebellion', 3),\n",
              " ('oceans', 3),\n",
              " ('conservative', 3),\n",
              " ('scotland', 3),\n",
              " ('schools', 3),\n",
              " ('laptops', 3),\n",
              " ('food and agriculture', 3),\n",
              " ('public health education', 3),\n",
              " ('vitamin d', 3),\n",
              " ('biotechnology and bioengineering', 3),\n",
              " ('kuiper belt', 3),\n",
              " ('northern lights', 3),\n",
              " ('poverty and learning', 3),\n",
              " ('consumer behavior', 3),\n",
              " ('dysfunction', 3),\n",
              " ('dieting', 3),\n",
              " ('understanding', 3),\n",
              " ('evidence', 3),\n",
              " ('risks', 3),\n",
              " ('medicine', 3),\n",
              " ('cancerous', 3),\n",
              " ('white tea', 3),\n",
              " ('resveratrol', 3),\n",
              " ('insulin', 3),\n",
              " ('rubella', 3),\n",
              " ('deaf', 3),\n",
              " ('mouth', 3),\n",
              " ('patient', 3),\n",
              " ('pregnant', 3),\n",
              " ('mumps', 3),\n",
              " ('genetic information', 3),\n",
              " ('urology', 3),\n",
              " ('vision', 3),\n",
              " ('cell division', 3),\n",
              " ('childbirth', 3),\n",
              " ('infectious', 3),\n",
              " ('hair loss', 3),\n",
              " ('measles', 3),\n",
              " ('prostate health', 3),\n",
              " ('dentistry', 3),\n",
              " ('diagnostic tests', 3),\n",
              " ('biochip', 3),\n",
              " ('add', 3),\n",
              " ('anthrax', 3),\n",
              " ('symptoms', 3),\n",
              " ('blind', 3),\n",
              " ('psoriasis', 3),\n",
              " ('pneumonia', 3),\n",
              " ('forensics', 3),\n",
              " ('overeating', 3),\n",
              " ('diets', 3),\n",
              " ('fusion', 3),\n",
              " ('adult stem cells', 3),\n",
              " ('migraine', 3),\n",
              " ('gender differences', 3),\n",
              " ('empathy', 3),\n",
              " ('crystal meth', 3),\n",
              " ('alzheimer', 3),\n",
              " ('acetylcholine', 3),\n",
              " (\"alzheimer's diagnosis\", 3),\n",
              " ('stages of alzheimers', 3),\n",
              " ('symptoms of alzheimers', 3),\n",
              " (\"what is alzheimer's disease\", 3),\n",
              " ('alzheimer drugs', 3),\n",
              " ('alzheimers drugs', 3),\n",
              " ('alzheimers test', 3),\n",
              " ('alzheimers symptoms', 3),\n",
              " ('senile dementia', 3),\n",
              " (\"alzheimer's treatment\", 3),\n",
              " (\"alzheimer's causes\", 3),\n",
              " ('alzheimers prevention', 3),\n",
              " ('tacrine', 3),\n",
              " ('acetylcholinesterase', 3),\n",
              " (\"alzheimer's symptoms\", 3),\n",
              " ('alzheimers dementia', 3),\n",
              " ('iron particles', 3),\n",
              " ('donepezil', 3),\n",
              " ('alzheimer disease', 3),\n",
              " ('alzheimers disease', 3),\n",
              " ('amyloid plaques', 3),\n",
              " ('choroid plexus', 3),\n",
              " ('senility', 3),\n",
              " ('stress management', 3),\n",
              " ('post traumatic stress disorder', 3),\n",
              " ('depressed', 3),\n",
              " ('clinical depression', 3),\n",
              " ('teen stress', 3),\n",
              " ('clinical depression symptoms', 3),\n",
              " ('mental illness', 3),\n",
              " ('teen depression', 3),\n",
              " ('treatments for depression', 3),\n",
              " ('childhood depression', 3),\n",
              " ('stress causes', 3),\n",
              " ('adolescent depression', 3),\n",
              " ('bipolar depression', 3),\n",
              " ('gifted', 3),\n",
              " ('motivation', 3),\n",
              " ('smarter', 3),\n",
              " ('intelligent', 3),\n",
              " ('cognitive tasks', 3),\n",
              " ('learn', 3),\n",
              " ('distinguish faces', 3),\n",
              " ('social perception', 3),\n",
              " ('contaminants', 3),\n",
              " ('affection', 3),\n",
              " ('controlling', 3),\n",
              " ('altruism', 3),\n",
              " ('socially disabling', 3),\n",
              " ('altruistic behaviour', 3),\n",
              " ('relationship', 3),\n",
              " ('cooperating', 3),\n",
              " ('erotic images', 3),\n",
              " ('control', 3),\n",
              " ('powerful', 3),\n",
              " ('subculture', 3),\n",
              " ('boss', 3),\n",
              " ('prosopagnosia', 3),\n",
              " ('perceived', 3),\n",
              " ('bargaining', 3),\n",
              " ('animals altruistic', 3),\n",
              " ('cooperation', 3),\n",
              " ('pesticides', 3),\n",
              " ('lonely', 3),\n",
              " ('homosexual', 3),\n",
              " ('mimicking', 3),\n",
              " ('telepathy', 3),\n",
              " ('lesbian', 3),\n",
              " ('schizophrenia', 3),\n",
              " ('life', 3),\n",
              " ('ultrasound', 3),\n",
              " ('thin skin', 3),\n",
              " ('skin', 3),\n",
              " ('wrinkles', 3),\n",
              " ('age spots', 3),\n",
              " ('urinary incontinence', 3),\n",
              " ('calcium', 3),\n",
              " ('e-coli', 3),\n",
              " ('swimming', 3),\n",
              " ('slow-frozen people', 3),\n",
              " (\"alzheimer's disease\", 3),\n",
              " ('cyrogenics', 3),\n",
              " ('chemical bond', 3),\n",
              " ('biofuel', 3),\n",
              " ('engineer', 3),\n",
              " ('matter', 3),\n",
              " ('ethanol', 3),\n",
              " ('alternative energy', 3),\n",
              " ('spaceflight', 3),\n",
              " ('oled', 3),\n",
              " ('industrial', 3),\n",
              " ('nano-scale', 3),\n",
              " ('hierarchical topology', 3),\n",
              " ('chemical', 3),\n",
              " ('atoms', 3),\n",
              " ('molecule', 3),\n",
              " ('foams', 3),\n",
              " ('macroscale', 3),\n",
              " ('microscale', 3),\n",
              " ('auxetic', 3),\n",
              " ('friction', 3),\n",
              " ('bituminous coal', 3),\n",
              " ('diesel', 3),\n",
              " ('buckyegg', 3),\n",
              " ('ionic', 3),\n",
              " ('metal', 3),\n",
              " ('microstructure', 3),\n",
              " ('semiconductor', 3),\n",
              " ('spectograph', 3),\n",
              " ('composite', 3),\n",
              " ('crystallography', 3),\n",
              " ('molecular bonds', 3),\n",
              " ('covalent', 3),\n",
              " ('intermetallic', 3),\n",
              " ('electron microscope', 3),\n",
              " ('dark matter', 3),\n",
              " ('cosmology', 3),\n",
              " ('star', 3),\n",
              " ('cassini', 3),\n",
              " ('corona', 3),\n",
              " ('video games', 3),\n",
              " ('supercomputer', 3),\n",
              " ('supercomputers', 3),\n",
              " ('computing power', 3),\n",
              " ('computing', 3),\n",
              " ('erasing magnetic storage', 3),\n",
              " ('computer security', 3),\n",
              " ('transmitting sensitive data', 3),\n",
              " ('criminals', 3),\n",
              " ('quantum encryption', 3),\n",
              " ('codebreakers', 3),\n",
              " ('breach', 3),\n",
              " ('steganography', 3),\n",
              " ('classroom', 3),\n",
              " ('bionic', 3),\n",
              " ('software', 3),\n",
              " ('blue whale', 3),\n",
              " ('insects', 3),\n",
              " ('deer', 3),\n",
              " ('dogs', 3),\n",
              " ('birds', 3),\n",
              " ('cats', 3),\n",
              " ('biological', 3),\n",
              " ('lion', 3),\n",
              " ('aquaculture', 3),\n",
              " ('gmo', 3),\n",
              " ('cattle', 3),\n",
              " ('mad cow disease', 3),\n",
              " ('ecosystem', 3),\n",
              " ('cat', 3),\n",
              " ('elephant', 3),\n",
              " ('chimpanzee', 3),\n",
              " ('seals', 3),\n",
              " ('dolphin', 3),\n",
              " ('mouse', 3),\n",
              " ('genetic engineering', 3),\n",
              " ('stem cell', 3),\n",
              " ('protein', 3),\n",
              " ('medications', 3),\n",
              " ('recombinant dna', 3),\n",
              " ('genetically engineered', 3),\n",
              " ('pharmaceutical', 3),\n",
              " ('ongoing decline', 3),\n",
              " ('mass extinction', 3),\n",
              " ('mutant mice', 3),\n",
              " ('antibiotic', 3),\n",
              " ('antibiotics', 3),\n",
              " ('bacterium', 3),\n",
              " ('volcano', 3),\n",
              " ('ice sheet', 3),\n",
              " ('tornado', 3),\n",
              " ('indoor air quality', 3),\n",
              " ('earthquake', 3),\n",
              " ('ecological', 3),\n",
              " ('warmest on record', 3),\n",
              " ('paleoclimatology', 3),\n",
              " ('earthquakes', 3),\n",
              " ('plate tectonics', 3),\n",
              " ('environmental engineering', 3),\n",
              " ('outdoor air pollution', 3),\n",
              " ('pollutants', 3),\n",
              " ('non-renewable energy', 3),\n",
              " ('air pollution monitoring', 3),\n",
              " ('biofuels', 3),\n",
              " ('antarctic', 3),\n",
              " ('hydrothermal vents', 3),\n",
              " ('ocean currents', 3),\n",
              " ('contaminated', 3),\n",
              " ('polluted', 3),\n",
              " ('smog check', 3),\n",
              " ('particulates', 3),\n",
              " ('03', 3),\n",
              " ('smog test', 3),\n",
              " ('ancient civilizations', 3),\n",
              " ('fossil', 3),\n",
              " ('archaeology', 3),\n",
              " ('evolutionary', 3),\n",
              " ('report', 3),\n",
              " ('cultural', 3),\n",
              " ('management', 3),\n",
              " ('cost', 3),\n",
              " ('fund', 3),\n",
              " ('federal support', 3),\n",
              " ('financial incentives', 3),\n",
              " ('economics', 3),\n",
              " ('transportation', 3),\n",
              " ('politics', 3),\n",
              " ('bush', 3),\n",
              " ('administration', 3),\n",
              " ('poverty', 3),\n",
              " ('industry', 3),\n",
              " ('metamaterial', 3),\n",
              " ('wheelchair', 3),\n",
              " ('radiology', 3),\n",
              " ('medication', 3),\n",
              " ('therapeutics', 3),\n",
              " ('over-the-counter', 3),\n",
              " ('drug design', 3),\n",
              " ('potent drug', 3),\n",
              " ('drug interactions', 3),\n",
              " ('inexpensive medicines', 3),\n",
              " ('new drug', 3),\n",
              " ('multi-drug resistant', 3),\n",
              " ('pharmacy', 3),\n",
              " ('safe to use', 3),\n",
              " ('drug counterfeiting', 3),\n",
              " ('drug-resistant parasites', 3),\n",
              " ('grapefruit juice', 3),\n",
              " ('potent drug candidate', 3),\n",
              " ('off-label', 3),\n",
              " ('prescribed', 3),\n",
              " ('antidote', 3),\n",
              " ('drug discovery', 3),\n",
              " ('high school', 3),\n",
              " ('toddler', 3),\n",
              " ('educational policy', 3),\n",
              " ('whales', 3),\n",
              " ('red tide', 3),\n",
              " ('serotonin', 3),\n",
              " ('epa', 3),\n",
              " ('motor system', 3),\n",
              " ('transcription', 3),\n",
              " ('nerve cells communicate', 3),\n",
              " ('soma', 3),\n",
              " ('dendrite', 3),\n",
              " ('brain areas', 3),\n",
              " ('neuron', 3),\n",
              " ('motor neuron', 3),\n",
              " ('mrna', 3),\n",
              " ('specialized neurons', 3),\n",
              " ('marketing', 3),\n",
              " ('mechanical stress', 3),\n",
              " ('sewer', 3),\n",
              " ('drought', 3),\n",
              " ('groundwater depletion', 3),\n",
              " ('ice', 3),\n",
              " ('sons', 3),\n",
              " ('daughters', 3),\n",
              " ('happiness', 3),\n",
              " ('drug-resistant', 3),\n",
              " ('chronic illness', 3),\n",
              " ('patient education and counseling', 3),\n",
              " ('neuropharmacological', 3),\n",
              " ('anger management', 3),\n",
              " ('stroke prevention', 3),\n",
              " ('quantum hall effect', 3),\n",
              " ('zero-gap semiconductor', 3),\n",
              " ('ultra-fast photonics', 3),\n",
              " ('hall conductivity', 3),\n",
              " ('graphene oxide paper', 3),\n",
              " ('transmission electron microscopy (tem)', 3),\n",
              " ('dna strand', 3),\n",
              " ('opium', 3),\n",
              " ('communications', 3),\n",
              " ('aquatic biodiversity', 3),\n",
              " ('cell biology', 3),\n",
              " ('wildfire', 3),\n",
              " ('avian flu', 3),\n",
              " ('avian flu infection', 3),\n",
              " ('bird flu vaccine', 3),\n",
              " ('lethal avian influenza', 3),\n",
              " ('human flu', 3),\n",
              " ('pandemic influenza strains', 3),\n",
              " ('h5n1', 3),\n",
              " ('against bird flu virus', 3),\n",
              " ('avian influenza', 3),\n",
              " ('next flu pandemic', 3),\n",
              " ('h5', 3),\n",
              " ('avian flu virus', 3),\n",
              " ('bird flu spread', 3),\n",
              " ('flu pandemic', 3),\n",
              " ('wildfires', 3),\n",
              " ('tsunamis', 3),\n",
              " ('endangered species act', 3),\n",
              " ('trees', 3),\n",
              " ('stem education', 3),\n",
              " ('travel and recreation', 3),\n",
              " ('detectors', 3),\n",
              " ('staying healthy', 3),\n",
              " ('teen health', 3),\n",
              " ('severe weather', 3),\n",
              " ('energy and the environment', 3),\n",
              " ('wounds and healing', 3),\n",
              " ('sports science', 3),\n",
              " ('golden globes 2020', 2),\n",
              " ('jennifer lopez', 2),\n",
              " ('jerrold nadler', 2),\n",
              " ('hunter biden', 2),\n",
              " ('daniel goldman', 2),\n",
              " ('rudy giuliani', 2),\n",
              " ('sergey lavrov', 2),\n",
              " ('mike pompeo', 2),\n",
              " ('dominic raab', 2),\n",
              " ('ivanka trump', 2),\n",
              " ('william barr', 2),\n",
              " ('big business', 2),\n",
              " ('2020 democrats', 2),\n",
              " ('emmanuel macron', 2),\n",
              " ('pete buttigieg', 2),\n",
              " ('roxette', 2),\n",
              " ('marie frediksson', 2),\n",
              " ('jamal khashoggi', 2),\n",
              " ('jacinda ardern', 2),\n",
              " ('new york city', 2),\n",
              " ('banksy', 2),\n",
              " ('birmingham', 2),\n",
              " ('aung san suu kyi', 2),\n",
              " ('myanmar', 2),\n",
              " ('the hague', 2),\n",
              " ('rohingya', 2),\n",
              " ('antarctica', 2),\n",
              " ('natural disaster', 2),\n",
              " ('love actually', 2),\n",
              " ('whatsapp', 2),\n",
              " ('android', 2),\n",
              " ('makeup', 2),\n",
              " ('hair', 2),\n",
              " ('lipstick', 2),\n",
              " ('sports & fitness', 2),\n",
              " ('running', 2),\n",
              " ('gadgets and tech', 2),\n",
              " ('home and garden', 2),\n",
              " ('house judiciary committee', 2),\n",
              " ('jose mourinho', 2),\n",
              " ('lille', 2),\n",
              " ('inter milan', 2),\n",
              " ('fc barcelona', 2),\n",
              " ('chelsea fc', 2),\n",
              " ('rb salzburg', 2),\n",
              " ('instagram', 2),\n",
              " ('ps4', 2),\n",
              " ('mac', 2),\n",
              " ('mac pro', 2),\n",
              " ('huawei', 2),\n",
              " ('intel', 2),\n",
              " ('enceladus', 2),\n",
              " ('tesla', 2),\n",
              " ('elon musk', 2),\n",
              " ('drax', 2),\n",
              " ('carbon capture and storage', 2),\n",
              " ('uk economy', 2),\n",
              " ('manchester united', 2),\n",
              " ('nfl', 2),\n",
              " ('all blacks', 2),\n",
              " ('new zealand rugby', 2),\n",
              " ('south africa cricket', 2),\n",
              " ('chase carey', 2),\n",
              " ('world anti-doping agency', 2),\n",
              " ('best luxury hotels', 2),\n",
              " ('winter sun', 2),\n",
              " ('christmas travel', 2),\n",
              " ('easyjet', 2),\n",
              " ('volcanoes', 2),\n",
              " ('uk travel', 2),\n",
              " ('lner', 2),\n",
              " ('rail delivery group', 2),\n",
              " ('usman khan', 2),\n",
              " ('savannah', 2),\n",
              " ('georgia', 2),\n",
              " ('gaming', 2),\n",
              " ('disabilities', 2),\n",
              " ('neonatal care', 2),\n",
              " ('home office', 2),\n",
              " (\"crohn's disease\", 2),\n",
              " ('marie yovanovitch', 2),\n",
              " ('freshers week', 2),\n",
              " ('halls of residence', 2),\n",
              " ('university of bristol', 2),\n",
              " ...]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5zwE4OXrYYrR",
        "colab_type": "code",
        "outputId": "9a9c399e-d484-4a89-bd95-c9ea4b8d3127",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "dff.groupby('outlet').count()\n",
        "#deal with foreign languages that utilizes ascii later... (url filtering)\n",
        "#newspaper3k didnt parse keywords for reuters properly"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>url</th>\n",
              "      <th>date</th>\n",
              "      <th>category</th>\n",
              "      <th>title</th>\n",
              "      <th>text</th>\n",
              "      <th>meta</th>\n",
              "      <th>uuid</th>\n",
              "      <th>meta_tags</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>outlet</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>abcnews.go.com</th>\n",
              "      <td>226</td>\n",
              "      <td>226</td>\n",
              "      <td>226</td>\n",
              "      <td>226</td>\n",
              "      <td>226</td>\n",
              "      <td>226</td>\n",
              "      <td>226</td>\n",
              "      <td>226</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ap.org</th>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>asiaone.com</th>\n",
              "      <td>353</td>\n",
              "      <td>353</td>\n",
              "      <td>353</td>\n",
              "      <td>353</td>\n",
              "      <td>353</td>\n",
              "      <td>353</td>\n",
              "      <td>353</td>\n",
              "      <td>353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>bbc.com</th>\n",
              "      <td>209</td>\n",
              "      <td>209</td>\n",
              "      <td>209</td>\n",
              "      <td>209</td>\n",
              "      <td>209</td>\n",
              "      <td>209</td>\n",
              "      <td>209</td>\n",
              "      <td>209</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>businessinsider.sg</th>\n",
              "      <td>706</td>\n",
              "      <td>706</td>\n",
              "      <td>706</td>\n",
              "      <td>706</td>\n",
              "      <td>706</td>\n",
              "      <td>706</td>\n",
              "      <td>706</td>\n",
              "      <td>706</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>businesstimes.com.sg</th>\n",
              "      <td>492</td>\n",
              "      <td>492</td>\n",
              "      <td>492</td>\n",
              "      <td>492</td>\n",
              "      <td>492</td>\n",
              "      <td>492</td>\n",
              "      <td>492</td>\n",
              "      <td>492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>cbsnews.com</th>\n",
              "      <td>442</td>\n",
              "      <td>442</td>\n",
              "      <td>442</td>\n",
              "      <td>442</td>\n",
              "      <td>442</td>\n",
              "      <td>442</td>\n",
              "      <td>442</td>\n",
              "      <td>442</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>channelnewsasia.com</th>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>cnbc.com</th>\n",
              "      <td>267</td>\n",
              "      <td>267</td>\n",
              "      <td>267</td>\n",
              "      <td>267</td>\n",
              "      <td>267</td>\n",
              "      <td>267</td>\n",
              "      <td>267</td>\n",
              "      <td>267</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>cnn.com</th>\n",
              "      <td>1155</td>\n",
              "      <td>1155</td>\n",
              "      <td>1155</td>\n",
              "      <td>1155</td>\n",
              "      <td>1155</td>\n",
              "      <td>1155</td>\n",
              "      <td>1155</td>\n",
              "      <td>1155</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dailymail.co.uk</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>discovery.com</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>economist.com</th>\n",
              "      <td>429</td>\n",
              "      <td>429</td>\n",
              "      <td>429</td>\n",
              "      <td>429</td>\n",
              "      <td>429</td>\n",
              "      <td>429</td>\n",
              "      <td>429</td>\n",
              "      <td>429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>express.co.uk</th>\n",
              "      <td>1695</td>\n",
              "      <td>1695</td>\n",
              "      <td>1695</td>\n",
              "      <td>1695</td>\n",
              "      <td>1695</td>\n",
              "      <td>1695</td>\n",
              "      <td>1695</td>\n",
              "      <td>1695</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>foreignaffairs.com</th>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "      <td>49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>heraldsun.com.au</th>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>huffpost.com</th>\n",
              "      <td>94</td>\n",
              "      <td>94</td>\n",
              "      <td>94</td>\n",
              "      <td>94</td>\n",
              "      <td>94</td>\n",
              "      <td>94</td>\n",
              "      <td>94</td>\n",
              "      <td>94</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>independent.co.uk</th>\n",
              "      <td>860</td>\n",
              "      <td>860</td>\n",
              "      <td>860</td>\n",
              "      <td>860</td>\n",
              "      <td>860</td>\n",
              "      <td>860</td>\n",
              "      <td>860</td>\n",
              "      <td>860</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>latimes.com</th>\n",
              "      <td>210</td>\n",
              "      <td>210</td>\n",
              "      <td>210</td>\n",
              "      <td>210</td>\n",
              "      <td>210</td>\n",
              "      <td>210</td>\n",
              "      <td>210</td>\n",
              "      <td>210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mirror.co.uk</th>\n",
              "      <td>395</td>\n",
              "      <td>395</td>\n",
              "      <td>395</td>\n",
              "      <td>395</td>\n",
              "      <td>395</td>\n",
              "      <td>395</td>\n",
              "      <td>395</td>\n",
              "      <td>395</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nbcnews.com</th>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>news.com.au</th>\n",
              "      <td>54</td>\n",
              "      <td>54</td>\n",
              "      <td>54</td>\n",
              "      <td>54</td>\n",
              "      <td>54</td>\n",
              "      <td>54</td>\n",
              "      <td>54</td>\n",
              "      <td>54</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>newsroompost.com</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>newyorker.com</th>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>npr.org</th>\n",
              "      <td>1006</td>\n",
              "      <td>1006</td>\n",
              "      <td>1006</td>\n",
              "      <td>1006</td>\n",
              "      <td>1006</td>\n",
              "      <td>1006</td>\n",
              "      <td>1006</td>\n",
              "      <td>1006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nst.com.my</th>\n",
              "      <td>306</td>\n",
              "      <td>306</td>\n",
              "      <td>306</td>\n",
              "      <td>306</td>\n",
              "      <td>306</td>\n",
              "      <td>306</td>\n",
              "      <td>306</td>\n",
              "      <td>306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nydailynews.com</th>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nytimes.com</th>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "      <td>80</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nzherald.co.nz</th>\n",
              "      <td>69</td>\n",
              "      <td>69</td>\n",
              "      <td>69</td>\n",
              "      <td>69</td>\n",
              "      <td>69</td>\n",
              "      <td>69</td>\n",
              "      <td>69</td>\n",
              "      <td>69</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>pbs.org</th>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>phys.org</th>\n",
              "      <td>114</td>\n",
              "      <td>114</td>\n",
              "      <td>114</td>\n",
              "      <td>114</td>\n",
              "      <td>114</td>\n",
              "      <td>114</td>\n",
              "      <td>114</td>\n",
              "      <td>114</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>reuters.com</th>\n",
              "      <td>851</td>\n",
              "      <td>851</td>\n",
              "      <td>851</td>\n",
              "      <td>851</td>\n",
              "      <td>851</td>\n",
              "      <td>851</td>\n",
              "      <td>851</td>\n",
              "      <td>851</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>sciencedaily.com</th>\n",
              "      <td>847</td>\n",
              "      <td>847</td>\n",
              "      <td>847</td>\n",
              "      <td>847</td>\n",
              "      <td>847</td>\n",
              "      <td>847</td>\n",
              "      <td>847</td>\n",
              "      <td>847</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>scientificamerican.com</th>\n",
              "      <td>61</td>\n",
              "      <td>61</td>\n",
              "      <td>61</td>\n",
              "      <td>61</td>\n",
              "      <td>61</td>\n",
              "      <td>61</td>\n",
              "      <td>61</td>\n",
              "      <td>61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>straitstimes.com</th>\n",
              "      <td>531</td>\n",
              "      <td>531</td>\n",
              "      <td>531</td>\n",
              "      <td>531</td>\n",
              "      <td>531</td>\n",
              "      <td>531</td>\n",
              "      <td>531</td>\n",
              "      <td>531</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>telegraph.co.uk</th>\n",
              "      <td>290</td>\n",
              "      <td>290</td>\n",
              "      <td>290</td>\n",
              "      <td>290</td>\n",
              "      <td>290</td>\n",
              "      <td>290</td>\n",
              "      <td>290</td>\n",
              "      <td>290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>theatlantic.com</th>\n",
              "      <td>148</td>\n",
              "      <td>148</td>\n",
              "      <td>148</td>\n",
              "      <td>148</td>\n",
              "      <td>148</td>\n",
              "      <td>148</td>\n",
              "      <td>148</td>\n",
              "      <td>148</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>time.com</th>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tnp.sg</th>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "      <td>109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>today.com</th>\n",
              "      <td>118</td>\n",
              "      <td>118</td>\n",
              "      <td>118</td>\n",
              "      <td>118</td>\n",
              "      <td>118</td>\n",
              "      <td>118</td>\n",
              "      <td>118</td>\n",
              "      <td>118</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>usatoday.com</th>\n",
              "      <td>345</td>\n",
              "      <td>345</td>\n",
              "      <td>345</td>\n",
              "      <td>345</td>\n",
              "      <td>345</td>\n",
              "      <td>345</td>\n",
              "      <td>345</td>\n",
              "      <td>345</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>washingtonpost.com</th>\n",
              "      <td>282</td>\n",
              "      <td>282</td>\n",
              "      <td>282</td>\n",
              "      <td>282</td>\n",
              "      <td>282</td>\n",
              "      <td>282</td>\n",
              "      <td>282</td>\n",
              "      <td>282</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>washingtontimes.com</th>\n",
              "      <td>850</td>\n",
              "      <td>850</td>\n",
              "      <td>850</td>\n",
              "      <td>850</td>\n",
              "      <td>850</td>\n",
              "      <td>850</td>\n",
              "      <td>850</td>\n",
              "      <td>850</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>wsj.com</th>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "      <td>40</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         url  date  category  ...  meta  uuid  meta_tags\n",
              "outlet                                        ...                       \n",
              "abcnews.go.com           226   226       226  ...   226   226        226\n",
              "ap.org                    18    18        18  ...    18    18         18\n",
              "asiaone.com              353   353       353  ...   353   353        353\n",
              "bbc.com                  209   209       209  ...   209   209        209\n",
              "businessinsider.sg       706   706       706  ...   706   706        706\n",
              "businesstimes.com.sg     492   492       492  ...   492   492        492\n",
              "cbsnews.com              442   442       442  ...   442   442        442\n",
              "channelnewsasia.com      109   109       109  ...   109   109        109\n",
              "cnbc.com                 267   267       267  ...   267   267        267\n",
              "cnn.com                 1155  1155      1155  ...  1155  1155       1155\n",
              "dailymail.co.uk            1     1         1  ...     1     1          1\n",
              "discovery.com              3     3         3  ...     3     3          3\n",
              "economist.com            429   429       429  ...   429   429        429\n",
              "express.co.uk           1695  1695      1695  ...  1695  1695       1695\n",
              "foreignaffairs.com        49    49        49  ...    49    49         49\n",
              "heraldsun.com.au          15    15        15  ...    15    15         15\n",
              "huffpost.com              94    94        94  ...    94    94         94\n",
              "independent.co.uk        860   860       860  ...   860   860        860\n",
              "latimes.com              210   210       210  ...   210   210        210\n",
              "mirror.co.uk             395   395       395  ...   395   395        395\n",
              "nbcnews.com                7     7         7  ...     7     7          7\n",
              "news.com.au               54    54        54  ...    54    54         54\n",
              "newsroompost.com           3     3         3  ...     3     3          3\n",
              "newyorker.com             80    80        80  ...    80    80         80\n",
              "npr.org                 1006  1006      1006  ...  1006  1006       1006\n",
              "nst.com.my               306   306       306  ...   306   306        306\n",
              "nydailynews.com            4     4         4  ...     4     4          4\n",
              "nytimes.com               80    80        80  ...    80    80         80\n",
              "nzherald.co.nz            69    69        69  ...    69    69         69\n",
              "pbs.org                    7     7         7  ...     7     7          7\n",
              "phys.org                 114   114       114  ...   114   114        114\n",
              "reuters.com              851   851       851  ...   851   851        851\n",
              "sciencedaily.com         847   847       847  ...   847   847        847\n",
              "scientificamerican.com    61    61        61  ...    61    61         61\n",
              "straitstimes.com         531   531       531  ...   531   531        531\n",
              "telegraph.co.uk          290   290       290  ...   290   290        290\n",
              "theatlantic.com          148   148       148  ...   148   148        148\n",
              "time.com                  24    24        24  ...    24    24         24\n",
              "tnp.sg                   109   109       109  ...   109   109        109\n",
              "today.com                118   118       118  ...   118   118        118\n",
              "usatoday.com             345   345       345  ...   345   345        345\n",
              "washingtonpost.com       282   282       282  ...   282   282        282\n",
              "washingtontimes.com      850   850       850  ...   850   850        850\n",
              "wsj.com                   40    40        40  ...    40    40         40\n",
              "\n",
              "[44 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bQgC4DempUEB",
        "colab_type": "code",
        "outputId": "22f8cf80-bbca-4597-8f0b-750db2cad6dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 759
        }
      },
      "source": [
        "dff"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>url</th>\n",
              "      <th>date</th>\n",
              "      <th>outlet</th>\n",
              "      <th>category</th>\n",
              "      <th>title</th>\n",
              "      <th>text</th>\n",
              "      <th>meta</th>\n",
              "      <th>uuid</th>\n",
              "      <th>meta_tags</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>http://www.channelnewsasia.com/news/cna938</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>channelnewsasia.com</td>\n",
              "      <td>https://www.channelnewsasia.com/news/cna938</td>\n",
              "      <td>CNA938</td>\n",
              "      <td>6AM to 7AM The Weekly with Keith De Souza and ...</td>\n",
              "      <td>{\"viewport\": \"width=device-width, initial-scal...</td>\n",
              "      <td>5fa0da01-b2b4-4f74-904a-349eebfe6923</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://www.channelnewsasia.com/news/rss</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>channelnewsasia.com</td>\n",
              "      <td>https://www.channelnewsasia.com/news/rss</td>\n",
              "      <td>Breaking news, latest Singapore, Asia and worl...</td>\n",
              "      <td>RSS (Really Simple Syndication) gives you the ...</td>\n",
              "      <td>{\"viewport\": \"width=device-width, initial-scal...</td>\n",
              "      <td>832489e6-ac8e-4166-85cd-5f368dd39375</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>http://www.channelnewsasia.com/news/singapore/...</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>channelnewsasia.com</td>\n",
              "      <td>https://www.channelnewsasia.com/news/singapore...</td>\n",
              "      <td>Companies hunt for talent as Singapore tech se...</td>\n",
              "      <td>SINGAPORE: As a child, Huang Wei Long liked to...</td>\n",
              "      <td>{\"viewport\": \"width=device-width, initial-scal...</td>\n",
              "      <td>a8031ffb-9db7-42dc-988b-24ec53dc63a5</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>http://www.channelnewsasia.com/news/singapore/...</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>channelnewsasia.com</td>\n",
              "      <td>https://www.channelnewsasia.com/news/singapore...</td>\n",
              "      <td>PIE viaduct collapse: Jail for engineer who kn...</td>\n",
              "      <td>SINGAPORE: An engineer overseeing works who di...</td>\n",
              "      <td>{\"viewport\": \"width=device-width, initial-scal...</td>\n",
              "      <td>94796e25-6ad2-49ed-b927-dca8bf574134</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>http://www.channelnewsasia.com/news/singapore/...</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>channelnewsasia.com</td>\n",
              "      <td>https://www.channelnewsasia.com/news/singapore...</td>\n",
              "      <td>The Big Read: After the sound and fury, the du...</td>\n",
              "      <td>Have the actions of some black sheep caused au...</td>\n",
              "      <td>{\"viewport\": \"width=device-width, initial-scal...</td>\n",
              "      <td>f71dfba8-abab-4a0e-820f-8ffe5a87c951</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14000</th>\n",
              "      <td>http://de.reuters.com/article/deutschland-spor...</td>\n",
              "      <td>1.576056e+09</td>\n",
              "      <td>reuters.com</td>\n",
              "      <td>https://de.reuters.com/article/deutschland-spo...</td>\n",
              "      <td>Blatt - Warenhausriese Karstadt Kaufhof uberni...</td>\n",
              "      <td>A warehouse of the German department store cha...</td>\n",
              "      <td>{\"robots\": \"index, follow\", \"viewport\": \"width...</td>\n",
              "      <td>7a230e27-c153-4dd4-a8f9-b5722dfc4925</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14001</th>\n",
              "      <td>http://de.reuters.com/article/usa-milit-r-erde...</td>\n",
              "      <td>1.576057e+09</td>\n",
              "      <td>reuters.com</td>\n",
              "      <td>https://de.reuters.com/article/usa-milit-r-erd...</td>\n",
              "      <td>US-Militar will Bau von Seltene-Erden-Fabriken...</td>\n",
              "      <td>People walk past the U.S. Capitol Building in ...</td>\n",
              "      <td>{\"robots\": \"index, follow\", \"viewport\": \"width...</td>\n",
              "      <td>cfa1fc59-18e5-4da6-aa99-dff58e7306ec</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14002</th>\n",
              "      <td>http://de.reuters.com/article/sterreich-regier...</td>\n",
              "      <td>1.576057e+09</td>\n",
              "      <td>reuters.com</td>\n",
              "      <td>https://de.reuters.com/article/sterreich-regie...</td>\n",
              "      <td>Osterreichs Ubergangskanzlerin hofft auf baldi...</td>\n",
              "      <td>Austria's Chancellor Brigitte Bierlein arrives...</td>\n",
              "      <td>{\"robots\": \"index, follow\", \"viewport\": \"width...</td>\n",
              "      <td>9bd83a59-a68d-4537-807e-d8d47733535d</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14003</th>\n",
              "      <td>http://de.reuters.com/article/china-journalism...</td>\n",
              "      <td>1.576053e+09</td>\n",
              "      <td>reuters.com</td>\n",
              "      <td>https://de.reuters.com/article/china-journalis...</td>\n",
              "      <td>Organisation - China hat 2019 die meisten Jour...</td>\n",
              "      <td>A graduate stands behind the Chinese flag duri...</td>\n",
              "      <td>{\"robots\": \"index, follow\", \"viewport\": \"width...</td>\n",
              "      <td>8c464c42-e45f-4f2a-b533-2e8a7322771d</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14004</th>\n",
              "      <td>https://www.reuters.com/article/us-newzealand-...</td>\n",
              "      <td>1.576058e+09</td>\n",
              "      <td>reuters.com</td>\n",
              "      <td>https://www.reuters.com/article/us-newzealand-...</td>\n",
              "      <td>Exclusive: Disasters, downturn challenge New Z...</td>\n",
              "      <td>WELLINGTON (Reuters) - New Zealand Prime Minis...</td>\n",
              "      <td>{\"robots\": \"index, follow\", \"viewport\": \"width...</td>\n",
              "      <td>7261e015-cf35-48fc-a762-620f120c7412</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>14005 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     url  ...  meta_tags\n",
              "0             http://www.channelnewsasia.com/news/cna938  ...         {}\n",
              "1               https://www.channelnewsasia.com/news/rss  ...         {}\n",
              "2      http://www.channelnewsasia.com/news/singapore/...  ...         {}\n",
              "3      http://www.channelnewsasia.com/news/singapore/...  ...         {}\n",
              "4      http://www.channelnewsasia.com/news/singapore/...  ...         {}\n",
              "...                                                  ...  ...        ...\n",
              "14000  http://de.reuters.com/article/deutschland-spor...  ...         {}\n",
              "14001  http://de.reuters.com/article/usa-milit-r-erde...  ...         {}\n",
              "14002  http://de.reuters.com/article/sterreich-regier...  ...         {}\n",
              "14003  http://de.reuters.com/article/china-journalism...  ...         {}\n",
              "14004  https://www.reuters.com/article/us-newzealand-...  ...         {}\n",
              "\n",
              "[14005 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 167
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liUMHzCunmQX",
        "colab_type": "code",
        "outputId": "57c95974-6d3b-4e6e-ec7b-545310caadd6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        }
      },
      "source": [
        "for _,row in dff.iterrows():\n",
        "    if 'indybest' in row.meta_tags:\n",
        "        print(row.title)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best Christmas wreaths 2019: Real and artificial arrangements for your door\n",
            "11 best log burners that are better for the environment\n",
            "9 best men's ski and snowboard pants to stay protected on the slopes\n",
            "8 best Christmas pudding alternatives to impress your guests with\n",
            "Best Christmas beauty gifts under 50: Make-up, skincare, candles and more\n",
            "15 best Christmas gifts for teenagers that they'll actually use\n",
            "Best Cyber Monday Amazon deals: Best offers live to buy today\n",
            "Cyber Monday Apple deals UK: Apple Watch, Airpods, iPhone offers to buy today\n",
            "Best Cyber Monday laptop deals: Apple, Microsoft and more of the best offers to buy today\n",
            "Best Cyber Monday kids' toys deals: Offers on Lego, Harry Potter, Frozen and more\n",
            "Best Cyber Monday 2019 furniture deals from Zara Home, Made, Swoon and more\n",
            "Best Cyber Monday TV deals: Top offers from Currys PC World, Amazon and Argos today\n",
            "Best Cyber Monday console deals 2019: Nintendo Switch, PS4, Xbox One and more gaming deals to buy today\n",
            "Best Argos, John Lewis and Currys Cyber Monday deals 2019 live now\n",
            "Cyber Monday TV and broadband deals: Best offers from EE, Sky, BT and more\n",
            "Cyber Monday fashion and beauty deals 2019: Best offers from Asos, H&M, Charlotte Tilbury and more\n",
            "Best Christmas puddings to suit every guest, from vegan to traditional\n",
            "Best kids' toys 2019: Puzzles, games and crafts that make the perfect Christmas gift\n",
            "14 best beauty stocking fillers full of make-up and skincare treats\n",
            "15 best wine gifts that are worth raising a glass to\n",
            "8 best vegan cheeses to impress your dairy-free guests this Christmas\n",
            "Best panettones for Christmas 2019: Gin, salted caramel, chocolate and more\n",
            "Best alcohol Christmas crackers 2019: Gin, whisky wine and more\n",
            "11 best food subscription gifts to treat the foodie in your life\n",
            "Best Christmas crackers of 2019, from luxury to cheap to unusual gifts\n",
            "Best vegan Christmas dinner alternatives to master the meat-free roast\n",
            "8 best laptops under 250 that are real value for money\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mSU9F8fveXyI",
        "colab_type": "text"
      },
      "source": [
        "# Scrape"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R8Au9oqsXqaP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "config = Config()\n",
        "config.memoize_articles = False\n",
        "config.language = 'en'\n",
        "config.fetch_images = False\n",
        "config.request_timeout = 60\n",
        "config.thread_request_timeout = 60\n",
        "config.follow_meta_refresh = True\n",
        "config.MIN_WORD_COUNT = 150\n",
        "config.MIN_SENT_COUNT = 4\n",
        "root = 'http://www.'\n",
        "config.browser_user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nzlGmoffMB_Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('./news_outlets.txt','r') as f:\n",
        "    outlets = f.read().splitlines()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9MDeNSl5d7eR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "outlets = [\n",
        "            'channelnewsasia.com',\n",
        "            'straitstimes.com',\n",
        "            'ap.org',\n",
        "            'reuters.com',\n",
        "            'todayonline.com',\n",
        "            'tnp.sg',\n",
        "            'bbc.com',\n",
        "            'economist.com',\n",
        "            'businesstimes.com.sg',\n",
        "            'businessinsider.sg',\n",
        "            'nytimes.com',\n",
        "            'independent.co.uk',\n",
        "            'express.co.uk',\n",
        "            'mirror.co.uk',\n",
        "            'cnbc.com',\n",
        "            'telegraph.co.uk',\n",
        "            'washingtontimes.com',\n",
        "            'washingtonpost.com',\n",
        "            'nbcnews.com',\n",
        "            'cnn.com',\n",
        "            'time.com',\n",
        "            'latimes.com',\n",
        "            'nydailynews.com',\n",
        "            'newsroompost.com',\n",
        "            'npr.org',\n",
        "            'nationalgeographic.com',\n",
        "            'medium.com',\n",
        "            'discovery.com',\n",
        "            'sciencedaily.com',\n",
        "            'phys.org',\n",
        "            'dailymail.co.uk',\n",
        "            'cbsnews.com',\n",
        "            'huffpost.com',\n",
        "            'abcnews.go.com',\n",
        "            'dailymail.co.uk',\n",
        "            'forbes.com',\n",
        "            'scientificamerican.com',\n",
        "            'today.com',\n",
        "            'pbs.org',\n",
        "            'news.com.au',\n",
        "            'theatlantic.com',\n",
        "            'bloomberg.com',\n",
        "            'foreignaffairs.com',\n",
        "            'wsj.com',\n",
        "            'newyorker.com',\n",
        "            'fortune.com',\n",
        "            'usatoday.com',\n",
        "            'asiaone.com',\n",
        "            'nst.com.my',\n",
        "            'heraldsun.com.au',\n",
        "            'nzherald.co.nz'\n",
        "            ]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntxAyO25_JiT",
        "colab_type": "code",
        "outputId": "8c69bfd2-e427-4192-970d-8f9dac08ed6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 884
        }
      },
      "source": [
        "papers = {}\n",
        "for outlet in outlets:\n",
        "    print(outlet)\n",
        "    catemap = {}\n",
        "    paper = news.build(root+outlet,config)\n",
        "    paper.build()\n",
        "    papers[outlet] = paper"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "channelnewsasia.com\n",
            "straitstimes.com\n",
            "ap.org\n",
            "reuters.com\n",
            "todayonline.com\n",
            "tnp.sg\n",
            "bbc.com\n",
            "economist.com\n",
            "businesstimes.com.sg\n",
            "businessinsider.sg\n",
            "nytimes.com\n",
            "independent.co.uk\n",
            "express.co.uk\n",
            "mirror.co.uk\n",
            "cnbc.com\n",
            "telegraph.co.uk\n",
            "washingtontimes.com\n",
            "washingtonpost.com\n",
            "nbcnews.com\n",
            "cnn.com\n",
            "time.com\n",
            "latimes.com\n",
            "nydailynews.com\n",
            "newsroompost.com\n",
            "npr.org\n",
            "nationalgeographic.com\n",
            "medium.com\n",
            "discovery.com\n",
            "sciencedaily.com\n",
            "phys.org\n",
            "dailymail.co.uk\n",
            "cbsnews.com\n",
            "huffpost.com\n",
            "abcnews.go.com\n",
            "dailymail.co.uk\n",
            "forbes.com\n",
            "scientificamerican.com\n",
            "today.com\n",
            "pbs.org\n",
            "news.com.au\n",
            "theatlantic.com\n",
            "bloomberg.com\n",
            "foreignaffairs.com\n",
            "wsj.com\n",
            "newyorker.com\n",
            "fortune.com\n",
            "usatoday.com\n",
            "asiaone.com\n",
            "nst.com.my\n",
            "heraldsun.com.au\n",
            "nzherald.co.nz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "nArv9Hpwdj-w",
        "colab": {}
      },
      "source": [
        "news_pool = NewsPool(config)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kQ9hJWwWdjvY",
        "colab": {}
      },
      "source": [
        "news_pool.set(papers.values(), threads_per_source=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NtENlITYeYzt",
        "colab_type": "code",
        "outputId": "5c880714-6ecc-4c0b-ad1c-8231b5e3bd26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "#%config Application.log_level=\"DEBUG\"\n",
        "import datetime,dateutil,json\n",
        "import pandas as pd\n",
        "procPapers = pd.DataFrame({},columns=('url','date','outlet','category','title','text','meta_tags','meta'))\n",
        "not_done = True\n",
        "while not_done:\n",
        "    not_done = False\n",
        "    for outlet,source in papers.items():\n",
        "        for article in source.articles:\n",
        "            #resolves articles while threads are still downloading them\n",
        "            if article.is_parsed: continue\n",
        "            if article.download_state == 0:\n",
        "                not_done = True\n",
        "                continue\n",
        "            if article.download_state == 1: continue #1 is the enum value of failed\n",
        "            if article.is_media_news(): continue #if its video/picture based\n",
        "\n",
        "            article.parse()\n",
        "            article_dat = {}\n",
        "            article_dat['outlet'] = outlet\n",
        "            article_dat['meta_tags'] = set(article.meta_keywords + list(article.tags))\n",
        "            article_dat['url'] = article.url\n",
        "            article_dat['meta'] = json.dumps(article.meta_data)\n",
        "            article_dat['category'] = article.canonical_link\n",
        "            article_dat['title'] = article.title\n",
        "            article_dat['text'] = article.text\n",
        "\n",
        "            if type(article.publish_date) == datetime.datetime:\n",
        "                article_dat['date'] = article.publish_date.timestamp()\n",
        "            else:\n",
        "                try:\n",
        "                    article_dat['date'] = int(dateutil.parser.parse(article.meta_data.get('pubdate','None')))\n",
        "                except:\n",
        "                    article_dat['date'] = -1\n",
        "            procPapers = procPapers.append(article_dat,ignore_index=True)\n",
        "\n",
        "            #save ram\n",
        "            article.html = ''\n",
        "            article.article_html = ''\n",
        "            article.doc = None\n",
        "            article.top_node = None\n",
        "            article.clean_top_node = None\n",
        "            article.clean_doc = None\n",
        "            \n",
        "\n",
        "\n",
        "procPapers.to_pickle('./raw/myNews4.pkl')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Building prefix dict from /usr/local/lib/python3.6/dist-packages/jieba/dict.txt ...\n",
            "Dumping model to file cache /tmp/jieba.cache\n",
            "Loading model cost 1.6617436408996582 seconds.\n",
            "Prefix dict has been built succesfully.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dp-jgIScheV3",
        "colab_type": "text"
      },
      "source": [
        "Newspaper's keyword extraction is frequency not tfidf... <br>\n",
        "Newspaper's summarization is sentence extraction based on frequency of key word....<br>\n",
        "In conclusion: Use own extractor & paraphraser? or summarizer?<br>\n",
        "final size: 12k perhaps was ip blocked or smth (evident from the article count from certain sites), create workaround."
      ]
    }
  ]
}