news_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import urllib\n",
    "from urllib.request import Request, urlopen\n",
    "import re\n",
    "gdeltdata = './20200622073000.export.CSV'\n",
    "reader = csv.reader(open(gdeltdata),\n",
    "                    delimiter='\\t')\n",
    "counter = 0\n",
    "with open('news_raw_contents.csv', 'w', newline='') as csvfile:\n",
    "    newswriter = csv.writer(csvfile, delimiter='|')\n",
    "    \n",
    "    \n",
    "    for row in reader:\n",
    "        #print (row[40:42],row[44],row[-1])\n",
    "        if counter < 225:\n",
    "            counter+=1\n",
    "            continue\n",
    "        url = row[-1]\n",
    "        print(counter)\n",
    "\n",
    "        try:\n",
    "            req = Request(url,headers={'User-Agent': 'Mozilla/5.0'})\n",
    "            resp = urlopen(req)\n",
    "            respData = resp.read()\n",
    "\n",
    "            paragraphs = re.findall(r'<p>(.*?)</p>',str(respData))\n",
    "            rawtext=''\n",
    "            for eachP in paragraphs:\n",
    "                rawtext+=eachP.replace(\"|\", \" \")\n",
    "            newswriter.writerow([row[0],row[40],row[41],row[44],rawtext])\n",
    "        except urllib.request.HTTPError as e:\n",
    "            print(url)\n",
    "        except Exception as e:\n",
    "            print(f\"Unexpected error {e} at {url}\")\n",
    "        counter +=1\n",
    "#         if counter > 2:\n",
    "#             break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_src = pd.read_csv('climate_change_us.csv').source.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19148"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(unique_src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['forimmediaterelease.net', 'valuewalk.com', 'news957.com',\n",
       "       'news12.com', 'sfgate.com', 'prnewswire.com', 'lexology.com',\n",
       "       'myjournalcourier.com', 'mercurynews.com', 'wfsu.org', 'ksgf.com',\n",
       "       'thegardnernews.com', 'greensboro.com', 'bigislandnow.com',\n",
       "       'southwhidbeyrecord.com', 'presstelegram.com', 'lmtonline.com',\n",
       "       'newsadvance.com', 'sgvtribune.com', 'wect.com'], dtype=object)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_src[0:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import urllib\n",
    "from urllib.request import Request, urlopen\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('raw_climate_news_us_aa.csv',delimiter='|',header=None,names=['timestamp','domain','url','rawtext'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.lexology.com/library/detail.aspx?g=715bb882-603c-445c-b9b2-a2e6539778d2'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df.loc[4]['url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "float"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df.loc[2]['rawtext'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "count=0\n",
    "for row in df.itertuples():\n",
    "    if type(row[4]) == str:\n",
    "        soup = BeautifulSoup(row[4], 'html.parser')\n",
    "        txt = soup.get_text()\n",
    "        idx = txt.find('climate')\n",
    "        if idx>0:\n",
    "            count+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5092058581204917"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count/len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>domain</th>\n",
       "      <th>url</th>\n",
       "      <th>rawtext</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20180516153000</td>\n",
       "      <td>valuewalk.com</td>\n",
       "      <td>https://www.valuewalk.com/2018/05/mammal-evolu...</td>\n",
       "      <td>For decades, biologists and conservationists h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20180516154500</td>\n",
       "      <td>news957.com</td>\n",
       "      <td>http://www.news957.com/national/2018/05/16/tru...</td>\n",
       "      <td>OTTAWA &amp;#8211; Prime Minister Justin Trudeau h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20180516160000</td>\n",
       "      <td>news12.com</td>\n",
       "      <td>http://westchester.news12.com/story/38201800/j...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20180516160000</td>\n",
       "      <td>prnewswire.com</td>\n",
       "      <td>https://www.prnewswire.com/news-releases/u-hau...</td>\n",
       "      <td>&lt;span class=\"xn-location\"&gt;HOUMA, La.&lt;/span&gt;, &lt;...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20180516161500</td>\n",
       "      <td>lexology.com</td>\n",
       "      <td>https://www.lexology.com/library/detail.aspx?g...</td>\n",
       "      <td>&lt;strong&gt;Introducing PRO Compliance&lt;br&gt;&lt;/strong...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59468</th>\n",
       "      <td>20181025174500</td>\n",
       "      <td>iheart.com</td>\n",
       "      <td>https://powertalk967.iheart.com/content/2018-1...</td>\n",
       "      <td>The spookiest time of year has arrived! The th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59469</th>\n",
       "      <td>20181025174500</td>\n",
       "      <td>iheart.com</td>\n",
       "      <td>https://hotabq.iheart.com/content/2018-10-25-n...</td>\n",
       "      <td>The spookiest time of year has arrived! The th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59470</th>\n",
       "      <td>20181025174500</td>\n",
       "      <td>iheart.com</td>\n",
       "      <td>https://thebullcountry.iheart.com/content/2018...</td>\n",
       "      <td>The spookiest time of year has arrived! The th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59471</th>\n",
       "      <td>20181025174500</td>\n",
       "      <td>iheart.com</td>\n",
       "      <td>https://newsradio1170.iheart.com/content/2018-...</td>\n",
       "      <td>The spookiest time of year has arrived! The th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59472</th>\n",
       "      <td>20181025180000</td>\n",
       "      <td>iheart.com</td>\n",
       "      <td>https://wsfcam.iheart.com/content/2018-10-25-n...</td>\n",
       "      <td>The spookiest time of year has arrived! The th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>59473 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            timestamp          domain  \\\n",
       "0      20180516153000   valuewalk.com   \n",
       "1      20180516154500     news957.com   \n",
       "2      20180516160000      news12.com   \n",
       "3      20180516160000  prnewswire.com   \n",
       "4      20180516161500    lexology.com   \n",
       "...               ...             ...   \n",
       "59468  20181025174500      iheart.com   \n",
       "59469  20181025174500      iheart.com   \n",
       "59470  20181025174500      iheart.com   \n",
       "59471  20181025174500      iheart.com   \n",
       "59472  20181025180000      iheart.com   \n",
       "\n",
       "                                                     url  \\\n",
       "0      https://www.valuewalk.com/2018/05/mammal-evolu...   \n",
       "1      http://www.news957.com/national/2018/05/16/tru...   \n",
       "2      http://westchester.news12.com/story/38201800/j...   \n",
       "3      https://www.prnewswire.com/news-releases/u-hau...   \n",
       "4      https://www.lexology.com/library/detail.aspx?g...   \n",
       "...                                                  ...   \n",
       "59468  https://powertalk967.iheart.com/content/2018-1...   \n",
       "59469  https://hotabq.iheart.com/content/2018-10-25-n...   \n",
       "59470  https://thebullcountry.iheart.com/content/2018...   \n",
       "59471  https://newsradio1170.iheart.com/content/2018-...   \n",
       "59472  https://wsfcam.iheart.com/content/2018-10-25-n...   \n",
       "\n",
       "                                                 rawtext  \n",
       "0      For decades, biologists and conservationists h...  \n",
       "1      OTTAWA &#8211; Prime Minister Justin Trudeau h...  \n",
       "2                                                    NaN  \n",
       "3      <span class=\"xn-location\">HOUMA, La.</span>, <...  \n",
       "4      <strong>Introducing PRO Compliance<br></strong...  \n",
       "...                                                  ...  \n",
       "59468  The spookiest time of year has arrived! The th...  \n",
       "59469  The spookiest time of year has arrived! The th...  \n",
       "59470  The spookiest time of year has arrived! The th...  \n",
       "59471  The spookiest time of year has arrived! The th...  \n",
       "59472  The spookiest time of year has arrived! The th...  \n",
       "\n",
       "[59473 rows x 4 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59473"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## aa\n",
    "0:Findings of the study should be good news for conservationists worried about habitat loss and climate change speeding up animal extinctions\n",
    "1:Prime Minister Justin Trudeau has spoken to German Chancellor Angela Merkel about making progress on climate change and combating protectionism when they meet at the G7 next month in Quebec\n",
    "2:NA\n",
    "3:Uhaul climate-controlled\n",
    "4: In a controversial decision, the FAC argued that the negative consequences originating from climate change (especially the high carbon dioxide emissions) possess greater weight than the public benefits of the major airport expansion.\n",
    "5:His policy work has included climate change, health care, infrastructure, mental health and disability rights.\n",
    "6:NA\n",
    "7:NA\n",
    "8:NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit ('sesync': venv)",
   "language": "python",
   "name": "python37764bitsesyncvenv55ad1c69b6864cd3b51430c610567e43"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}