scraper_sun.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "39fd35c0-a8ed-4ee5-9384-9a4ca7880b9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the two main libraries\n",
    "from bs4 import BeautifulSoup       # to process html\n",
    "import requests\n",
    "import urllib.request,sys,time\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cb1d5441",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18c6936f",
   "metadata": {},
   "source": [
    "Here we scrape all the links of the sun about the war, in this case, we download the page with requests and then we look for all the anchor tags that have a class teaser-anchor teaser-anchor--search and then we save the hypertext reference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b27f8ba6-0a6e-4ed1-b85e-3ce139b62757",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 31/31 [00:22<00:00,  1.36it/s]\n"
     ]
    }
   ],
   "source": [
    "#get all the links of the articles\n",
    "\n",
    "sun_links = []\n",
    "for page in tqdm(range(1,32)):\n",
    "\n",
    "    url = 'https://www.the-sun.com/page/'+str(page)+'/?s=ukraine+war'\n",
    "    page = requests.get(url)\n",
    "    soup = BeautifulSoup(page.text, \"html.parser\")\n",
    "    links = soup.find_all('a',{'teaser-anchor teaser-anchor--search'})\n",
    "    \n",
    "    for link in links:\n",
    "        sun_links.append(link['href'].strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12f83104",
   "metadata": {},
   "source": [
    "Since sometimes the reply by the server is an HTML page because there is an issue loading the JSON file we need to try to parse the JSON in order to see if it's a JSON or not and then remove the ones that are HTML."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ed8c543b-55a4-442c-bb93-3ac62455d02c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "file_name = \"sun_links.pkl\"\n",
    "\n",
    "open_file = open(file_name, \"wb\")\n",
    "pickle.dump(sun_links, open_file)\n",
    "open_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7671ca93-5df0-48ef-9f2c-1520d9eaed30",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to load\n",
    "open_file = open(\"sun_links.pkl\", \"rb\")\n",
    "sun_links = pickle.load(open_file)\n",
    "open_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "85e71d91-dfc4-449d-82f7-16b3541ed2b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating the dataframe\n",
    "\n",
    "SunArticles = pd.DataFrame({'title': pd.Series([], dtype='string'),\n",
    "                             'author': pd.Series([], dtype='string'),\n",
    "                             'date': pd.Series([], dtype='float'),\n",
    "                             'text': pd.Series([], dtype='string'),\n",
    "                             })\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c15881a3",
   "metadata": {},
   "source": [
    "Here we load the links that we found before and we store the data scraped on the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "9e51c2a5-e10f-43fe-8012-462b9e65de2c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "304it [03:18,  1.53it/s]\n"
     ]
    }
   ],
   "source": [
    "for i,link in tqdm(enumerate(sun_links)):\n",
    "    \n",
    "    page = requests.get(link)\n",
    "    soup = BeautifulSoup(page.text, \"html.parser\")\n",
    "    \n",
    "    #get title\n",
    "    title = soup.find_all(\"h1\", {\"class\": \"article__headline\"})[0].get_text()\n",
    "    \n",
    "    #get author\n",
    "    authors = soup.find_all('a', {'class': 'author url fn article__author-link t-p-color'}) \n",
    "    if len(authors) > 0:\n",
    "        author= authors[0].get_text()\n",
    "    elif len(soup.find_all('span', {'class': 'article__author-name t-p-color'})) > 0:\n",
    "        author = soup.find_all('span', {'class': 'article__author-name t-p-color'})[0].get_text()\n",
    "    else:\n",
    "        author = ''\n",
    "    #get date\n",
    "    date = soup.find_all('span', {'class': 'article__timestamp'})[0].get_text()\n",
    "    date = pd.to_datetime(date).strftime('%d/%m/%y')\n",
    "    \n",
    "    #get text\n",
    "    text = ''\n",
    "    for p in soup.find_all('p'):\n",
    "        text += p.text\n",
    "    \n",
    "    SunArticles = SunArticles.append({\"title\": title, \"author\": author, \"date\": date, \"text\": text}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "b9ec7781-24a8-4b60-8823-9e390903237d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ukraine news latest — Kyiv issues fierce messa...</td>\n",
       "      <td>Louis Allwood</td>\n",
       "      <td>29/04/23</td>\n",
       "      <td>UKRAINE sent a defiant message to Russia's Vla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pentagon leak suspect Jack Teixeira, 21, ‘used...</td>\n",
       "      <td>Israel Salas-Rodriguez</td>\n",
       "      <td>27/04/23</td>\n",
       "      <td>PENTAGON leak suspect Jack Teixeria had used h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>China’s Xi Jinping makes pledge to Zelensky in...</td>\n",
       "      <td>Nick Parker</td>\n",
       "      <td>26/04/23</td>\n",
       "      <td>CHINA’s leader Xi Jinping pledged not to add “...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Schoolkids chanting ‘he’s OUR King’ drown out ...</td>\n",
       "      <td>Matt Wilkinson</td>\n",
       "      <td>26/04/23</td>\n",
       "      <td>FLAG-WAVING school children drowned out a nois...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Russian fighter jet bursts into flames before ...</td>\n",
       "      <td>Henry Holloway</td>\n",
       "      <td>26/04/23</td>\n",
       "      <td>A RUSSIAN fighter jet burst into flames and cr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>Putin ‘may exploit holidays &amp; take military ac...</td>\n",
       "      <td>Chris Bradford</td>\n",
       "      <td>22/12/21</td>\n",
       "      <td>VLADIMIR Putin may exploit Christmas and take ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>US detects Russia massing 175,000 troops on bo...</td>\n",
       "      <td>H. J. Hayes</td>\n",
       "      <td>03/12/21</td>\n",
       "      <td>US intelligence has detected Russia massing 17...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>Dramatic moment US fighter jets intercept Russ...</td>\n",
       "      <td>Will Stewart</td>\n",
       "      <td>28/04/21</td>\n",
       "      <td>DRAMATIC video catches the cat-and-mouse momen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>Biden warns Putin the US will act firmly in de...</td>\n",
       "      <td>Mollie Mansfield</td>\n",
       "      <td>13/04/21</td>\n",
       "      <td>JOE Biden warned Vladimir Putin that the Unite...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>What is Martial law? Meaning explained</td>\n",
       "      <td>David Boroff</td>\n",
       "      <td>12/01/21</td>\n",
       "      <td>MARTIAL Law can be imposed in times of intense...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>304 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 title  \\\n",
       "0    Ukraine news latest — Kyiv issues fierce messa...   \n",
       "1    Pentagon leak suspect Jack Teixeira, 21, ‘used...   \n",
       "2    China’s Xi Jinping makes pledge to Zelensky in...   \n",
       "3    Schoolkids chanting ‘he’s OUR King’ drown out ...   \n",
       "4    Russian fighter jet bursts into flames before ...   \n",
       "..                                                 ...   \n",
       "299  Putin ‘may exploit holidays & take military ac...   \n",
       "300  US detects Russia massing 175,000 troops on bo...   \n",
       "301  Dramatic moment US fighter jets intercept Russ...   \n",
       "302  Biden warns Putin the US will act firmly in de...   \n",
       "303             What is Martial law? Meaning explained   \n",
       "\n",
       "                     author      date  \\\n",
       "0             Louis Allwood  29/04/23   \n",
       "1    Israel Salas-Rodriguez  27/04/23   \n",
       "2               Nick Parker  26/04/23   \n",
       "3            Matt Wilkinson  26/04/23   \n",
       "4            Henry Holloway  26/04/23   \n",
       "..                      ...       ...   \n",
       "299          Chris Bradford  22/12/21   \n",
       "300             H. J. Hayes  03/12/21   \n",
       "301            Will Stewart  28/04/21   \n",
       "302        Mollie Mansfield  13/04/21   \n",
       "303            David Boroff  12/01/21   \n",
       "\n",
       "                                                  text  \n",
       "0    UKRAINE sent a defiant message to Russia's Vla...  \n",
       "1    PENTAGON leak suspect Jack Teixeria had used h...  \n",
       "2    CHINA’s leader Xi Jinping pledged not to add “...  \n",
       "3    FLAG-WAVING school children drowned out a nois...  \n",
       "4    A RUSSIAN fighter jet burst into flames and cr...  \n",
       "..                                                 ...  \n",
       "299  VLADIMIR Putin may exploit Christmas and take ...  \n",
       "300  US intelligence has detected Russia massing 17...  \n",
       "301  DRAMATIC video catches the cat-and-mouse momen...  \n",
       "302  JOE Biden warned Vladimir Putin that the Unite...  \n",
       "303  MARTIAL Law can be imposed in times of intense...  \n",
       "\n",
       "[304 rows x 4 columns]"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SunArticles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "c117913b-0bc7-4691-bbd0-bc3efed8770f",
   "metadata": {},
   "outputs": [],
   "source": [
    "SunArticles.to_csv(\"SunArticles.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}