Beautiful Soup - Web Scraping with Turtle Shellter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "prefix = \"https://content.codecademy.com/courses/beautifulsoup/\"\n",
    "webpage_response = requests.get('https://content.codecademy.com/courses/beautifulsoup/shellter.html')\n",
    "\n",
    "webpage = webpage_response.content\n",
    "soup = BeautifulSoup(webpage, \"html.parser\")\n",
    "\n",
    "turtle_links = soup.find_all(\"a\")\n",
    "links = []\n",
    "#go through all of the a tags and get the links associated with them\"\n",
    "for a in turtle_links:\n",
    "  links.append(prefix+a[\"href\"])\n",
    "    \n",
    "#Define turtle_data:\n",
    "turtle_data = {}\n",
    "\n",
    "#follow each link:\n",
    "for link in links:\n",
    "  webpage = requests.get(link)\n",
    "  turtle = BeautifulSoup(webpage.content, \"html.parser\")\n",
    "  turtle_name = turtle.select(\".name\")[0].get_text()\n",
    "  \n",
    "  stats = turtle.find(\"ul\")\n",
    "  stats_text = stats.get_text(\"|\")\n",
    "  stats_split = stats_text.split(\"|\")\n",
    "  stats_good = stats_split[1: :2]\n",
    "  turtle_data[turtle_name] = stats_good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "  turtle_df = pd.DataFrame.from_dict(turtle_data, orient='index', columns=['Age', 'Weight', 'Sex', 'Breed', 'Source'])\n",
    "  turtle_df.Sex = turtle_df.Sex.replace(r'\\SEX:',' ', regex=True)\n",
    "  turtle_df.Breed = turtle_df.Breed.replace(r'\\BREED:',' ', regex=True)\n",
    "  turtle_df.Source = turtle_df.Source.replace(r'\\SOURCE:',' ', regex=True)\n",
    "  turtle_df.Age = turtle_df.Age.str.extract('(\\d+)').astype(float)\n",
    "  turtle_df.Weight = turtle_df.Weight.str.extract('(\\d+)').astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Breed</th>\n",
       "      <th>Source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Aesop</th>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>Female</td>\n",
       "      <td>B  African Aquatic Sideneck Turtle</td>\n",
       "      <td>found in Lake Erie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Caesar</th>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Male</td>\n",
       "      <td>B  Greek Tortoise</td>\n",
       "      <td>hatched in house</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sulla</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Male</td>\n",
       "      <td>B  African Aquatic Sideneck Turtle</td>\n",
       "      <td>found in Lake Erie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spyro</th>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Female</td>\n",
       "      <td>B  Greek Tortoise</td>\n",
       "      <td>hatched in house</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zelda</th>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Female</td>\n",
       "      <td>B  Eastern Box Turtle</td>\n",
       "      <td>surrendered by owner</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bandicoot</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Male</td>\n",
       "      <td>B  African Aquatic Sideneck Turtle</td>\n",
       "      <td>hatched in house</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hal</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Female</td>\n",
       "      <td>B  Eastern Box Turtle</td>\n",
       "      <td>surrendered by owner</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mock</th>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>Male</td>\n",
       "      <td>B  Greek Tortoise</td>\n",
       "      <td>surrendered by owner</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sparrow</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Female</td>\n",
       "      <td>B  African Aquatic Sideneck Turtle</td>\n",
       "      <td>found in Lake Erie</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Age  Weight       Sex                               Breed  \\\n",
       "Aesop       7.0     6.0    Female  B  African Aquatic Sideneck Turtle   \n",
       "Caesar      2.0     4.0      Male                   B  Greek Tortoise   \n",
       "Sulla       1.0     1.0      Male  B  African Aquatic Sideneck Turtle   \n",
       "Spyro       6.0     3.0    Female                   B  Greek Tortoise   \n",
       "Zelda       3.0     2.0    Female               B  Eastern Box Turtle   \n",
       "Bandicoot   2.0     2.0      Male  B  African Aquatic Sideneck Turtle   \n",
       "Hal         1.0     1.0    Female               B  Eastern Box Turtle   \n",
       "Mock       10.0    10.0      Male                   B  Greek Tortoise   \n",
       "Sparrow     1.0     4.0    Female  B  African Aquatic Sideneck Turtle   \n",
       "\n",
       "                           Source  \n",
       "Aesop          found in Lake Erie  \n",
       "Caesar           hatched in house  \n",
       "Sulla          found in Lake Erie  \n",
       "Spyro            hatched in house  \n",
       "Zelda        surrendered by owner  \n",
       "Bandicoot        hatched in house  \n",
       "Hal          surrendered by owner  \n",
       "Mock         surrendered by owner  \n",
       "Sparrow        found in Lake Erie  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "turtle_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}