DataAnalyst_JobPostings_Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "70d779e4-9595-4643-8857-70ca8c398666",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "39afca03-1d1d-4149-b5a8-18be23bce250",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "def get_page(url):\n",
    "    headers = {\"User-Agent\": \"Mozilla/5.0\"}\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    return soup\n",
    "\n",
    "def parse_postings(soup):\n",
    "    # Find all posting elements\n",
    "    postings = soup.find_all(\"div\", {\"class\":\"base-search-card\"})\n",
    "        \n",
    "    data = []\n",
    "    for p in postings: \n",
    "        title = p.find(\"h3\").text.strip() \n",
    "        company = p.find(\"h4\").text.strip()\n",
    "        \n",
    "        location = p.find(\"span\", {\"class\":\"job-search-card__location\"}).text \n",
    "        skills = [s.text for s in p.find_all(\"span\", {\"class\":\"job-search-card__muted-link\"})]\n",
    "        date = p.find(\"time\")[\"datetime\"]\n",
    "        \n",
    "        link = \"https://www.linkedin.com\" + p.find(\"a\")[\"href\"]\n",
    "        \n",
    "        data.append({\"title\": title, \"company\": company, \"location\": location, \n",
    "                     \"skills\": skills, \"date\": date, \"link\": link})\n",
    "            \n",
    "    return data\n",
    "\n",
    "def main():\n",
    "    url = \"https://www.linkedin.com/jobs/data-analyst-entry-level-jobs-greater-hartford/?currentJobId=3804545423\" \n",
    "    soup = get_page(url)\n",
    "    jobs = parse_postings(soup)\n",
    "    print(jobs)\n",
    "  \n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "10129a6c-798d-4d60-924e-5c059d2edcb6",
   "metadata": {},
   "source": [
    "def get_url(position, location):\n",
    "  \"\"\"Generate LinkedIn url from search terms\"\"\"\n",
    "  template = \"https://www.linkedin.com/jobs/search/?keywords={}&location={}\"  \n",
    "  return template.format(position, location)\n",
    "\n",
    "def extract_postings(page):\n",
    "  \"\"\"Extract all job posting elements from search results\"\"\"\n",
    "  soup = BeautifulSoup(page, \"html.parser\")\n",
    "  return soup.find_all(\"div\", {\"class\": \"base-search-card\"})   \n",
    "\n",
    "def parse_posting(posting):\n",
    "  \"\"\"Extract data from a single job posting\"\"\" \n",
    "  title = posting.find(\"h3\").text.strip()\n",
    "  company = posting.find(\"h4\").text\n",
    "  location = posting.find(\"span\", {\"class\":\"job-search-card__location\"}).text\n",
    "  \n",
    "  return {\n",
    "    \"title\": title, \n",
    "    \"company\": company,\n",
    "    \"location\": location\n",
    "  }\n",
    "  \n",
    "def main():\n",
    "\n",
    "  positions = [\"data analyst\", \"data analysis\"]\n",
    "  locations = [\"Connecticut\", \"Remote\"]\n",
    "\n",
    "  for position in positions:\n",
    "    for location in locations:\n",
    "    \n",
    "      # Generate url  \n",
    "      url = get_url(position, location)\n",
    "      \n",
    "      # Fetch page and extract postings\n",
    "      page = requests.get(url)\n",
    "      postings = extract_postings(page.text)  \n",
    "      \n",
    "      jobs = []\n",
    "      for post in postings:\n",
    "         jobs.append(parse_posting(post))\n",
    "         \n",
    "      # Save listings \n",
    "      with open(f\"{position}_{location}_jobs.csv\", mode=\"w\") as f:\n",
    "         writer = csv.writer(f)  \n",
    "         headers = [\"title\", \"company\", \"location\"]  \n",
    "         writer.writerow(headers)\n",
    "         writer.writerows(jobs)\n",
    "         \n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ef44032-32c5-44d7-9814-0740d6b703f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import smtplib\n",
    "from email.message import EmailMessage\n",
    "import schedule \n",
    "import time\n",
    "\n",
    "def send_email():\n",
    "\n",
    "  # Generate CSV files with scraping script\n",
    "  \n",
    "  files = ['data_analyst_jobs.csv', 'data_analysis_jobs.csv']\n",
    "\n",
    "  msg = EmailMessage() \n",
    "  msg['Subject'] = 'Web Scraped Job Postings Data'  \n",
    "  msg['From'] = 'sender@email.com'\n",
    "  msg['To'] = 'clairehelenscanlon@gmail.com'  \n",
    "\n",
    "  for file in files:\n",
    "    with open(file, 'rb') as f:\n",
    "      file_data = f.read()\n",
    "      msg.add_attachment(file_data, maintype='application', subtype='octet-stream', filename=file)\n",
    "\n",
    "  with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp:  \n",
    "    smtp.login('myemail@gmail.com', 'password')  \n",
    "    smtp.send_message(msg)\n",
    "\n",
    "schedule.every(1).hours.do(send_email)  \n",
    "\n",
    "while True:\n",
    "    schedule.run_pending()  \n",
    "    time.sleep(1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}