LinkedIn_sraping_file.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimaze the Linkedin parsing code "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import requests\n",
    "import time\n",
    "import csv\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "## First we need a browser to run the selenium automation \n",
    "browser = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Next let's Open the page that we will scrape in our case we will scrape the linkedin platform \n",
    "# Following the step we are in the sign in page of linkedin and it's time to insert our personal informations\n",
    "browser.get('https://www.linkedin.com/login')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "## In this step I will read a file that contains my informations in order to keep them private\n",
    "file = open(\"info.txt\")\n",
    "lines = file.readlines()\n",
    "username = lines[0]\n",
    "password = lines[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Let's Sign in\n",
    "# Find the username box and add your username\n",
    "elementID = browser.find_element_by_id('username')\n",
    "elementID.send_keys(username)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the password box and add your password\n",
    "elementID = browser.find_element_by_id('password')\n",
    "elementID.send_keys(password)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "## click to submit join option\n",
    "elementID.submit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Determine the link of the search  \n",
    "link = 'https://www.linkedin.com/search/results/companies/?keywords=staffing%20%26%20recruiting&origin=SWITCH_SEARCH_VERTICAL'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Get the link \n",
    "browser.get(link)\n",
    "time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now we are in the page that we want to scrape\n",
    "## I will use the beautiful soup library to scrape the page\n",
    "# Our goal is to parse all the companies with some details that will be useful for the competition analysis \n",
    "page = bs(browser.page_source, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating a function in order to collect the information that we need from the pages\n",
    "\n",
    "def get_features(card):\n",
    "    href = card.a\n",
    "    title = card.find(\"span\", \"entity-result__title-line flex-shrink-1 entity-result__title-text--black\").text.replace(\"\\n\",\"\")\n",
    "    link = href.get(\"href\")\n",
    "    try:\n",
    "        sector = card.find(\"div\", \"entity-result__primary-subtitle t-14 t-black\").text.replace(\"\\n\", \"\")\n",
    "    except:\n",
    "        sector = \"None\"\n",
    "    followers = card.find(\"div\", \"entity-result__secondary-subtitle t-14\").text.replace(\"\\n\",\"\")\n",
    "    try:\n",
    "        description = card.find(\"p\", \"entity-result__summary t-12 t-black--light mb1\").text.replace(\"\\n\",\"\")\n",
    "    except:\n",
    "        description = \"None\"\n",
    "    try:  \n",
    "        current_positions = card.find(\"span\", \"entity-result__simple-insight-text\").text.replace(\"\\n\",\"\")\n",
    "    except:\n",
    "        current_positions = \"None\"\n",
    "        \n",
    "    feature = (title, link, sector, followers, description, current_positions)    \n",
    "    return feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Determing the number of the pages that you want to scrape:4\n"
     ]
    }
   ],
   "source": [
    "## Determining the number of the pages that you want to parse data from.\n",
    "n = input(\"Determing the number of the pages that you want to scrape:\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Append the different url's into a list\n",
    "pages = []\n",
    "pages.append(link) \n",
    "\n",
    "def number_of_pages(n):\n",
    "    new_url = 'https://www.linkedin.com/search/results/companies/?keywords=staffing%20%26%20recruiting&origin=SWITCH_SEARCH_VERTICAL' +'&page={num:.2f}'\n",
    "    for i in range(2,n+1):\n",
    "        try:\n",
    "            pages.append(new_url.format(num=i))\n",
    "        except AttributeError:\n",
    "            break\n",
    "    return pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.linkedin.com/search/results/companies/?keywords=staffing%20%26%20recruiting&origin=SWITCH_SEARCH_VERTICAL',\n",
       " 'https://www.linkedin.com/search/results/companies/?keywords=staffing%20%26%20recruiting&origin=SWITCH_SEARCH_VERTICAL&page=2.00',\n",
       " 'https://www.linkedin.com/search/results/companies/?keywords=staffing%20%26%20recruiting&origin=SWITCH_SEARCH_VERTICAL&page=3.00',\n",
       " 'https://www.linkedin.com/search/results/companies/?keywords=staffing%20%26%20recruiting&origin=SWITCH_SEARCH_VERTICAL&page=4.00']"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Inspecting the URLs list\n",
    "n = int(n)\n",
    "number_of_pages(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Iterating through the pages and parse all the features used the scraping function\n",
    "final_list = []\n",
    "for link in pages:\n",
    "    browser.get(link)\n",
    "    page = bs(browser.page_source, \"html.parser\")\n",
    "    cards = page.find_all(\"div\", \"entity-result\")\n",
    "    \n",
    "    time.sleep(4)\n",
    "    \n",
    "    for card in cards:\n",
    "            features = get_features(card)\n",
    "            final_list.append(features)\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Print the length  of the link\n",
    "len(final_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Convert the list to csv file\n",
    "import csv\n",
    "with open('new_oper.csv', 'w', newline='', encoding='utf-8') as f:\n",
    "        writer = csv.writer(f)\n",
    "        writer.writerow([\"title\", \"link\", \"sector\", \"followers\", \"description\", \"current_positions\"])\n",
    "        writer.writerows(final_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40, 6)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"new_oper.csv\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50, 6)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Remove duplicates \n",
    "new_df = df.drop_duplicates(subset=['title'])\n",
    "new_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Save the dataframe to a new csv file \n",
    "new_df.to_csv(\"final_file.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7 (tensorflow)",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}