Naukri_data_scraping/Naukri Web Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WEB SCRAPING using SELENIUM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Web Scraping:-"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load library\n",
    "import csv\n",
    "import time\n",
    "from selenium import webdriver\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# chromedriver url link\n",
    "driver=webdriver.Chrome('C:\\Program Files (x86)\\Google\\chromedriver')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dictionary\n",
    "jobs={\"roles\":[],\n",
    "     \"companies\":[],\n",
    "     \"locations\":[],\n",
    "     \"experience\":[],\n",
    "     \"skills\":[]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# link of website you want to scrap\n",
    "for i in range(2):\n",
    "    driver.get(\"https://www.naukri.com/data-scientist-jobs-{}\".format(i))\n",
    "    time.sleep(3)\n",
    "    lst=driver.find_elements_by_css_selector(\".jobTuple.bgWhite.br4.mb-8\")\n",
    "    \n",
    "    # scrape the data from website\n",
    "    for job in lst:\n",
    "        driver.implicitly_wait(10)\n",
    "        role=job.find_element_by_css_selector(\"a.title.fw500.ellipsis\").text\n",
    "        company=job.find_element_by_css_selector(\"a.subTitle.ellipsis.fleft\").text\n",
    "        location=job.find_element_by_css_selector(\".fleft.grey-text.br2.placeHolderLi.location\").text\n",
    "        exp=job.find_element_by_css_selector(\".fleft.grey-text.br2.placeHolderLi.experience\").text\n",
    "        skills=job.find_element_by_css_selector(\".tags.has-description\").text\n",
    "        jobs[\"roles\"].append(role)\n",
    "        jobs[\"companies\"].append(company)\n",
    "        jobs[\"locations\"].append(location)\n",
    "        jobs[\"experience\"].append(exp)\n",
    "        jobs[\"skills\"].append(skills)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'roles': ['Senior / Lead Data Scientist - Data Analysis, Machine Learning', 'Gig Opportunity - Internal Audit: Data Scientist - Data Modelling', 'Data Scientist/ Data Analyst', 'Data Scientist - Image Recognition/Machine Learning', 'Data Scientist - Image Processing/ Machine Learning', 'Data Scientist - Image Processing/Machine Learning', 'Senior Data Scientist', 'Senior Data Scientist - Machine Learning/Deep Learning', 'Data Scientist/ Analyst - CRMnext - 2-5 years - Mumbai', 'Senior Data Scientist, Rider Analytics', 'Data Scientist / Research Analyst', 'Data Scientist / Research Analyst', 'Senior Data Scientist [ML Model,azure ML Pipeline,python,r ]', 'Senior Expert Data Scientist', 'Technical Specialist - Data Scientist', 'Urgent opening For Data Scientists', 'Data Scientist - Machine/ Deep Learning Algorithms', 'Data Scientist', 'Senior Manager – Data Scientist', 'Data Scientist / Machine Learning Engineer', 'Gig Opportunity - Internal Audit: Data Scientist - Data Modelling', 'Data Scientist/ Data Analyst', 'Data Scientist - Image Recognition/Machine Learning', 'Data Scientist - Image Processing/ Machine Learning', 'Data Scientist - Image Processing/Machine Learning', 'Senior / Lead Data Scientist - Data Analysis, Machine Learning', 'Senior Data Scientist', 'Senior Data Scientist - Machine Learning/Deep Learning', 'Data Scientist/ Analyst - CRMnext - 2-5 years - Mumbai', 'Data Scientist / Machine Learning Engineer', 'Senior Data Scientist, Rider Analytics', 'Data Scientist / Research Analyst', 'Data Scientist / Research Analyst', 'Senior Data Scientist [ML Model,azure ML Pipeline,python,r ]', 'Senior Expert Data Scientist', 'Technical Specialist - Data Scientist', 'Urgent opening For Data Scientists', 'Data Scientist', 'Senior Manager – Data Scientist', 'Senior Data Scientist - Credit risk'], 'companies': ['HERE Technologies', 'Axis Bank Limited', 'CARS24', 'CarbyneTech India', 'CarbyneTech India', 'CarbyneTech India', 'TVS Credit', 'Tekion', 'Acidaes Solutions Pvt. Ltd.', 'Uber', 'Franklin Templeton International', 'Franklin Templeton International', 'eInfochips Limited', 'Novartis Healthcare Pvt. Ltd.', 'BIRLASOFT LIMITED', 'Sitel India Pvt Ltd', 'CarbyneTech India', 'GANIT BUSINESS SOLUTIONS PRIVATE LIMITED', 'Piramal Swasthya', 'INFISWIFT TECHNOLOGIES', 'Axis Bank Limited', 'CARS24', 'CarbyneTech India', 'CarbyneTech India', 'CarbyneTech India', 'HERE Technologies', 'TVS Credit', 'Tekion', 'Acidaes Solutions Pvt. Ltd.', 'INFISWIFT TECHNOLOGIES', 'Uber', 'Franklin Templeton International', 'Franklin Templeton International', 'eInfochips Limited', 'Novartis Healthcare Pvt. Ltd.', 'BIRLASOFT LIMITED', 'Sitel India Pvt Ltd', 'GANIT BUSINESS SOLUTIONS PRIVATE LIMITED', 'Piramal Swasthya', 'Scienaptic Systems'], 'locations': ['Mumbai, Mumbai City', 'Chennai, Pune, Delhi NCR, Mumbai, Ahmedabad, Bengaluru, Chandigarh, Hyderabad, Kolkata', 'Gurgaon Gurugram', 'Hyderabad', 'Hyderabad', 'Hyderabad', 'Chennai, Pune, Bengaluru', 'Bengaluru', 'Mumbai Suburbs, Navi Mumbai, Mumbai', 'Bengaluru', 'Hyderabad', 'Hyderabad', 'Bengaluru(6th block Koramangala)', 'Hyderabad', 'Pune, Bengaluru', 'Mumbai, Hyderabad', 'Hyderabad', 'Chennai, Delhi NCR, Mumbai, Bengaluru, Hyderabad, Gurgaon', 'Hyderabad', 'Pune', 'Chennai, Pune, Delhi NCR, Mumbai, Ahmedabad, Bengaluru, Chandigarh, Hyderabad, Kolkata', 'Gurgaon Gurugram', 'Hyderabad', 'Hyderabad', 'Hyderabad', 'Mumbai, Mumbai City', 'Chennai, Pune, Bengaluru', 'Bengaluru', 'Mumbai Suburbs, Navi Mumbai, Mumbai', 'Pune', 'Bengaluru', 'Hyderabad', 'Hyderabad', 'Bengaluru(6th block Koramangala)', 'Hyderabad', 'Pune, Bengaluru', 'Mumbai, Hyderabad', 'Chennai, Delhi NCR, Mumbai, Bengaluru, Hyderabad, Gurgaon', 'Hyderabad', 'Bengaluru'], 'experience': ['4-9 Yrs', '5-10 Yrs', '1-5 Yrs', '5-10 Yrs', '6-11 Yrs', '6-11 Yrs', '4-9 Yrs', '5-9 Yrs', '2-5 Yrs', '5-10 Yrs', '2-6 Yrs', '2-6 Yrs', '8-12 Yrs', '5-9 Yrs', '3-7 Yrs', '8-11 Yrs', '3-8 Yrs', '2-5 Yrs', '7-10 Yrs', '3-7 Yrs', '5-10 Yrs', '1-5 Yrs', '5-10 Yrs', '6-11 Yrs', '6-11 Yrs', '4-9 Yrs', '4-9 Yrs', '5-9 Yrs', '2-5 Yrs', '3-7 Yrs', '5-10 Yrs', '2-6 Yrs', '2-6 Yrs', '8-12 Yrs', '5-9 Yrs', '3-7 Yrs', '8-11 Yrs', '2-5 Yrs', '7-10 Yrs', '6-11 Yrs'], 'skills': ['IT Skills\\nPython\\nData Science\\nMachine Learning\\nCloud\\nBig Data\\nAWS\\nStatistical modeling', 'Data Science\\nR\\nReport Generation\\nRisk Assessment\\nArtificial Intelligence\\nControl Testing\\nData Modeling\\nMachine Learning', 'Data Science\\nPredictive Modeling\\nData Scientist\\nData Management\\nBig Data\\nTableau\\nData Analytics\\nAnalytics', 'IT Skills\\nJava\\nPython\\nMachine Learning\\nArtificial Intelligence\\nNLP\\nData Scientist\\nTime Series Analysis', 'Predictive Modeling\\nData Science\\nData Scientist\\nTime Series Analysis\\nImage Processing\\nData Management\\nStatistical Analyst\\nMachine Learning', 'Predictive Modeling\\nData Science\\nData Scientist\\nTime Series Analysis\\nImage Processing\\nData Management\\nStatistical Analyst\\nMachine Learning', 'Big Data Analytics\\nHadoop\\nData Transformation Services\\nData Acquisition\\nMachine Learning\\nSQL\\nNoSQL\\nData Scientist', 'IT Skills\\nPython\\nData Science\\nMachine Learning\\nBig Data\\nData Science\\nNLP\\nData Scientist', 'IT Skills\\nPython\\nCloud\\nBig Data\\nData Science\\nData Scientist\\nData Analyst\\nMachine Learning', 'Computer science\\nProduct engineering\\ndata science\\nmanagement\\nHypothesis Testing\\nStatistics\\nAnalytics\\nSQL', 'Data analysis\\nCoding\\nFixed income\\nMachine learning\\nHTTP\\nInvestment management\\nMedical insurance\\nData mining', 'Data analysis\\nCoding\\nFixed income\\nMachine learning\\nHTTP\\nInvestment management\\nMedical insurance\\nData mining', 'IT Skills\\nPython\\nData Science\\nAzure\\nData Science\\nR\\ntensorflow\\nPickle', 'Diagnostics\\ndata science\\nAnalytical\\nMachine learning\\nExpert\\nClinical trials\\nPharmacokinetics\\nInternship', 'Boosting\\nIT Skills\\nPython\\nMachine Learning\\nCloud\\nR\\nNLP\\nGLM/Regression', 'Pytorch\\nData Science\\nSupervised Learning\\nNLP\\nNatural Language Processing\\nKeras\\nMachine Learning\\nDeep Learning', 'IT Skills\\nPython\\nMachine Learning\\nData Science\\nR\\nNLP\\nData Scientist\\nArtificial Intelligence', 'python\\ndata analytics\\nEDA\\nSAS\\nPower Bi\\ntime series\\nbusiness analytics\\nbusiness intelligence', 'Business Strategy\\nPredictive Modeling\\nA/B Testing\\nData Mining\\nStatistics\\nR\\nAlgorithms\\nSLQ', 'Computer science\\nData analysis\\nImage processing\\nBusiness analytics\\nMachine learning\\nNatural language processing\\nSensors\\nbig data', 'Data Science\\nR\\nReport Generation\\nRisk Assessment\\nArtificial Intelligence\\nControl Testing\\nData Modeling\\nMachine Learning', 'Data Science\\nPredictive Modeling\\nData Scientist\\nData Management\\nBig Data\\nTableau\\nData Analytics\\nAnalytics', 'IT Skills\\nJava\\nPython\\nMachine Learning\\nArtificial Intelligence\\nNLP\\nData Scientist\\nTime Series Analysis', 'Predictive Modeling\\nData Science\\nData Scientist\\nTime Series Analysis\\nImage Processing\\nData Management\\nStatistical Analyst\\nMachine Learning', 'Predictive Modeling\\nData Science\\nData Scientist\\nTime Series Analysis\\nImage Processing\\nData Management\\nStatistical Analyst\\nMachine Learning', 'IT Skills\\nPython\\nData Science\\nMachine Learning\\nCloud\\nBig Data\\nAWS\\nStatistical modeling', 'Big Data Analytics\\nHadoop\\nData Transformation Services\\nData Acquisition\\nMachine Learning\\nSQL\\nNoSQL\\nData Scientist', 'IT Skills\\nPython\\nData Science\\nMachine Learning\\nBig Data\\nData Science\\nNLP\\nData Scientist', 'IT Skills\\nPython\\nCloud\\nBig Data\\nData Science\\nData Scientist\\nData Analyst\\nMachine Learning', 'Computer science\\nData analysis\\nImage processing\\nBusiness analytics\\nMachine learning\\nNatural language processing\\nSensors\\nbig data', 'Computer science\\nProduct engineering\\ndata science\\nmanagement\\nHypothesis Testing\\nStatistics\\nAnalytics\\nSQL', 'Data analysis\\nCoding\\nFixed income\\nMachine learning\\nHTTP\\nInvestment management\\nMedical insurance\\nData mining', 'Data analysis\\nCoding\\nFixed income\\nMachine learning\\nHTTP\\nInvestment management\\nMedical insurance\\nData mining', 'IT Skills\\nPython\\nData Science\\nAzure\\nData Science\\nR\\ntensorflow\\nPickle', 'Diagnostics\\ndata science\\nAnalytical\\nMachine learning\\nExpert\\nClinical trials\\nPharmacokinetics\\nInternship', 'Boosting\\nIT Skills\\nPython\\nMachine Learning\\nCloud\\nR\\nNLP\\nGLM/Regression', 'Pytorch\\nData Science\\nSupervised Learning\\nNLP\\nNatural Language Processing\\nKeras\\nMachine Learning\\nDeep Learning', 'python\\ndata analytics\\nEDA\\nSAS\\nPower Bi\\ntime series\\nbusiness analytics\\nbusiness intelligence', 'Business Strategy\\nPredictive Modeling\\nA/B Testing\\nData Mining\\nStatistics\\nR\\nAlgorithms\\nSLQ', 'IT Skills\\nPython\\nTesting\\nData Science\\nMachine Learning\\nComputer science\\nText mining\\nAutomation']}\n"
     ]
    }
   ],
   "source": [
    "# print scrape data\n",
    "print(jobs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>roles</th>\n",
       "      <th>companies</th>\n",
       "      <th>locations</th>\n",
       "      <th>experience</th>\n",
       "      <th>skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Senior / Lead Data Scientist - Data Analysis, ...</td>\n",
       "      <td>HERE Technologies</td>\n",
       "      <td>Mumbai, Mumbai City</td>\n",
       "      <td>4-9 Yrs</td>\n",
       "      <td>IT Skills\\nPython\\nData Science\\nMachine Learn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gig Opportunity - Internal Audit: Data Scienti...</td>\n",
       "      <td>Axis Bank Limited</td>\n",
       "      <td>Chennai, Pune, Delhi NCR, Mumbai, Ahmedabad, B...</td>\n",
       "      <td>5-10 Yrs</td>\n",
       "      <td>Data Science\\nR\\nReport Generation\\nRisk Asses...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Scientist/ Data Analyst</td>\n",
       "      <td>CARS24</td>\n",
       "      <td>Gurgaon Gurugram</td>\n",
       "      <td>1-5 Yrs</td>\n",
       "      <td>Data Science\\nPredictive Modeling\\nData Scient...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data Scientist - Image Recognition/Machine Lea...</td>\n",
       "      <td>CarbyneTech India</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>5-10 Yrs</td>\n",
       "      <td>IT Skills\\nJava\\nPython\\nMachine Learning\\nArt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Data Scientist - Image Processing/ Machine Lea...</td>\n",
       "      <td>CarbyneTech India</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>6-11 Yrs</td>\n",
       "      <td>Predictive Modeling\\nData Science\\nData Scient...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Data Scientist - Image Processing/Machine Lear...</td>\n",
       "      <td>CarbyneTech India</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>6-11 Yrs</td>\n",
       "      <td>Predictive Modeling\\nData Science\\nData Scient...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Senior Data Scientist</td>\n",
       "      <td>TVS Credit</td>\n",
       "      <td>Chennai, Pune, Bengaluru</td>\n",
       "      <td>4-9 Yrs</td>\n",
       "      <td>Big Data Analytics\\nHadoop\\nData Transformatio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Senior Data Scientist - Machine Learning/Deep ...</td>\n",
       "      <td>Tekion</td>\n",
       "      <td>Bengaluru</td>\n",
       "      <td>5-9 Yrs</td>\n",
       "      <td>IT Skills\\nPython\\nData Science\\nMachine Learn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Data Scientist/ Analyst - CRMnext - 2-5 years ...</td>\n",
       "      <td>Acidaes Solutions Pvt. Ltd.</td>\n",
       "      <td>Mumbai Suburbs, Navi Mumbai, Mumbai</td>\n",
       "      <td>2-5 Yrs</td>\n",
       "      <td>IT Skills\\nPython\\nCloud\\nBig Data\\nData Scien...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Senior Data Scientist, Rider Analytics</td>\n",
       "      <td>Uber</td>\n",
       "      <td>Bengaluru</td>\n",
       "      <td>5-10 Yrs</td>\n",
       "      <td>Computer science\\nProduct engineering\\ndata sc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               roles  \\\n",
       "0  Senior / Lead Data Scientist - Data Analysis, ...   \n",
       "1  Gig Opportunity - Internal Audit: Data Scienti...   \n",
       "2                       Data Scientist/ Data Analyst   \n",
       "3  Data Scientist - Image Recognition/Machine Lea...   \n",
       "4  Data Scientist - Image Processing/ Machine Lea...   \n",
       "5  Data Scientist - Image Processing/Machine Lear...   \n",
       "6                              Senior Data Scientist   \n",
       "7  Senior Data Scientist - Machine Learning/Deep ...   \n",
       "8  Data Scientist/ Analyst - CRMnext - 2-5 years ...   \n",
       "9             Senior Data Scientist, Rider Analytics   \n",
       "\n",
       "                     companies  \\\n",
       "0            HERE Technologies   \n",
       "1            Axis Bank Limited   \n",
       "2                       CARS24   \n",
       "3            CarbyneTech India   \n",
       "4            CarbyneTech India   \n",
       "5            CarbyneTech India   \n",
       "6                   TVS Credit   \n",
       "7                       Tekion   \n",
       "8  Acidaes Solutions Pvt. Ltd.   \n",
       "9                         Uber   \n",
       "\n",
       "                                           locations experience  \\\n",
       "0                                Mumbai, Mumbai City    4-9 Yrs   \n",
       "1  Chennai, Pune, Delhi NCR, Mumbai, Ahmedabad, B...   5-10 Yrs   \n",
       "2                                   Gurgaon Gurugram    1-5 Yrs   \n",
       "3                                          Hyderabad   5-10 Yrs   \n",
       "4                                          Hyderabad   6-11 Yrs   \n",
       "5                                          Hyderabad   6-11 Yrs   \n",
       "6                           Chennai, Pune, Bengaluru    4-9 Yrs   \n",
       "7                                          Bengaluru    5-9 Yrs   \n",
       "8                Mumbai Suburbs, Navi Mumbai, Mumbai    2-5 Yrs   \n",
       "9                                          Bengaluru   5-10 Yrs   \n",
       "\n",
       "                                              skills  \n",
       "0  IT Skills\\nPython\\nData Science\\nMachine Learn...  \n",
       "1  Data Science\\nR\\nReport Generation\\nRisk Asses...  \n",
       "2  Data Science\\nPredictive Modeling\\nData Scient...  \n",
       "3  IT Skills\\nJava\\nPython\\nMachine Learning\\nArt...  \n",
       "4  Predictive Modeling\\nData Science\\nData Scient...  \n",
       "5  Predictive Modeling\\nData Science\\nData Scient...  \n",
       "6  Big Data Analytics\\nHadoop\\nData Transformatio...  \n",
       "7  IT Skills\\nPython\\nData Science\\nMachine Learn...  \n",
       "8  IT Skills\\nPython\\nCloud\\nBig Data\\nData Scien...  \n",
       "9  Computer science\\nProduct engineering\\ndata sc...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tranform data into dataframe\n",
    "df=pd.DataFrame.from_dict(jobs)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-Processing:-"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>roles</th>\n",
       "      <th>companies</th>\n",
       "      <th>locations</th>\n",
       "      <th>experience</th>\n",
       "      <th>skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>senior / lead data scientist - data analysis, ...</td>\n",
       "      <td>here technologies</td>\n",
       "      <td>mumbai, mumbai city</td>\n",
       "      <td>4-9 yrs</td>\n",
       "      <td>it skills\\npython\\ndata science\\nmachine learn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gig opportunity - internal audit: data scienti...</td>\n",
       "      <td>axis bank limited</td>\n",
       "      <td>chennai, pune, delhi ncr, mumbai, ahmedabad, b...</td>\n",
       "      <td>5-10 yrs</td>\n",
       "      <td>data science\\nr\\nreport generation\\nrisk asses...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data scientist/ data analyst</td>\n",
       "      <td>cars24</td>\n",
       "      <td>gurgaon gurugram</td>\n",
       "      <td>1-5 yrs</td>\n",
       "      <td>data science\\npredictive modeling\\ndata scient...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data scientist - image recognition/machine lea...</td>\n",
       "      <td>carbynetech india</td>\n",
       "      <td>hyderabad</td>\n",
       "      <td>5-10 yrs</td>\n",
       "      <td>it skills\\njava\\npython\\nmachine learning\\nart...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data scientist - image processing/ machine lea...</td>\n",
       "      <td>carbynetech india</td>\n",
       "      <td>hyderabad</td>\n",
       "      <td>6-11 yrs</td>\n",
       "      <td>predictive modeling\\ndata science\\ndata scient...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               roles          companies  \\\n",
       "0  senior / lead data scientist - data analysis, ...  here technologies   \n",
       "1  gig opportunity - internal audit: data scienti...  axis bank limited   \n",
       "2                       data scientist/ data analyst             cars24   \n",
       "3  data scientist - image recognition/machine lea...  carbynetech india   \n",
       "4  data scientist - image processing/ machine lea...  carbynetech india   \n",
       "\n",
       "                                           locations experience  \\\n",
       "0                                mumbai, mumbai city    4-9 yrs   \n",
       "1  chennai, pune, delhi ncr, mumbai, ahmedabad, b...   5-10 yrs   \n",
       "2                                   gurgaon gurugram    1-5 yrs   \n",
       "3                                          hyderabad   5-10 yrs   \n",
       "4                                          hyderabad   6-11 yrs   \n",
       "\n",
       "                                              skills  \n",
       "0  it skills\\npython\\ndata science\\nmachine learn...  \n",
       "1  data science\\nr\\nreport generation\\nrisk asses...  \n",
       "2  data science\\npredictive modeling\\ndata scient...  \n",
       "3  it skills\\njava\\npython\\nmachine learning\\nart...  \n",
       "4  predictive modeling\\ndata science\\ndata scient...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lower case all the strings to avoid redundancy\n",
    "df=df.apply(lambda x: x.astype(str).str.lower())\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>roles</th>\n",
       "      <th>companies</th>\n",
       "      <th>locations</th>\n",
       "      <th>experience</th>\n",
       "      <th>skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>urgent opening for data scientists</td>\n",
       "      <td>sitel india pvt ltd</td>\n",
       "      <td>[mumbai,  hyderabad]</td>\n",
       "      <td>8-11 yrs</td>\n",
       "      <td>[pytorch, data science, supervised learning, n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>data scientist - machine/ deep learning algori...</td>\n",
       "      <td>carbynetech india</td>\n",
       "      <td>[hyderabad]</td>\n",
       "      <td>3-8 yrs</td>\n",
       "      <td>[it skills, python, machine learning, data sci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>data scientist</td>\n",
       "      <td>ganit business solutions private limited</td>\n",
       "      <td>[chennai,  delhi ncr,  mumbai,  bengaluru,  hy...</td>\n",
       "      <td>2-5 yrs</td>\n",
       "      <td>[python, data analytics, eda, sas, power bi, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>senior manager – data scientist</td>\n",
       "      <td>piramal swasthya</td>\n",
       "      <td>[hyderabad]</td>\n",
       "      <td>7-10 yrs</td>\n",
       "      <td>[business strategy, predictive modeling, a/b t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>data scientist / machine learning engineer</td>\n",
       "      <td>infiswift technologies</td>\n",
       "      <td>[pune]</td>\n",
       "      <td>3-7 yrs</td>\n",
       "      <td>[computer science, data analysis, image proces...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>gig opportunity - internal audit: data scienti...</td>\n",
       "      <td>axis bank limited</td>\n",
       "      <td>[chennai,  pune,  delhi ncr,  mumbai,  ahmedab...</td>\n",
       "      <td>5-10 yrs</td>\n",
       "      <td>[data science, r, report generation, risk asse...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>data scientist/ data analyst</td>\n",
       "      <td>cars24</td>\n",
       "      <td>[gurgaon gurugram]</td>\n",
       "      <td>1-5 yrs</td>\n",
       "      <td>[data science, predictive modeling, data scien...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>data scientist - image recognition/machine lea...</td>\n",
       "      <td>carbynetech india</td>\n",
       "      <td>[hyderabad]</td>\n",
       "      <td>5-10 yrs</td>\n",
       "      <td>[it skills, java, python, machine learning, ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>data scientist - image processing/ machine lea...</td>\n",
       "      <td>carbynetech india</td>\n",
       "      <td>[hyderabad]</td>\n",
       "      <td>6-11 yrs</td>\n",
       "      <td>[predictive modeling, data science, data scien...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>data scientist - image processing/machine lear...</td>\n",
       "      <td>carbynetech india</td>\n",
       "      <td>[hyderabad]</td>\n",
       "      <td>6-11 yrs</td>\n",
       "      <td>[predictive modeling, data science, data scien...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                roles  \\\n",
       "15                 urgent opening for data scientists   \n",
       "16  data scientist - machine/ deep learning algori...   \n",
       "17                                     data scientist   \n",
       "18                    senior manager – data scientist   \n",
       "19         data scientist / machine learning engineer   \n",
       "20  gig opportunity - internal audit: data scienti...   \n",
       "21                       data scientist/ data analyst   \n",
       "22  data scientist - image recognition/machine lea...   \n",
       "23  data scientist - image processing/ machine lea...   \n",
       "24  data scientist - image processing/machine lear...   \n",
       "\n",
       "                                   companies  \\\n",
       "15                       sitel india pvt ltd   \n",
       "16                         carbynetech india   \n",
       "17  ganit business solutions private limited   \n",
       "18                          piramal swasthya   \n",
       "19                    infiswift technologies   \n",
       "20                         axis bank limited   \n",
       "21                                    cars24   \n",
       "22                         carbynetech india   \n",
       "23                         carbynetech india   \n",
       "24                         carbynetech india   \n",
       "\n",
       "                                            locations experience  \\\n",
       "15                               [mumbai,  hyderabad]   8-11 yrs   \n",
       "16                                        [hyderabad]    3-8 yrs   \n",
       "17  [chennai,  delhi ncr,  mumbai,  bengaluru,  hy...    2-5 yrs   \n",
       "18                                        [hyderabad]   7-10 yrs   \n",
       "19                                             [pune]    3-7 yrs   \n",
       "20  [chennai,  pune,  delhi ncr,  mumbai,  ahmedab...   5-10 yrs   \n",
       "21                                 [gurgaon gurugram]    1-5 yrs   \n",
       "22                                        [hyderabad]   5-10 yrs   \n",
       "23                                        [hyderabad]   6-11 yrs   \n",
       "24                                        [hyderabad]   6-11 yrs   \n",
       "\n",
       "                                               skills  \n",
       "15  [pytorch, data science, supervised learning, n...  \n",
       "16  [it skills, python, machine learning, data sci...  \n",
       "17  [python, data analytics, eda, sas, power bi, t...  \n",
       "18  [business strategy, predictive modeling, a/b t...  \n",
       "19  [computer science, data analysis, image proces...  \n",
       "20  [data science, r, report generation, risk asse...  \n",
       "21  [data science, predictive modeling, data scien...  \n",
       "22  [it skills, java, python, machine learning, ar...  \n",
       "23  [predictive modeling, data science, data scien...  \n",
       "24  [predictive modeling, data science, data scien...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# there are more than one loaction and skills are attached to each job, let split the location and skill\n",
    "df.skills=[skill.split(\"\\n\") for skill in df.skills]\n",
    "df.locations=[location.split(\",\") for location in df.locations]\n",
    "df[15:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 40 entries, 0 to 39\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   roles       40 non-null     object\n",
      " 1   companies   40 non-null     object\n",
      " 2   locations   40 non-null     object\n",
      " 3   experience  40 non-null     object\n",
      " 4   skills      40 non-null     object\n",
      "dtypes: object(5)\n",
      "memory usage: 1.7+ KB\n"
     ]
    }
   ],
   "source": [
    "# information about columns in dataframe\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "roles         0\n",
       "companies     0\n",
       "locations     0\n",
       "experience    0\n",
       "skills        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if any data have null values\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the missing value\n",
    "df=df.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving:-"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('naukri.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}