1_data_wrangling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Data Wrangling Lab**\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Objectives\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-   Identify duplicate values in the dataset.\n",
    "\n",
    "-   Remove duplicate values from the dataset.\n",
    "\n",
    "-   Identify missing values in the dataset.\n",
    "\n",
    "-   Impute the missing values in the dataset.\n",
    "\n",
    "-   Normalize data in the dataset.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hands on\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import pandas module.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the dataset into a dataframe.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Respondent</th>\n",
       "      <th>MainBranch</th>\n",
       "      <th>Hobbyist</th>\n",
       "      <th>OpenSourcer</th>\n",
       "      <th>OpenSource</th>\n",
       "      <th>Employment</th>\n",
       "      <th>Country</th>\n",
       "      <th>Student</th>\n",
       "      <th>EdLevel</th>\n",
       "      <th>UndergradMajor</th>\n",
       "      <th>...</th>\n",
       "      <th>WelcomeChange</th>\n",
       "      <th>SONewContent</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Trans</th>\n",
       "      <th>Sexuality</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>SurveyLength</th>\n",
       "      <th>SurveyEase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>No</td>\n",
       "      <td>Never</td>\n",
       "      <td>The quality of OSS and closed source software ...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>United States</td>\n",
       "      <td>No</td>\n",
       "      <td>Bachelor’s degree (BA, BS, B.Eng., etc.)</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>...</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>22.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>White or of European descent</td>\n",
       "      <td>No</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Easy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Once a month or more often</td>\n",
       "      <td>The quality of OSS and closed source software ...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>No</td>\n",
       "      <td>Some college/university study without earning ...</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>...</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Bisexual</td>\n",
       "      <td>White or of European descent</td>\n",
       "      <td>No</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once a month but more than once per ...</td>\n",
       "      <td>OSS is, on average, of HIGHER quality than pro...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>United States</td>\n",
       "      <td>No</td>\n",
       "      <td>Master’s degree (MA, MS, M.Eng., MBA, etc.)</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>...</td>\n",
       "      <td>Somewhat more welcome now than last year</td>\n",
       "      <td>Tech articles written by other developers;Cour...</td>\n",
       "      <td>28.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>White or of European descent</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Easy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Never</td>\n",
       "      <td>The quality of OSS and closed source software ...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>United Kingdom</td>\n",
       "      <td>No</td>\n",
       "      <td>Master’s degree (MA, MS, M.Eng., MBA, etc.)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>26.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>White or of European descent</td>\n",
       "      <td>No</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Neither easy nor difficult</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Less than once a month but more than once per ...</td>\n",
       "      <td>The quality of OSS and closed source software ...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>Australia</td>\n",
       "      <td>No</td>\n",
       "      <td>Bachelor’s degree (BA, BS, B.Eng., etc.)</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>...</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>Hispanic or Latino/Latina;Multiracial</td>\n",
       "      <td>No</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Easy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 85 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Respondent                      MainBranch Hobbyist  \\\n",
       "0           4  I am a developer by profession       No   \n",
       "1           9  I am a developer by profession      Yes   \n",
       "2          13  I am a developer by profession      Yes   \n",
       "3          16  I am a developer by profession      Yes   \n",
       "4          17  I am a developer by profession      Yes   \n",
       "\n",
       "                                         OpenSourcer  \\\n",
       "0                                              Never   \n",
       "1                         Once a month or more often   \n",
       "2  Less than once a month but more than once per ...   \n",
       "3                                              Never   \n",
       "4  Less than once a month but more than once per ...   \n",
       "\n",
       "                                          OpenSource          Employment  \\\n",
       "0  The quality of OSS and closed source software ...  Employed full-time   \n",
       "1  The quality of OSS and closed source software ...  Employed full-time   \n",
       "2  OSS is, on average, of HIGHER quality than pro...  Employed full-time   \n",
       "3  The quality of OSS and closed source software ...  Employed full-time   \n",
       "4  The quality of OSS and closed source software ...  Employed full-time   \n",
       "\n",
       "          Country Student                                            EdLevel  \\\n",
       "0   United States      No           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
       "1     New Zealand      No  Some college/university study without earning ...   \n",
       "2   United States      No        Master’s degree (MA, MS, M.Eng., MBA, etc.)   \n",
       "3  United Kingdom      No        Master’s degree (MA, MS, M.Eng., MBA, etc.)   \n",
       "4       Australia      No           Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
       "\n",
       "                                      UndergradMajor  ...  \\\n",
       "0  Computer science, computer engineering, or sof...  ...   \n",
       "1  Computer science, computer engineering, or sof...  ...   \n",
       "2  Computer science, computer engineering, or sof...  ...   \n",
       "3                                                NaN  ...   \n",
       "4  Computer science, computer engineering, or sof...  ...   \n",
       "\n",
       "                              WelcomeChange  \\\n",
       "0   Just as welcome now as I felt last year   \n",
       "1   Just as welcome now as I felt last year   \n",
       "2  Somewhat more welcome now than last year   \n",
       "3   Just as welcome now as I felt last year   \n",
       "4   Just as welcome now as I felt last year   \n",
       "\n",
       "                                        SONewContent   Age Gender Trans  \\\n",
       "0  Tech articles written by other developers;Indu...  22.0    Man    No   \n",
       "1                                                NaN  23.0    Man    No   \n",
       "2  Tech articles written by other developers;Cour...  28.0    Man    No   \n",
       "3  Tech articles written by other developers;Indu...  26.0    Man    No   \n",
       "4  Tech articles written by other developers;Indu...  29.0    Man    No   \n",
       "\n",
       "                 Sexuality                              Ethnicity Dependents  \\\n",
       "0  Straight / Heterosexual           White or of European descent         No   \n",
       "1                 Bisexual           White or of European descent         No   \n",
       "2  Straight / Heterosexual           White or of European descent        Yes   \n",
       "3  Straight / Heterosexual           White or of European descent         No   \n",
       "4  Straight / Heterosexual  Hispanic or Latino/Latina;Multiracial         No   \n",
       "\n",
       "            SurveyLength                  SurveyEase  \n",
       "0  Appropriate in length                        Easy  \n",
       "1  Appropriate in length  Neither easy nor difficult  \n",
       "2  Appropriate in length                        Easy  \n",
       "3  Appropriate in length  Neither easy nor difficult  \n",
       "4  Appropriate in length                        Easy  \n",
       "\n",
       "[5 rows x 85 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"survey_data.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11552, 85)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding duplicates\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Find how many duplicate rows exist in the dataframe.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "154"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.duplicated(subset=None, keep='first')].shape[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "154"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.duplicated(subset='Respondent', keep='first')].shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing duplicates\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove the duplicate rows from the dataframe.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11398"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean = df.drop_duplicates(subset=None, keep='first')\n",
    "df_clean.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Verify if duplicates were actually dropped.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 85)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean[df_clean.duplicated(subset=None, keep='first')].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding Missing values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the missing values for all columns.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30967"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Country    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean[['Country']].isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find out how many rows are missing in the column 'WorkLoc'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean['WorkLoc'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WorkLoc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>866</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1753</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     WorkLoc\n",
       "130      NaN\n",
       "242      NaN\n",
       "866      NaN\n",
       "1455     NaN\n",
       "1753     NaN"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean.loc[df_clean['WorkLoc'].isnull(),['WorkLoc']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imputing missing values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the  value counts for the column WorkLoc.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WorkLoc\n",
       "Office                                            6806\n",
       "Home                                              3589\n",
       "Other place, such as a coworking space or cafe     971\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean['WorkLoc'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1.058200e+04\n",
       "mean     1.315967e+05\n",
       "std      2.947865e+05\n",
       "min      0.000000e+00\n",
       "25%      2.686800e+04\n",
       "50%      5.774500e+04\n",
       "75%      1.000000e+05\n",
       "max      2.000000e+06\n",
       "Name: ConvertedComp, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean['ConvertedComp'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "UndergradMajor\n",
       "Computer science, computer engineering, or software engineering          6953\n",
       "Information systems, information technology, or system administration     794\n",
       "Another engineering discipline (ex. civil, electrical, mechanical)        759\n",
       "Web development or web design                                             410\n",
       "A natural science (ex. biology, chemistry, physics)                       403\n",
       "Mathematics or statistics                                                 372\n",
       "A business discipline (ex. accounting, finance, marketing)                244\n",
       "A social science (ex. anthropology, psychology, political science)        210\n",
       "A humanities discipline (ex. literature, history, philosophy)             207\n",
       "Fine arts or performing arts (ex. graphic design, music, studio art)      161\n",
       "I never declared a major                                                  124\n",
       "A health science (ex. nursing, pharmacy, radiology)                        24\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean['UndergradMajor'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Identify the value that is most frequent (majority) in the WorkLoc column.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Respondent</th>\n",
       "      <th>MainBranch</th>\n",
       "      <th>Hobbyist</th>\n",
       "      <th>OpenSourcer</th>\n",
       "      <th>OpenSource</th>\n",
       "      <th>Employment</th>\n",
       "      <th>Country</th>\n",
       "      <th>Student</th>\n",
       "      <th>EdLevel</th>\n",
       "      <th>UndergradMajor</th>\n",
       "      <th>...</th>\n",
       "      <th>WelcomeChange</th>\n",
       "      <th>SONewContent</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Trans</th>\n",
       "      <th>Sexuality</th>\n",
       "      <th>Ethnicity</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>SurveyLength</th>\n",
       "      <th>SurveyEase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>11398.000000</td>\n",
       "      <td>11398</td>\n",
       "      <td>11398</td>\n",
       "      <td>11398</td>\n",
       "      <td>11317</td>\n",
       "      <td>11398</td>\n",
       "      <td>11398</td>\n",
       "      <td>11347</td>\n",
       "      <td>11286</td>\n",
       "      <td>10661</td>\n",
       "      <td>...</td>\n",
       "      <td>11313</td>\n",
       "      <td>9433</td>\n",
       "      <td>11111.000000</td>\n",
       "      <td>11325</td>\n",
       "      <td>11275</td>\n",
       "      <td>10856</td>\n",
       "      <td>10723</td>\n",
       "      <td>11258</td>\n",
       "      <td>11379</td>\n",
       "      <td>11384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>135</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>89</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>I am a developer by profession</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Never</td>\n",
       "      <td>The quality of OSS and closed source software ...</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>United States</td>\n",
       "      <td>No</td>\n",
       "      <td>Bachelor’s degree (BA, BS, B.Eng., etc.)</td>\n",
       "      <td>Computer science, computer engineering, or sof...</td>\n",
       "      <td>...</td>\n",
       "      <td>Just as welcome now as I felt last year</td>\n",
       "      <td>Tech articles written by other developers;Indu...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Man</td>\n",
       "      <td>No</td>\n",
       "      <td>Straight / Heterosexual</td>\n",
       "      <td>White or of European descent</td>\n",
       "      <td>No</td>\n",
       "      <td>Appropriate in length</td>\n",
       "      <td>Easy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>10618</td>\n",
       "      <td>9273</td>\n",
       "      <td>3652</td>\n",
       "      <td>5468</td>\n",
       "      <td>10968</td>\n",
       "      <td>3127</td>\n",
       "      <td>9658</td>\n",
       "      <td>5852</td>\n",
       "      <td>6953</td>\n",
       "      <td>...</td>\n",
       "      <td>8704</td>\n",
       "      <td>2099</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10480</td>\n",
       "      <td>11196</td>\n",
       "      <td>10066</td>\n",
       "      <td>7357</td>\n",
       "      <td>6699</td>\n",
       "      <td>8048</td>\n",
       "      <td>8245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>12490.392437</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.778895</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7235.461999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.393686</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>6264.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>12484.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>18784.750000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>25142.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11 rows × 85 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Respondent                      MainBranch Hobbyist OpenSourcer  \\\n",
       "count   11398.000000                           11398    11398       11398   \n",
       "unique           NaN                               2        2           4   \n",
       "top              NaN  I am a developer by profession      Yes       Never   \n",
       "freq             NaN                           10618     9273        3652   \n",
       "mean    12490.392437                             NaN      NaN         NaN   \n",
       "std      7235.461999                             NaN      NaN         NaN   \n",
       "min         4.000000                             NaN      NaN         NaN   \n",
       "25%      6264.250000                             NaN      NaN         NaN   \n",
       "50%     12484.000000                             NaN      NaN         NaN   \n",
       "75%     18784.750000                             NaN      NaN         NaN   \n",
       "max     25142.000000                             NaN      NaN         NaN   \n",
       "\n",
       "                                               OpenSource          Employment  \\\n",
       "count                                               11317               11398   \n",
       "unique                                                  3                   2   \n",
       "top     The quality of OSS and closed source software ...  Employed full-time   \n",
       "freq                                                 5468               10968   \n",
       "mean                                                  NaN                 NaN   \n",
       "std                                                   NaN                 NaN   \n",
       "min                                                   NaN                 NaN   \n",
       "25%                                                   NaN                 NaN   \n",
       "50%                                                   NaN                 NaN   \n",
       "75%                                                   NaN                 NaN   \n",
       "max                                                   NaN                 NaN   \n",
       "\n",
       "              Country Student                                   EdLevel  \\\n",
       "count           11398   11347                                     11286   \n",
       "unique            135       3                                         9   \n",
       "top     United States      No  Bachelor’s degree (BA, BS, B.Eng., etc.)   \n",
       "freq             3127    9658                                      5852   \n",
       "mean              NaN     NaN                                       NaN   \n",
       "std               NaN     NaN                                       NaN   \n",
       "min               NaN     NaN                                       NaN   \n",
       "25%               NaN     NaN                                       NaN   \n",
       "50%               NaN     NaN                                       NaN   \n",
       "75%               NaN     NaN                                       NaN   \n",
       "max               NaN     NaN                                       NaN   \n",
       "\n",
       "                                           UndergradMajor  ...  \\\n",
       "count                                               10661  ...   \n",
       "unique                                                 12  ...   \n",
       "top     Computer science, computer engineering, or sof...  ...   \n",
       "freq                                                 6953  ...   \n",
       "mean                                                  NaN  ...   \n",
       "std                                                   NaN  ...   \n",
       "min                                                   NaN  ...   \n",
       "25%                                                   NaN  ...   \n",
       "50%                                                   NaN  ...   \n",
       "75%                                                   NaN  ...   \n",
       "max                                                   NaN  ...   \n",
       "\n",
       "                                  WelcomeChange  \\\n",
       "count                                     11313   \n",
       "unique                                        6   \n",
       "top     Just as welcome now as I felt last year   \n",
       "freq                                       8704   \n",
       "mean                                        NaN   \n",
       "std                                         NaN   \n",
       "min                                         NaN   \n",
       "25%                                         NaN   \n",
       "50%                                         NaN   \n",
       "75%                                         NaN   \n",
       "max                                         NaN   \n",
       "\n",
       "                                             SONewContent           Age  \\\n",
       "count                                                9433  11111.000000   \n",
       "unique                                                 15           NaN   \n",
       "top     Tech articles written by other developers;Indu...           NaN   \n",
       "freq                                                 2099           NaN   \n",
       "mean                                                  NaN     30.778895   \n",
       "std                                                   NaN      7.393686   \n",
       "min                                                   NaN     16.000000   \n",
       "25%                                                   NaN     25.000000   \n",
       "50%                                                   NaN     29.000000   \n",
       "75%                                                   NaN     35.000000   \n",
       "max                                                   NaN     99.000000   \n",
       "\n",
       "       Gender  Trans                Sexuality                     Ethnicity  \\\n",
       "count   11325  11275                    10856                         10723   \n",
       "unique      7      2                        7                            89   \n",
       "top       Man     No  Straight / Heterosexual  White or of European descent   \n",
       "freq    10480  11196                    10066                          7357   \n",
       "mean      NaN    NaN                      NaN                           NaN   \n",
       "std       NaN    NaN                      NaN                           NaN   \n",
       "min       NaN    NaN                      NaN                           NaN   \n",
       "25%       NaN    NaN                      NaN                           NaN   \n",
       "50%       NaN    NaN                      NaN                           NaN   \n",
       "75%       NaN    NaN                      NaN                           NaN   \n",
       "max       NaN    NaN                      NaN                           NaN   \n",
       "\n",
       "       Dependents           SurveyLength SurveyEase  \n",
       "count       11258                  11379      11384  \n",
       "unique          2                      3          3  \n",
       "top            No  Appropriate in length       Easy  \n",
       "freq         6699                   8048       8245  \n",
       "mean          NaN                    NaN        NaN  \n",
       "std           NaN                    NaN        NaN  \n",
       "min           NaN                    NaN        NaN  \n",
       "25%           NaN                    NaN        NaN  \n",
       "50%           NaN                    NaN        NaN  \n",
       "75%           NaN                    NaN        NaN  \n",
       "max           NaN                    NaN        NaN  \n",
       "\n",
       "[11 rows x 85 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Note of the WorkLoc majority value here, for future reference \n",
    "# 6806\n",
    "df_clean.describe(include='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Impute (replace) all the empty rows in the column WorkLoc with the value identified as majority.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                                  Home\n",
       "1                                                Office\n",
       "2                                                  Home\n",
       "3                                                  Home\n",
       "4        Other place, such as a coworking space or cafe\n",
       "                              ...                      \n",
       "11547                                              Home\n",
       "11548                                              Home\n",
       "11549                                            Office\n",
       "11550                                              Home\n",
       "11551                                            Office\n",
       "Name: WorkLoc, Length: 11398, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean['WorkLoc'].replace( to_replace = np.nan, value = 'Office')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After imputation there should ideally not be any empty rows in the WorkLoc column.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Verify if imputing was successful.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WorkLoc\n",
       "Office                                            6806\n",
       "Home                                              3589\n",
       "Other place, such as a coworking space or cafe     971\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean['WorkLoc'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalizing data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are two columns in the dataset that talk about compensation.\n",
    "\n",
    "One is \"CompFreq\". This column shows how often a developer is paid (Yearly, Monthly, Weekly).\n",
    "\n",
    "The other is \"CompTotal\". This column talks about how much the developer is paid per Year, Month, or Week depending upon his/her \"CompFreq\". \n",
    "\n",
    "This makes it difficult to compare the total compensation of the developers.\n",
    "\n",
    "In this section, a new column called 'NormalizedAnnualCompensation' which contains the 'Annual Compensation' irrespective of the 'CompFreq' is created.\n",
    "\n",
    "Once this column is ready, it makes comparison of salaries easy.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List out the various categories in the column 'CompFreq'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CompFreq</th>\n",
       "      <th>CompTotal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>61000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>138000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>90000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>29000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>90000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>9500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>3000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>103000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>69000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>8000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  CompFreq  CompTotal\n",
       "0   Yearly    61000.0\n",
       "1   Yearly   138000.0\n",
       "2   Yearly    90000.0\n",
       "3  Monthly    29000.0\n",
       "4   Yearly    90000.0\n",
       "5  Monthly     9500.0\n",
       "6  Monthly     3000.0\n",
       "7   Yearly   103000.0\n",
       "8   Yearly    69000.0\n",
       "9  Monthly     8000.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean[['CompFreq','CompTotal']].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CompFreq      object\n",
       "CompTotal    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean[['CompFreq','CompTotal']].dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "New column named 'NormalizedAnnualCompensation' created\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_run_diff( freq , P ):\n",
    "    if freq == 'Yearly':\n",
    "        return P\n",
    "    elif freq == 'Monthly':\n",
    "        return P*12\n",
    "    else :\n",
    "        return P*52"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\alexander\\AppData\\Local\\Temp\\ipykernel_11968\\3048704717.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_clean['NormalizedAnnualCompensation'] = df_clean.apply(\n"
     ]
    }
   ],
   "source": [
    "df_clean['NormalizedAnnualCompensation'] = df_clean.apply(\n",
    "         lambda row: calc_run_diff(row['CompFreq'], row['CompTotal']),\n",
    "         axis=1)\n",
    "#baseball_df['RD'] = run_diffs_apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CompFreq</th>\n",
       "      <th>CompTotal</th>\n",
       "      <th>NormalizedAnnualCompensation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>61000.0</td>\n",
       "      <td>61000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>138000.0</td>\n",
       "      <td>138000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>90000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>29000.0</td>\n",
       "      <td>348000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>90000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>9500.0</td>\n",
       "      <td>114000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>36000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>103000.0</td>\n",
       "      <td>103000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>69000.0</td>\n",
       "      <td>69000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>8000.0</td>\n",
       "      <td>96000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>7000.0</td>\n",
       "      <td>84000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Yearly</td>\n",
       "      <td>114000.0</td>\n",
       "      <td>114000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Weekly</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>104000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Weekly</td>\n",
       "      <td>22000.0</td>\n",
       "      <td>1144000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Monthly</td>\n",
       "      <td>96000.0</td>\n",
       "      <td>1152000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CompFreq  CompTotal  NormalizedAnnualCompensation\n",
       "0    Yearly    61000.0                       61000.0\n",
       "1    Yearly   138000.0                      138000.0\n",
       "2    Yearly    90000.0                       90000.0\n",
       "3   Monthly    29000.0                      348000.0\n",
       "4    Yearly    90000.0                       90000.0\n",
       "5   Monthly     9500.0                      114000.0\n",
       "6   Monthly     3000.0                       36000.0\n",
       "7    Yearly   103000.0                      103000.0\n",
       "8    Yearly    69000.0                       69000.0\n",
       "9   Monthly     8000.0                       96000.0\n",
       "10  Monthly     7000.0                       84000.0\n",
       "11   Yearly   114000.0                      114000.0\n",
       "12   Weekly     2000.0                      104000.0\n",
       "13   Weekly    22000.0                     1144000.0\n",
       "14  Monthly    96000.0                     1152000.0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean[['CompFreq','CompTotal','NormalizedAnnualCompensation']].head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CompFreq\n",
       "Yearly     6073\n",
       "Monthly    4788\n",
       "Weekly      331\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean['CompFreq'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CompTotal</th>\n",
       "      <th>NormalizedAnnualCompensation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.058900e+04</td>\n",
       "      <td>1.058900e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.570477e+05</td>\n",
       "      <td>6.170771e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9.705598e+06</td>\n",
       "      <td>9.842866e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.500000e+04</td>\n",
       "      <td>5.200000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.500000e+04</td>\n",
       "      <td>1.000000e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.200000e+05</td>\n",
       "      <td>3.600000e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.000000e+08</td>\n",
       "      <td>8.400000e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          CompTotal  NormalizedAnnualCompensation\n",
       "count  1.058900e+04                  1.058900e+04\n",
       "mean   7.570477e+05                  6.170771e+06\n",
       "std    9.705598e+06                  9.842866e+07\n",
       "min    0.000000e+00                  0.000000e+00\n",
       "25%    2.500000e+04                  5.200000e+04\n",
       "50%    6.500000e+04                  1.000000e+05\n",
       "75%    1.200000e+05                  3.600000e+05\n",
       "max    7.000000e+08                  8.400000e+09"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean[['CompFreq','CompTotal','NormalizedAnnualCompensation']].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean.to_csv('survey_data_clean.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}