lessons/CRISP_DM/Removing Data Part II.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Removing Data Part II\n",
    "\n",
    "So, you now have seen how we can fit a model by dropping rows with missing values.  This is great in that sklearn doesn't break! However, this means future observations will not obtain a prediction if they have missing values in any of the columns.\n",
    "\n",
    "In this notebook, you will answer a few questions about what happened in the last screencast, and take a few additional steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Salary</th>\n",
       "      <th>CareerSatisfaction</th>\n",
       "      <th>HoursPerWeek</th>\n",
       "      <th>JobSatisfaction</th>\n",
       "      <th>StackOverflowSatisfaction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>113750.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Salary  CareerSatisfaction  HoursPerWeek  JobSatisfaction  \\\n",
       "0       NaN                 NaN           0.0              NaN   \n",
       "1       NaN                 NaN           NaN              NaN   \n",
       "2  113750.0                 8.0           NaN              9.0   \n",
       "3       NaN                 6.0           5.0              3.0   \n",
       "4       NaN                 6.0           NaN              8.0   \n",
       "\n",
       "   StackOverflowSatisfaction  \n",
       "0                        9.0  \n",
       "1                        8.0  \n",
       "2                        8.0  \n",
       "3                       10.0  \n",
       "4                        NaN  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 505 ms (started: 2021-08-10 13:43:39 -07:00)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import r2_score, mean_squared_error\n",
    "import seaborn as sns\n",
    "import RemovingData as t\n",
    "%matplotlib inline\n",
    "\n",
    "# Load autotime\n",
    "%load_ext autotime\n",
    "\n",
    "df = pd.read_csv('./survey_results_public.csv')\n",
    "\n",
    "#Subset to only quantitative vars\n",
    "num_vars = df[['Salary', 'CareerSatisfaction', 'HoursPerWeek', 'JobSatisfaction', 'StackOverflowSatisfaction']]\n",
    "\n",
    "\n",
    "num_vars.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Question 1\n",
    "\n",
    "**1.** What proportion of individuals in the dataset reported a salary?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2622238509056643"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2.5 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "prop_sals = num_vars.Salary.notna().mean() # Proportion of individuals in the dataset with salary reported\n",
    "\n",
    "prop_sals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice job! That looks right!\n",
      "time: 862 µs (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "t.prop_sals_test(prop_sals) #test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Question 2\n",
    "\n",
    "**2.** Remove the rows associated with nan values in Salary (only Salary) from the dataframe **num_vars**.  Store the dataframe with these rows removed in **sal_rem**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Salary</th>\n",
       "      <th>CareerSatisfaction</th>\n",
       "      <th>HoursPerWeek</th>\n",
       "      <th>JobSatisfaction</th>\n",
       "      <th>StackOverflowSatisfaction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>113750.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>100000.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>130000.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>82500.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>100764.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Salary  CareerSatisfaction  HoursPerWeek  JobSatisfaction  \\\n",
       "2   113750.0                 8.0           NaN              9.0   \n",
       "14  100000.0                 8.0           NaN              8.0   \n",
       "17  130000.0                 9.0           NaN              8.0   \n",
       "18   82500.0                 5.0           NaN              3.0   \n",
       "22  100764.0                 8.0           NaN              9.0   \n",
       "\n",
       "    StackOverflowSatisfaction  \n",
       "2                         8.0  \n",
       "14                        8.0  \n",
       "17                        8.0  \n",
       "18                        NaN  \n",
       "22                        8.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 15.9 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "sal_rm = num_vars.dropna(axis=0, how='any', subset=['Salary']) # dataframe with rows for nan Salaries removed\n",
    "\n",
    "sal_rm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice job! That looks right!\n",
      "time: 1.7 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "t.sal_rm_test(sal_rm) #test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Question 3\n",
    "\n",
    "**3.** Using **sal_rm**, create **X** be a dataframe (matrix) of all of the numeric feature variables.  Then, let **y** be the response vector you would like to predict (Salary).  Run the cell below once you have split the data, and use the result of the code to assign the correct letter to **question3_solution**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oh no! It doesn't work!!!\n",
      "time: 6.58 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "features = list(sal_rm.columns)\n",
    "features.remove('Salary')\n",
    "X = sal_rm[features] # Create X using explanatory variables from sal_rm\n",
    "y = sal_rm['Salary'] # Create y using the response variable of Salary\n",
    "\n",
    "# Split data into training and test data, and fit a linear model\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y , test_size=.30, random_state=42)\n",
    "lm_model = LinearRegression(normalize=True)\n",
    "\n",
    "# If our model works, it should just fit our model to the data. Otherwise, it will let us know.\n",
    "try:\n",
    "    lm_model.fit(X_train, y_train)\n",
    "except:\n",
    "    print(\"Oh no! It doesn't work!!!\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice job! That's right! Those missing values in the X matrix will still not allow us to predict the response.\n",
      "time: 913 µs (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "a = 'Python just likes to break sometimes for no reason at all.' \n",
    "b = 'It worked, because Python is magic.'\n",
    "c = 'It broke because we still have missing values in X'\n",
    "\n",
    "question3_solution = c\n",
    "\n",
    "#test\n",
    "t.question3_check(question3_solution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Question 4\n",
    "\n",
    "**4.** Remove the rows associated with nan values in any column from **num_vars** (this was the removal process used in the screencast).  Store the dataframe with these rows removed in **all_rem**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Salary</th>\n",
       "      <th>CareerSatisfaction</th>\n",
       "      <th>HoursPerWeek</th>\n",
       "      <th>JobSatisfaction</th>\n",
       "      <th>StackOverflowSatisfaction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>175000.000000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>14838.709677</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>15674.203822</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>43010.752688</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>65000.000000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Salary  CareerSatisfaction  HoursPerWeek  JobSatisfaction  \\\n",
       "25  175000.000000                 7.0           0.0              7.0   \n",
       "34   14838.709677                10.0           1.0              8.0   \n",
       "52   15674.203822                 6.0           4.0              5.0   \n",
       "57   43010.752688                10.0           2.0              6.0   \n",
       "70   65000.000000                 8.0           2.0              5.0   \n",
       "\n",
       "    StackOverflowSatisfaction  \n",
       "25                        9.0  \n",
       "34                       10.0  \n",
       "52                        8.0  \n",
       "57                       10.0  \n",
       "70                        7.0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 17 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "all_rm = num_vars.dropna(axis=0) # dataframe with rows for nan Salaries removed\n",
    "\n",
    "all_rm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice job! That looks right.  The default is to drop any row with a missing value in any column, so we didn't need to specify any arguments in this case.\n",
      "time: 752 µs (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "t.all_rm_test(all_rm) #test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Question 5\n",
    "\n",
    "**5.** Using **all_rm**, create **X_2** be a dataframe (matrix) of all of the numeric feature variables.  Then, let **y_2** be the response vector you would like to predict (Salary).  Run the cell below once you have split the data, and use the result of the code to assign the correct letter to **question5_solution**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 9.84 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "features = list(all_rm.columns)\n",
    "features.remove('Salary')\n",
    "X_2 = all_rm[features] # Create X using explanatory variables from all_rm\n",
    "y_2 = all_rm['Salary'] # Create y using Salary from sal_rm\n",
    "\n",
    "# Split data into training and test data, and fit a linear model\n",
    "X_2_train, X_2_test, y_2_train, y_2_test = train_test_split(X_2, y_2 , test_size=.30, random_state=42)\n",
    "lm_2_model = LinearRegression(normalize=True)\n",
    "\n",
    "# If our model works, it should just fit our model to the data. Otherwise, it will let us know.\n",
    "try:\n",
    "    lm_2_model.fit(X_2_train, y_2_train)\n",
    "except:\n",
    "    print(\"Oh no! It doesn't work!!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice job! That's right! Python isn't exactly magic, but sometimes it feels like it is!\n",
      "time: 1.43 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "a = 'Python just likes to break sometimes for no reason at all.' \n",
    "b = 'It worked, because Python is magic.'\n",
    "c = 'It broke because we still have missing values in X'\n",
    "\n",
    "question5_solution = b\n",
    "\n",
    "#test\n",
    "t.question5_check(question5_solution)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Question 6\n",
    "\n",
    "**6.** Now, use **lm_2_model** to predict the **y_2_test** response values, and obtain an r-squared value for how well the predicted values compare to the actual test values.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.019170661803761924"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 4.24 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "y_test_preds = lm_2_model.predict(X_2_test) # Predictions here using X_2 and lm_2_model\n",
    "r2_test = r2_score(y_2_test, y_test_preds) # Rsquared here for comparing test and preds from lm_2_model\n",
    "\n",
    "# Print r2 to see result\n",
    "r2_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice job! That's right! Your rsquared matches the solution.\n",
      "time: 342 µs (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "t.r2_test_check(r2_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Question 7\n",
    "\n",
    "**7.** Use what you have learned **from the second model you fit** (and as many cells as you need to find the answers) to complete the dictionary with the variables that link to the corresponding descriptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice job! That looks right to me!  We would really like to predict for anyone who provides a salary, but our model right now definitely has some limitations.\n",
      "time: 1.09 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "a = 5009\n",
    "b = 'Other'\n",
    "c = 645\n",
    "d = 'We still want to predict their salary'\n",
    "e = 'We do not care to predict their salary'\n",
    "f = False\n",
    "g = True\n",
    "\n",
    "question7_solution = {'The number of reported salaries in the original dataset': a,\n",
    "                       'The number of test salaries predicted using our model': c,\n",
    "                       'If an individual does not rate stackoverflow, but has a salary': d,\n",
    "                       'If an individual does not have a a job satisfaction, but has a salary': d,\n",
    "                       'Our model predicts salaries for the two individuals described above.': f}\n",
    "                      \n",
    "                      \n",
    "#Check your answers against the solution - you should be told you were right if your answers are correct!                     \n",
    "t.question7_check(question7_solution)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5009"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2.52 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "# The number of reported salaries in the original dataset\n",
    "num_vars.Salary.notna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "645"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.69 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "# The number of test salaries predicted using our model\n",
    "len(y_2_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8876033923149409"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 3.63 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "# Null proportion of the whole dataset\n",
    "1 - len(num_vars.dropna()) / len(num_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Salary                       0.737776\n",
       "CareerSatisfaction           0.157052\n",
       "HoursPerWeek                 0.595801\n",
       "JobSatisfaction              0.201497\n",
       "StackOverflowSatisfaction    0.295781\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 4.7 ms (started: 2021-08-10 13:43:40 -07:00)\n"
     ]
    }
   ],
   "source": [
    "# Null proportion for each column\n",
    "num_vars.isna().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}