Test of proportion and variance (1) (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import f\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('insurance.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>bmi</th>\n",
       "      <th>children</th>\n",
       "      <th>smoker</th>\n",
       "      <th>region</th>\n",
       "      <th>charges</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>female</td>\n",
       "      <td>27.900</td>\n",
       "      <td>0</td>\n",
       "      <td>yes</td>\n",
       "      <td>southwest</td>\n",
       "      <td>16884.92400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>male</td>\n",
       "      <td>33.770</td>\n",
       "      <td>1</td>\n",
       "      <td>no</td>\n",
       "      <td>southeast</td>\n",
       "      <td>1725.55230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>male</td>\n",
       "      <td>33.000</td>\n",
       "      <td>3</td>\n",
       "      <td>no</td>\n",
       "      <td>southeast</td>\n",
       "      <td>4449.46200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>33</td>\n",
       "      <td>male</td>\n",
       "      <td>22.705</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>northwest</td>\n",
       "      <td>21984.47061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32</td>\n",
       "      <td>male</td>\n",
       "      <td>28.880</td>\n",
       "      <td>0</td>\n",
       "      <td>no</td>\n",
       "      <td>northwest</td>\n",
       "      <td>3866.85520</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age     sex     bmi  children smoker     region      charges\n",
       "0   19  female  27.900         0    yes  southwest  16884.92400\n",
       "1   18    male  33.770         1     no  southeast   1725.55230\n",
       "2   28    male  33.000         3     no  southeast   4449.46200\n",
       "3   33    male  22.705         0     no  northwest  21984.47061\n",
       "4   32    male  28.880         0     no  northwest   3866.85520"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test of proportions "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 'sex' and 'smoker' are two categorical variables\n",
    "* We want to see if the proportion of smokers in the female population is significantly less than it is in the male population\n",
    "\n",
    "#### Ho = The proportions are equal\n",
    "#### Ha = The two proportions are not equal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "female_smokers = df[df['sex'] == 'female'].smoker.value_counts()[1]  # number of female smokers\n",
    "male_smokers = df[df['sex'] == 'male'].smoker.value_counts()[1] # number of male smokers\n",
    "n_females = df.sex.value_counts()[1] # number of females in the data\n",
    "n_males = df.sex.value_counts()[0] #number of males in the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[115, 159] [662, 676]\n",
      " Proportion of smokers in females, males = 0.17%, 0.24% respectively\n"
     ]
    }
   ],
   "source": [
    "print([female_smokers, male_smokers] , [n_females, n_males])\n",
    "print(f' Proportion of smokers in females, males = {round(115/662,2)}%, {round(159/676,2)}% respectively')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The proportions are different but are they statistically significant?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "With a p-value of 0.0053 the difference is significant. aka |We reject the null|\n"
     ]
    }
   ],
   "source": [
    "from statsmodels.stats.proportion import proportions_ztest\n",
    "\n",
    "stat, pval = proportions_ztest([female_smokers, male_smokers] , [n_females, n_males])\n",
    "\n",
    "if pval < 0.05:\n",
    "    print(f'With a p-value of {round(pval,4)} the difference is significant. aka |We reject the null|')\n",
    "else:\n",
    "    print(f'With a p-value of {round(pval,4)} the difference is not significant. aka |We fail to reject the null|')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test of Variance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For chi square table: https://people.smp.uq.edu.au/YoniNazarathy/stat_models_B_course_spring_07/distributions/chisqtab.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male      35\n",
       "female    33\n",
       "Name: sex, dtype: int64"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nineteen = df[df['age'] == 19]\n",
    "nineteen.sex.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_male = nineteen[nineteen['sex'] == 'male'].bmi.iloc[:-2]   #excluding the last two elements to match the size 2 samples\n",
    "sample_female = nineteen[nineteen['sex'] == 'female'].bmi\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30.765708585858583 35.05058650137741\n"
     ]
    }
   ],
   "source": [
    "v1, v2 = np.var(sample_female) , np.var(sample_male)\n",
    "print(v1,v2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Variances of bmi of men is higher than it is for women. But is the difference statistically significant?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ho : Variation in bmi of men and women is equal same\n",
    "#### Ha : Variation in bmi of men is greater than it is in women"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 33  # number of samples\n",
    "dof = n - 1  # degrees of freedom\n",
    "alpha = 0.05  # significance level\n",
    "chi_critical = 46.19    # critical chi_squared statistic. From the table "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Since the test statistic is less than the critical value, we fail to reject the null\n"
     ]
    }
   ],
   "source": [
    "chi = (dof*v1)/v2\n",
    "\n",
    "if chi < chi_critical:\n",
    "    print(\"Since the test statistic is less than the critical value, we fail to reject the null\")\n",
    "else:\n",
    "    print(\"Since the test statistic is more than the critical value, we reject the null\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}