13python-for-statisticians/python-for-statisticians.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sex</th>\n",
       "      <th>Length</th>\n",
       "      <th>Diameter</th>\n",
       "      <th>Height</th>\n",
       "      <th>Whole weight</th>\n",
       "      <th>Shucked weight</th>\n",
       "      <th>Viscera weight</th>\n",
       "      <th>Shell weight</th>\n",
       "      <th>Rings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M</td>\n",
       "      <td>0.455</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.095</td>\n",
       "      <td>0.5140</td>\n",
       "      <td>0.2245</td>\n",
       "      <td>0.1010</td>\n",
       "      <td>0.150</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M</td>\n",
       "      <td>0.350</td>\n",
       "      <td>0.265</td>\n",
       "      <td>0.090</td>\n",
       "      <td>0.2255</td>\n",
       "      <td>0.0995</td>\n",
       "      <td>0.0485</td>\n",
       "      <td>0.070</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>F</td>\n",
       "      <td>0.530</td>\n",
       "      <td>0.420</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.6770</td>\n",
       "      <td>0.2565</td>\n",
       "      <td>0.1415</td>\n",
       "      <td>0.210</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.5160</td>\n",
       "      <td>0.2155</td>\n",
       "      <td>0.1140</td>\n",
       "      <td>0.155</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I</td>\n",
       "      <td>0.330</td>\n",
       "      <td>0.255</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.0895</td>\n",
       "      <td>0.0395</td>\n",
       "      <td>0.055</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Sex  Length  Diameter  Height  Whole weight  Shucked weight  Viscera weight  \\\n",
       "0   M   0.455     0.365   0.095        0.5140          0.2245          0.1010   \n",
       "1   M   0.350     0.265   0.090        0.2255          0.0995          0.0485   \n",
       "2   F   0.530     0.420   0.135        0.6770          0.2565          0.1415   \n",
       "3   M   0.440     0.365   0.125        0.5160          0.2155          0.1140   \n",
       "4   I   0.330     0.255   0.080        0.2050          0.0895          0.0395   \n",
       "\n",
       "   Shell weight  Rings  \n",
       "0         0.150     15  \n",
       "1         0.070      7  \n",
       "2         0.210      9  \n",
       "3         0.155     10  \n",
       "4         0.055      7  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('abalone.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Описательная статистика "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Length</th>\n",
       "      <th>Diameter</th>\n",
       "      <th>Height</th>\n",
       "      <th>Whole weight</th>\n",
       "      <th>Shucked weight</th>\n",
       "      <th>Viscera weight</th>\n",
       "      <th>Shell weight</th>\n",
       "      <th>Rings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4177.000000</td>\n",
       "      <td>4177.000000</td>\n",
       "      <td>4177.000000</td>\n",
       "      <td>4177.000000</td>\n",
       "      <td>4177.000000</td>\n",
       "      <td>4177.000000</td>\n",
       "      <td>4177.000000</td>\n",
       "      <td>4177.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.523992</td>\n",
       "      <td>0.407881</td>\n",
       "      <td>0.139516</td>\n",
       "      <td>0.828742</td>\n",
       "      <td>0.359367</td>\n",
       "      <td>0.180594</td>\n",
       "      <td>0.238831</td>\n",
       "      <td>9.933684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.120093</td>\n",
       "      <td>0.099240</td>\n",
       "      <td>0.041827</td>\n",
       "      <td>0.490389</td>\n",
       "      <td>0.221963</td>\n",
       "      <td>0.109614</td>\n",
       "      <td>0.139203</td>\n",
       "      <td>3.224169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.075000</td>\n",
       "      <td>0.055000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002000</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>0.000500</td>\n",
       "      <td>0.001500</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.450000</td>\n",
       "      <td>0.350000</td>\n",
       "      <td>0.115000</td>\n",
       "      <td>0.441500</td>\n",
       "      <td>0.186000</td>\n",
       "      <td>0.093500</td>\n",
       "      <td>0.130000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.545000</td>\n",
       "      <td>0.425000</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.799500</td>\n",
       "      <td>0.336000</td>\n",
       "      <td>0.171000</td>\n",
       "      <td>0.234000</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.615000</td>\n",
       "      <td>0.480000</td>\n",
       "      <td>0.165000</td>\n",
       "      <td>1.153000</td>\n",
       "      <td>0.502000</td>\n",
       "      <td>0.253000</td>\n",
       "      <td>0.329000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.815000</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>1.130000</td>\n",
       "      <td>2.825500</td>\n",
       "      <td>1.488000</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>1.005000</td>\n",
       "      <td>29.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Length     Diameter       Height  Whole weight  Shucked weight  \\\n",
       "count  4177.000000  4177.000000  4177.000000   4177.000000     4177.000000   \n",
       "mean      0.523992     0.407881     0.139516      0.828742        0.359367   \n",
       "std       0.120093     0.099240     0.041827      0.490389        0.221963   \n",
       "min       0.075000     0.055000     0.000000      0.002000        0.001000   \n",
       "25%       0.450000     0.350000     0.115000      0.441500        0.186000   \n",
       "50%       0.545000     0.425000     0.140000      0.799500        0.336000   \n",
       "75%       0.615000     0.480000     0.165000      1.153000        0.502000   \n",
       "max       0.815000     0.650000     1.130000      2.825500        1.488000   \n",
       "\n",
       "       Viscera weight  Shell weight        Rings  \n",
       "count     4177.000000   4177.000000  4177.000000  \n",
       "mean         0.180594      0.238831     9.933684  \n",
       "std          0.109614      0.139203     3.224169  \n",
       "min          0.000500      0.001500     1.000000  \n",
       "25%          0.093500      0.130000     8.000000  \n",
       "50%          0.171000      0.234000     9.000000  \n",
       "75%          0.253000      0.329000    11.000000  \n",
       "max          0.760000      1.005000    29.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Проверка на нормальность"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Тест Шапиро-Уилк](https://en.wikipedia.org/wiki/Shapiro%E2%80%93Wilk_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics=0.931, p-value=0.000\n",
      "Отклонить гипотезу о нормальности\n"
     ]
    }
   ],
   "source": [
    "from scipy import stats\n",
    "\n",
    "stat, p = stats.shapiro(data['Rings'])\n",
    "print('Statistics=%.3f, p-value=%.3f' % (stat, p))\n",
    "\n",
    "alpha = 0.05\n",
    "if p > alpha:\n",
    "    print('Принять гипотезу о нормальности')\n",
    "else:\n",
    "    print('Отклонить гипотезу о нормальности')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## [Критерий согласия Пирсона ](https://ru.wikipedia.org/wiki/%D0%9A%D1%80%D0%B8%D1%82%D0%B5%D1%80%D0%B8%D0%B9_%D1%81%D0%BE%D0%B3%D0%BB%D0%B0%D1%81%D0%B8%D1%8F_%D0%9F%D0%B8%D1%80%D1%81%D0%BE%D0%BD%D0%B0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics=242.159, p-value=0.000\n",
      "Отклонить гипотезу о нормальности\n"
     ]
    }
   ],
   "source": [
    "stat, p = stats.normaltest(data['Length'])\n",
    "print('Statistics=%.3f, p-value=%.3f' % (stat, p))\n",
    "\n",
    "alpha = 0.05\n",
    "if p > alpha:\n",
    "    print('Принять гипотезу о нормальности')\n",
    "else:\n",
    "    print('Отклонить гипотезу о нормальности')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## t-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=1.5565212835974083, pvalue=0.11965998094160571)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Разделим на две выборки\n",
    "half = len(data['Length']) / 2\n",
    "sam1 = data.loc[:half, 'Length']\n",
    "sam2 = data.loc[half:, 'Length']\n",
    "\n",
    "stats.ttest_ind(sam2, sam1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.9605323551806582"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs = (half - 1) + (half - 1)\n",
    "scipy.stats.t.ppf(0.975, dfs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Линейная регрессия"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>          <td>Rings</td>      <th>  R-squared:         </th> <td>   0.330</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.330</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   2059.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Fri, 26 Jun 2020</td> <th>  Prob (F-statistic):</th>  <td>  0.00</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>16:34:53</td>     <th>  Log-Likelihood:    </th> <td> -9979.2</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  4177</td>      <th>  AIC:               </th> <td>1.996e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  4175</td>      <th>  BIC:               </th> <td>1.998e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     1</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>    2.3186</td> <td>    0.173</td> <td>   13.423</td> <td> 0.000</td> <td>    1.980</td> <td>    2.657</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Diameter</th>  <td>   18.6699</td> <td>    0.411</td> <td>   45.371</td> <td> 0.000</td> <td>   17.863</td> <td>   19.477</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>1414.851</td> <th>  Durbin-Watson:     </th> <td>   0.967</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th>  <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>4737.552</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>           <td> 1.714</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>       <td> 6.933</td>  <th>  Cond. No.          </th> <td>    11.8</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                  Rings   R-squared:                       0.330\n",
       "Model:                            OLS   Adj. R-squared:                  0.330\n",
       "Method:                 Least Squares   F-statistic:                     2059.\n",
       "Date:                Fri, 26 Jun 2020   Prob (F-statistic):               0.00\n",
       "Time:                        16:34:53   Log-Likelihood:                -9979.2\n",
       "No. Observations:                4177   AIC:                         1.996e+04\n",
       "Df Residuals:                    4175   BIC:                         1.998e+04\n",
       "Df Model:                           1                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "Intercept      2.3186      0.173     13.423      0.000       1.980       2.657\n",
       "Diameter      18.6699      0.411     45.371      0.000      17.863      19.477\n",
       "==============================================================================\n",
       "Omnibus:                     1414.851   Durbin-Watson:                   0.967\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             4737.552\n",
       "Skew:                           1.714   Prob(JB):                         0.00\n",
       "Kurtosis:                       6.933   Cond. No.                         11.8\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import statsmodels.formula.api as smf\n",
    "\n",
    "model = smf.ols('Rings ~ Diameter', data=data)\n",
    "res = model.fit()\n",
    "res.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       5.866905\n",
       "1      -0.266103\n",
       "2      -1.159940\n",
       "3       0.866905\n",
       "4      -0.079403\n",
       "          ...   \n",
       "4172    0.279962\n",
       "4173   -0.533339\n",
       "4174   -2.186786\n",
       "4175   -1.373485\n",
       "4176   -0.680380\n",
       "Length: 4177, dtype: float64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.resid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}