cancer_deathrate_prediction_OLS_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import linear_model\n",
    "from sklearn import ensemble\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>avgAnnCount</th>\n",
       "      <th>avgDeathsPerYear</th>\n",
       "      <th>TARGET_deathRate</th>\n",
       "      <th>incidenceRate</th>\n",
       "      <th>medIncome</th>\n",
       "      <th>popEst2015</th>\n",
       "      <th>povertyPercent</th>\n",
       "      <th>studyPerCap</th>\n",
       "      <th>binnedInc</th>\n",
       "      <th>MedianAge</th>\n",
       "      <th>...</th>\n",
       "      <th>PctPrivateCoverageAlone</th>\n",
       "      <th>PctEmpPrivCoverage</th>\n",
       "      <th>PctPublicCoverage</th>\n",
       "      <th>PctPublicCoverageAlone</th>\n",
       "      <th>PctWhite</th>\n",
       "      <th>PctBlack</th>\n",
       "      <th>PctAsian</th>\n",
       "      <th>PctOtherRace</th>\n",
       "      <th>PctMarriedHouseholds</th>\n",
       "      <th>BirthRate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1397.0</td>\n",
       "      <td>469</td>\n",
       "      <td>164.9</td>\n",
       "      <td>489.8</td>\n",
       "      <td>61898</td>\n",
       "      <td>260131</td>\n",
       "      <td>11.2</td>\n",
       "      <td>499.748204</td>\n",
       "      <td>(61494.5, 125635]</td>\n",
       "      <td>39.3</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>41.6</td>\n",
       "      <td>32.9</td>\n",
       "      <td>14.0</td>\n",
       "      <td>81.780529</td>\n",
       "      <td>2.594728</td>\n",
       "      <td>4.821857</td>\n",
       "      <td>1.843479</td>\n",
       "      <td>52.856076</td>\n",
       "      <td>6.118831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>173.0</td>\n",
       "      <td>70</td>\n",
       "      <td>161.3</td>\n",
       "      <td>411.6</td>\n",
       "      <td>48127</td>\n",
       "      <td>43269</td>\n",
       "      <td>18.6</td>\n",
       "      <td>23.111234</td>\n",
       "      <td>(48021.6, 51046.4]</td>\n",
       "      <td>33.0</td>\n",
       "      <td>...</td>\n",
       "      <td>53.8</td>\n",
       "      <td>43.6</td>\n",
       "      <td>31.1</td>\n",
       "      <td>15.3</td>\n",
       "      <td>89.228509</td>\n",
       "      <td>0.969102</td>\n",
       "      <td>2.246233</td>\n",
       "      <td>3.741352</td>\n",
       "      <td>45.372500</td>\n",
       "      <td>4.333096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>102.0</td>\n",
       "      <td>50</td>\n",
       "      <td>174.7</td>\n",
       "      <td>349.7</td>\n",
       "      <td>49348</td>\n",
       "      <td>21026</td>\n",
       "      <td>14.6</td>\n",
       "      <td>47.560164</td>\n",
       "      <td>(48021.6, 51046.4]</td>\n",
       "      <td>45.0</td>\n",
       "      <td>...</td>\n",
       "      <td>43.5</td>\n",
       "      <td>34.9</td>\n",
       "      <td>42.1</td>\n",
       "      <td>21.1</td>\n",
       "      <td>90.922190</td>\n",
       "      <td>0.739673</td>\n",
       "      <td>0.465898</td>\n",
       "      <td>2.747358</td>\n",
       "      <td>54.444868</td>\n",
       "      <td>3.729488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>427.0</td>\n",
       "      <td>202</td>\n",
       "      <td>194.8</td>\n",
       "      <td>430.4</td>\n",
       "      <td>44243</td>\n",
       "      <td>75882</td>\n",
       "      <td>17.1</td>\n",
       "      <td>342.637253</td>\n",
       "      <td>(42724.4, 45201]</td>\n",
       "      <td>42.8</td>\n",
       "      <td>...</td>\n",
       "      <td>40.3</td>\n",
       "      <td>35.0</td>\n",
       "      <td>45.3</td>\n",
       "      <td>25.0</td>\n",
       "      <td>91.744686</td>\n",
       "      <td>0.782626</td>\n",
       "      <td>1.161359</td>\n",
       "      <td>1.362643</td>\n",
       "      <td>51.021514</td>\n",
       "      <td>4.603841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57.0</td>\n",
       "      <td>26</td>\n",
       "      <td>144.4</td>\n",
       "      <td>350.1</td>\n",
       "      <td>49955</td>\n",
       "      <td>10321</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>(48021.6, 51046.4]</td>\n",
       "      <td>48.3</td>\n",
       "      <td>...</td>\n",
       "      <td>43.9</td>\n",
       "      <td>35.1</td>\n",
       "      <td>44.0</td>\n",
       "      <td>22.7</td>\n",
       "      <td>94.104024</td>\n",
       "      <td>0.270192</td>\n",
       "      <td>0.665830</td>\n",
       "      <td>0.492135</td>\n",
       "      <td>54.027460</td>\n",
       "      <td>6.796657</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   avgAnnCount  avgDeathsPerYear  TARGET_deathRate  incidenceRate  medIncome  \\\n",
       "0       1397.0               469             164.9          489.8      61898   \n",
       "1        173.0                70             161.3          411.6      48127   \n",
       "2        102.0                50             174.7          349.7      49348   \n",
       "3        427.0               202             194.8          430.4      44243   \n",
       "4         57.0                26             144.4          350.1      49955   \n",
       "\n",
       "   popEst2015  povertyPercent  studyPerCap           binnedInc  MedianAge  \\\n",
       "0      260131            11.2   499.748204   (61494.5, 125635]       39.3   \n",
       "1       43269            18.6    23.111234  (48021.6, 51046.4]       33.0   \n",
       "2       21026            14.6    47.560164  (48021.6, 51046.4]       45.0   \n",
       "3       75882            17.1   342.637253    (42724.4, 45201]       42.8   \n",
       "4       10321            12.5     0.000000  (48021.6, 51046.4]       48.3   \n",
       "\n",
       "     ...      PctPrivateCoverageAlone  PctEmpPrivCoverage PctPublicCoverage  \\\n",
       "0    ...                          NaN                41.6              32.9   \n",
       "1    ...                         53.8                43.6              31.1   \n",
       "2    ...                         43.5                34.9              42.1   \n",
       "3    ...                         40.3                35.0              45.3   \n",
       "4    ...                         43.9                35.1              44.0   \n",
       "\n",
       "   PctPublicCoverageAlone   PctWhite  PctBlack  PctAsian  PctOtherRace  \\\n",
       "0                    14.0  81.780529  2.594728  4.821857      1.843479   \n",
       "1                    15.3  89.228509  0.969102  2.246233      3.741352   \n",
       "2                    21.1  90.922190  0.739673  0.465898      2.747358   \n",
       "3                    25.0  91.744686  0.782626  1.161359      1.362643   \n",
       "4                    22.7  94.104024  0.270192  0.665830      0.492135   \n",
       "\n",
       "   PctMarriedHouseholds  BirthRate  \n",
       "0             52.856076   6.118831  \n",
       "1             45.372500   4.333096  \n",
       "2             54.444868   3.729488  \n",
       "3             51.021514   4.603841  \n",
       "4             54.027460   6.796657  \n",
       "\n",
       "[5 rows x 34 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('cancer_reg.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# missing value handling\n",
    "df['PctSomeCol18_24'] = df['PctSomeCol18_24'].fillna(df['PctSomeCol18_24'].mean())\n",
    "df['PctEmployed16_Over'] = df['PctEmployed16_Over'].fillna(df['PctEmployed16_Over'].mean())\n",
    "df['PctPrivateCoverageAlone'] = df['PctPrivateCoverageAlone'].fillna(df['PctPrivateCoverageAlone'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    164.9\n",
       "1    161.3\n",
       "2    174.7\n",
       "3    194.8\n",
       "4    144.4\n",
       "Name: TARGET_deathRate, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = df['TARGET_deathRate']\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>avgAnnCount</th>\n",
       "      <th>avgDeathsPerYear</th>\n",
       "      <th>incidenceRate</th>\n",
       "      <th>medIncome</th>\n",
       "      <th>popEst2015</th>\n",
       "      <th>povertyPercent</th>\n",
       "      <th>studyPerCap</th>\n",
       "      <th>MedianAge</th>\n",
       "      <th>MedianAgeMale</th>\n",
       "      <th>MedianAgeFemale</th>\n",
       "      <th>...</th>\n",
       "      <th>Geography_York County, Pennsylvania</th>\n",
       "      <th>Geography_York County, South Carolina</th>\n",
       "      <th>Geography_York County, Virginia</th>\n",
       "      <th>Geography_Young County, Texas</th>\n",
       "      <th>Geography_Yuba County, California</th>\n",
       "      <th>Geography_Yukon-Koyukuk Census Area, Alaska</th>\n",
       "      <th>Geography_Yuma County, Arizona</th>\n",
       "      <th>Geography_Yuma County, Colorado</th>\n",
       "      <th>Geography_Zapata County, Texas</th>\n",
       "      <th>Geography_Zavala County, Texas</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1397.0</td>\n",
       "      <td>469</td>\n",
       "      <td>489.8</td>\n",
       "      <td>61898</td>\n",
       "      <td>260131</td>\n",
       "      <td>11.2</td>\n",
       "      <td>499.748204</td>\n",
       "      <td>39.3</td>\n",
       "      <td>36.9</td>\n",
       "      <td>41.7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>173.0</td>\n",
       "      <td>70</td>\n",
       "      <td>411.6</td>\n",
       "      <td>48127</td>\n",
       "      <td>43269</td>\n",
       "      <td>18.6</td>\n",
       "      <td>23.111234</td>\n",
       "      <td>33.0</td>\n",
       "      <td>32.2</td>\n",
       "      <td>33.7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>102.0</td>\n",
       "      <td>50</td>\n",
       "      <td>349.7</td>\n",
       "      <td>49348</td>\n",
       "      <td>21026</td>\n",
       "      <td>14.6</td>\n",
       "      <td>47.560164</td>\n",
       "      <td>45.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>45.8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>427.0</td>\n",
       "      <td>202</td>\n",
       "      <td>430.4</td>\n",
       "      <td>44243</td>\n",
       "      <td>75882</td>\n",
       "      <td>17.1</td>\n",
       "      <td>342.637253</td>\n",
       "      <td>42.8</td>\n",
       "      <td>42.2</td>\n",
       "      <td>43.4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57.0</td>\n",
       "      <td>26</td>\n",
       "      <td>350.1</td>\n",
       "      <td>49955</td>\n",
       "      <td>10321</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>48.3</td>\n",
       "      <td>47.8</td>\n",
       "      <td>48.9</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3078 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   avgAnnCount  avgDeathsPerYear  incidenceRate  medIncome  popEst2015  \\\n",
       "0       1397.0               469          489.8      61898      260131   \n",
       "1        173.0                70          411.6      48127       43269   \n",
       "2        102.0                50          349.7      49348       21026   \n",
       "3        427.0               202          430.4      44243       75882   \n",
       "4         57.0                26          350.1      49955       10321   \n",
       "\n",
       "   povertyPercent  studyPerCap  MedianAge  MedianAgeMale  MedianAgeFemale  \\\n",
       "0            11.2   499.748204       39.3           36.9             41.7   \n",
       "1            18.6    23.111234       33.0           32.2             33.7   \n",
       "2            14.6    47.560164       45.0           44.0             45.8   \n",
       "3            17.1   342.637253       42.8           42.2             43.4   \n",
       "4            12.5     0.000000       48.3           47.8             48.9   \n",
       "\n",
       "                ...                Geography_York County, Pennsylvania  \\\n",
       "0               ...                                                  0   \n",
       "1               ...                                                  0   \n",
       "2               ...                                                  0   \n",
       "3               ...                                                  0   \n",
       "4               ...                                                  0   \n",
       "\n",
       "   Geography_York County, South Carolina  Geography_York County, Virginia  \\\n",
       "0                                      0                                0   \n",
       "1                                      0                                0   \n",
       "2                                      0                                0   \n",
       "3                                      0                                0   \n",
       "4                                      0                                0   \n",
       "\n",
       "   Geography_Young County, Texas  Geography_Yuba County, California  \\\n",
       "0                              0                                  0   \n",
       "1                              0                                  0   \n",
       "2                              0                                  0   \n",
       "3                              0                                  0   \n",
       "4                              0                                  0   \n",
       "\n",
       "   Geography_Yukon-Koyukuk Census Area, Alaska  \\\n",
       "0                                            0   \n",
       "1                                            0   \n",
       "2                                            0   \n",
       "3                                            0   \n",
       "4                                            0   \n",
       "\n",
       "   Geography_Yuma County, Arizona  Geography_Yuma County, Colorado  \\\n",
       "0                               0                                0   \n",
       "1                               0                                0   \n",
       "2                               0                                0   \n",
       "3                               0                                0   \n",
       "4                               0                                0   \n",
       "\n",
       "   Geography_Zapata County, Texas  Geography_Zavala County, Texas  \n",
       "0                               0                               0  \n",
       "1                               0                               0  \n",
       "2                               0                               0  \n",
       "3                               0                               0  \n",
       "4                               0                               0  \n",
       "\n",
       "[5 rows x 3078 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = df\n",
    "x = x.drop('TARGET_deathRate', axis=1)\n",
    "x = x.drop('binnedInc', axis=1) #removed binnedInc field\n",
    "x.head()\n",
    "x = pd.get_dummies(x)\n",
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_x,test_x,train_y,test_y = train_test_split(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = linear_model.LinearRegression()\n",
    "lr.fit(train_x,train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "166.75087262224966"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2.86465051e-03,  1.60135840e-02,  1.92088818e-01, ...,\n",
       "        1.48800406e+01,  0.00000000e+00, -4.57450314e+01])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('new data score: ', 0.4270123712613325, 'same data score: ', 1.0)\n"
     ]
    }
   ],
   "source": [
    "print('new data score: ',lr.score(test_x,test_y), 'same data score: ',lr.score(train_x,train_y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}