data processing/data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ECG-ViEW II Data Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From our paper on \"Explainable Prediction of Acute Myocardial Infarction using Machine Learning and Shapley Values\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Import libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pylab as plt\n",
    "import imblearn\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.pipeline import Pipeline\n",
    "import seaborn as sns\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "\n",
    "# Set the global default size of matplotlib figures\n",
    "plt.rc('figure', figsize=(10, 5))\n",
    "\n",
    "# Size of matplotlib histogram bins\n",
    "bin_size = 40"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. ECG Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>ecgdate</th>\n",
       "      <th>ecgdept</th>\n",
       "      <th>ecgsource</th>\n",
       "      <th>RR</th>\n",
       "      <th>PR</th>\n",
       "      <th>QRS</th>\n",
       "      <th>QT</th>\n",
       "      <th>QTc</th>\n",
       "      <th>P_wave_axis</th>\n",
       "      <th>QRS_axis</th>\n",
       "      <th>T_wave_axis</th>\n",
       "      <th>ACCI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>120971</td>\n",
       "      <td>2006-05-23 09:57:00</td>\n",
       "      <td>H</td>\n",
       "      <td>M</td>\n",
       "      <td>952.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>388</td>\n",
       "      <td>397</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>120971</td>\n",
       "      <td>2012-04-19 08:09:06</td>\n",
       "      <td>H</td>\n",
       "      <td>E</td>\n",
       "      <td>750.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>360</td>\n",
       "      <td>415</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>270</td>\n",
       "      <td>2004-07-03 09:05:13</td>\n",
       "      <td>O</td>\n",
       "      <td>M</td>\n",
       "      <td>600.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>318</td>\n",
       "      <td>410</td>\n",
       "      <td>62.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>306709</td>\n",
       "      <td>2013-04-06 22:10:16</td>\n",
       "      <td>E</td>\n",
       "      <td>M</td>\n",
       "      <td>577.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>362</td>\n",
       "      <td>476</td>\n",
       "      <td>17.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87768</td>\n",
       "      <td>1997-12-17 11:11:26</td>\n",
       "      <td>O</td>\n",
       "      <td>M</td>\n",
       "      <td>698.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>354</td>\n",
       "      <td>423</td>\n",
       "      <td>28.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid              ecgdate ecgdept ecgsource     RR     PR   QRS   QT  \\\n",
       "0    120971  2006-05-23 09:57:00       H         M  952.0    NaN   NaN  388   \n",
       "1    120971  2012-04-19 08:09:06       H         E  750.0  140.0  84.0  360   \n",
       "2       270  2004-07-03 09:05:13       O         M  600.0  122.0  78.0  318   \n",
       "3    306709  2013-04-06 22:10:16       E         M  577.0  150.0  92.0  362   \n",
       "4     87768  1997-12-17 11:11:26       O         M  698.0  136.0  88.0  354   \n",
       "\n",
       "   QTc  P_wave_axis  QRS_axis  T_wave_axis  ACCI  \n",
       "0  397          NaN       NaN          NaN     9  \n",
       "1  415          NaN       NaN          NaN     9  \n",
       "2  410         62.0      86.0         61.0     7  \n",
       "3  476         17.0      46.0         70.0    12  \n",
       "4  423         28.0      59.0         29.0     3  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECG = pd.read_csv('Electrocardiogram.csv')\n",
    "df_ECG.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(979273, 13)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECG.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 979273 entries, 0 to 979272\n",
      "Data columns (total 13 columns):\n",
      " #   Column       Non-Null Count   Dtype  \n",
      "---  ------       --------------   -----  \n",
      " 0   personid     979273 non-null  int64  \n",
      " 1   ecgdate      979273 non-null  object \n",
      " 2   ecgdept      979273 non-null  object \n",
      " 3   ecgsource    979273 non-null  object \n",
      " 4   RR           979239 non-null  float64\n",
      " 5   PR           774260 non-null  float64\n",
      " 6   QRS          810373 non-null  float64\n",
      " 7   QT           979273 non-null  int64  \n",
      " 8   QTc          979273 non-null  int64  \n",
      " 9   P_wave_axis  732827 non-null  float64\n",
      " 10  QRS_axis     762274 non-null  float64\n",
      " 11  T_wave_axis  762248 non-null  float64\n",
      " 12  ACCI         979273 non-null  int64  \n",
      "dtypes: float64(6), int64(4), object(3)\n",
      "memory usage: 97.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df_ECG.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>RR</th>\n",
       "      <th>PR</th>\n",
       "      <th>QRS</th>\n",
       "      <th>QT</th>\n",
       "      <th>QTc</th>\n",
       "      <th>P_wave_axis</th>\n",
       "      <th>QRS_axis</th>\n",
       "      <th>T_wave_axis</th>\n",
       "      <th>ACCI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>979273.000000</td>\n",
       "      <td>979239.000000</td>\n",
       "      <td>774260.000000</td>\n",
       "      <td>810373.000000</td>\n",
       "      <td>979273.000000</td>\n",
       "      <td>979273.000000</td>\n",
       "      <td>732827.000000</td>\n",
       "      <td>762274.000000</td>\n",
       "      <td>762248.000000</td>\n",
       "      <td>979273.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>231021.688946</td>\n",
       "      <td>851.683240</td>\n",
       "      <td>157.081364</td>\n",
       "      <td>91.061474</td>\n",
       "      <td>389.997336</td>\n",
       "      <td>425.415081</td>\n",
       "      <td>48.357848</td>\n",
       "      <td>45.215670</td>\n",
       "      <td>46.198097</td>\n",
       "      <td>7.580933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>133238.981417</td>\n",
       "      <td>196.989159</td>\n",
       "      <td>26.661289</td>\n",
       "      <td>15.149489</td>\n",
       "      <td>43.490019</td>\n",
       "      <td>31.493654</td>\n",
       "      <td>24.848064</td>\n",
       "      <td>38.048566</td>\n",
       "      <td>38.416339</td>\n",
       "      <td>3.062956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>204.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>53.000000</td>\n",
       "      <td>-89.000000</td>\n",
       "      <td>-89.000000</td>\n",
       "      <td>-89.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>115711.000000</td>\n",
       "      <td>732.000000</td>\n",
       "      <td>140.000000</td>\n",
       "      <td>82.000000</td>\n",
       "      <td>364.000000</td>\n",
       "      <td>405.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>231037.000000</td>\n",
       "      <td>857.000000</td>\n",
       "      <td>156.000000</td>\n",
       "      <td>88.000000</td>\n",
       "      <td>390.000000</td>\n",
       "      <td>420.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>346411.000000</td>\n",
       "      <td>968.000000</td>\n",
       "      <td>172.000000</td>\n",
       "      <td>98.000000</td>\n",
       "      <td>416.000000</td>\n",
       "      <td>440.000000</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>71.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>461795.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>1738.000000</td>\n",
       "      <td>380.000000</td>\n",
       "      <td>852.000000</td>\n",
       "      <td>878.000000</td>\n",
       "      <td>270.000000</td>\n",
       "      <td>270.000000</td>\n",
       "      <td>270.000000</td>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            personid             RR             PR            QRS  \\\n",
       "count  979273.000000  979239.000000  774260.000000  810373.000000   \n",
       "mean   231021.688946     851.683240     157.081364      91.061474   \n",
       "std    133238.981417     196.989159      26.661289      15.149489   \n",
       "min         1.000000     204.000000      36.000000       2.000000   \n",
       "25%    115711.000000     732.000000     140.000000      82.000000   \n",
       "50%    231037.000000     857.000000     156.000000      88.000000   \n",
       "75%    346411.000000     968.000000     172.000000      98.000000   \n",
       "max    461795.000000   30000.000000    1738.000000     380.000000   \n",
       "\n",
       "                  QT            QTc    P_wave_axis       QRS_axis  \\\n",
       "count  979273.000000  979273.000000  732827.000000  762274.000000   \n",
       "mean      389.997336     425.415081      48.357848      45.215670   \n",
       "std        43.490019      31.493654      24.848064      38.048566   \n",
       "min        70.000000      53.000000     -89.000000     -89.000000   \n",
       "25%       364.000000     405.000000      36.000000      22.000000   \n",
       "50%       390.000000     420.000000      52.000000      51.000000   \n",
       "75%       416.000000     440.000000      65.000000      71.000000   \n",
       "max       852.000000     878.000000     270.000000     270.000000   \n",
       "\n",
       "         T_wave_axis           ACCI  \n",
       "count  762248.000000  979273.000000  \n",
       "mean       46.198097       7.580933  \n",
       "std        38.416339       3.062956  \n",
       "min       -89.000000       0.000000  \n",
       "25%        28.000000       6.000000  \n",
       "50%        44.000000       8.000000  \n",
       "75%        59.000000      10.000000  \n",
       "max       270.000000      23.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECG.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "personid            0\n",
       "ecgdate             0\n",
       "ecgdept             0\n",
       "ecgsource           0\n",
       "RR                 34\n",
       "PR             205013\n",
       "QRS            168900\n",
       "QT                  0\n",
       "QTc                 0\n",
       "P_wave_axis    246446\n",
       "QRS_axis       216999\n",
       "T_wave_axis    217025\n",
       "ACCI                0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECG.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop instances that have missing values \n",
    "df_ECG_full = df_ECG.dropna(subset=['RR', 'PR', 'QRS', 'P_wave_axis','QRS_axis', 'T_wave_axis']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "personid       0\n",
       "ecgdate        0\n",
       "ecgdept        0\n",
       "ecgsource      0\n",
       "RR             0\n",
       "PR             0\n",
       "QRS            0\n",
       "QT             0\n",
       "QTc            0\n",
       "P_wave_axis    0\n",
       "QRS_axis       0\n",
       "T_wave_axis    0\n",
       "ACCI           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECG_full.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(726444, 13)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECG_full.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop unrelevant information \n",
    "df_ECG_full = df_ECG_full.drop(columns = ['ecgdate', 'ecgsource', 'ecgdept'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>RR</th>\n",
       "      <th>PR</th>\n",
       "      <th>QRS</th>\n",
       "      <th>QT</th>\n",
       "      <th>QTc</th>\n",
       "      <th>P_wave_axis</th>\n",
       "      <th>QRS_axis</th>\n",
       "      <th>T_wave_axis</th>\n",
       "      <th>ACCI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>270</td>\n",
       "      <td>600.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>318</td>\n",
       "      <td>410</td>\n",
       "      <td>62.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>306709</td>\n",
       "      <td>577.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>362</td>\n",
       "      <td>476</td>\n",
       "      <td>17.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87768</td>\n",
       "      <td>698.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>354</td>\n",
       "      <td>423</td>\n",
       "      <td>28.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>209545</td>\n",
       "      <td>638.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>368</td>\n",
       "      <td>460</td>\n",
       "      <td>14.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>269001</td>\n",
       "      <td>741.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>362</td>\n",
       "      <td>420</td>\n",
       "      <td>65.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid     RR     PR   QRS   QT  QTc  P_wave_axis  QRS_axis  T_wave_axis  \\\n",
       "2       270  600.0  122.0  78.0  318  410         62.0      86.0         61.0   \n",
       "3    306709  577.0  150.0  92.0  362  476         17.0      46.0         70.0   \n",
       "4     87768  698.0  136.0  88.0  354  423         28.0      59.0         29.0   \n",
       "5    209545  638.0  112.0  94.0  368  460         14.0     132.0         -3.0   \n",
       "8    269001  741.0  136.0  84.0  362  420         65.0      42.0         31.0   \n",
       "\n",
       "   ACCI  \n",
       "2     7  \n",
       "3    12  \n",
       "4     3  \n",
       "5     6  \n",
       "8     7  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ECG_full.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ECG_full['personid'].value_counts();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Diagnosis Code + Personal Info (Sex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>diagdate</th>\n",
       "      <th>diagcode</th>\n",
       "      <th>diaglocalcode</th>\n",
       "      <th>diagdept</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>141778</td>\n",
       "      <td>1997-06-04 00:00:00</td>\n",
       "      <td>L73.8</td>\n",
       "      <td>DC12467</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>141778</td>\n",
       "      <td>1997-06-07 00:00:00</td>\n",
       "      <td>L73.8</td>\n",
       "      <td>DC12467</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>141778</td>\n",
       "      <td>1997-06-16 00:00:00</td>\n",
       "      <td>L73.8</td>\n",
       "      <td>DC12467</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>141778</td>\n",
       "      <td>1997-06-23 00:00:00</td>\n",
       "      <td>L73.8</td>\n",
       "      <td>DC12467</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>141778</td>\n",
       "      <td>1997-06-30 00:00:00</td>\n",
       "      <td>L73.8</td>\n",
       "      <td>DC12467</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid             diagdate diagcode diaglocalcode diagdept\n",
       "0    141778  1997-06-04 00:00:00    L73.8       DC12467        O\n",
       "1    141778  1997-06-07 00:00:00    L73.8       DC12467        O\n",
       "2    141778  1997-06-16 00:00:00    L73.8       DC12467        O\n",
       "3    141778  1997-06-23 00:00:00    L73.8       DC12467        O\n",
       "4    141778  1997-06-30 00:00:00    L73.8       DC12467        O"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag = pd.read_csv('Diagnosis.csv', encoding='latin-1')\n",
    "df_diag.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7743772, 5)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop duplicate entries (repeated entry of the same ID)\n",
    "df_diag_two = df_diag.sort_values('diagdate').drop_duplicates('personid',keep='last')\n",
    "df_diag_two.sort_values(by=['personid']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diag_two['personid'].value_counts();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(424363, 5)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag_two.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>sex</th>\n",
       "      <th>Birthyeargroup</th>\n",
       "      <th>ethnicity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>369485</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>138392</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>188469</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>169164</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>434516</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid  sex  Birthyeargroup  ethnicity\n",
       "0    369485    1              16          1\n",
       "1    138392    0              14          1\n",
       "2    188469    1              20          1\n",
       "3    169164    0              12          1\n",
       "4    434516    1              14          1"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_person = pd.read_csv('Person.csv')\n",
    "df_person.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(461034, 4)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_person.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge the diagnosis dataset with the personal info dataset with the common column (personid)\n",
    "df_diag_person = pd.merge(df_diag_two, df_person, on='personid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>diagdate</th>\n",
       "      <th>diagcode</th>\n",
       "      <th>diaglocalcode</th>\n",
       "      <th>diagdept</th>\n",
       "      <th>sex</th>\n",
       "      <th>Birthyeargroup</th>\n",
       "      <th>ethnicity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>68536</td>\n",
       "      <td>1994-08-30 02:37:00</td>\n",
       "      <td>K01.1</td>\n",
       "      <td>DC9189</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>216182</td>\n",
       "      <td>1995-01-03 23:45:00</td>\n",
       "      <td>G91.9</td>\n",
       "      <td>DC78</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>458392</td>\n",
       "      <td>1995-01-04 16:27:00</td>\n",
       "      <td>S72.20</td>\n",
       "      <td>DC13004</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>396139</td>\n",
       "      <td>1995-01-06 15:19:00</td>\n",
       "      <td>M19.99</td>\n",
       "      <td>DC7130</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>251126</td>\n",
       "      <td>1995-01-12 17:33:00</td>\n",
       "      <td>M95303</td>\n",
       "      <td>DC13765</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid             diagdate diagcode diaglocalcode diagdept  sex  \\\n",
       "0     68536  1994-08-30 02:37:00    K01.1        DC9189        I    1   \n",
       "1    216182  1995-01-03 23:45:00    G91.9          DC78        I    1   \n",
       "2    458392  1995-01-04 16:27:00   S72.20       DC13004        I    1   \n",
       "3    396139  1995-01-06 15:19:00   M19.99        DC7130        I    0   \n",
       "4    251126  1995-01-12 17:33:00   M95303       DC13765        I    0   \n",
       "\n",
       "   Birthyeargroup  ethnicity  \n",
       "0              16          1  \n",
       "1               4          1  \n",
       "2              10          1  \n",
       "3               7          1  \n",
       "4              11          1  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag_person.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(424363, 8)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag_person.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "personid          0\n",
       "diagdate          0\n",
       "diagcode          0\n",
       "diaglocalcode     0\n",
       "diagdept          0\n",
       "sex               0\n",
       "Birthyeargroup    0\n",
       "ethnicity         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag_person.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>diagdate</th>\n",
       "      <th>diagcode</th>\n",
       "      <th>diaglocalcode</th>\n",
       "      <th>diagdept</th>\n",
       "      <th>sex</th>\n",
       "      <th>Birthyeargroup</th>\n",
       "      <th>ethnicity</th>\n",
       "      <th>MI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>68536</td>\n",
       "      <td>1994-08-30 02:37:00</td>\n",
       "      <td>K01.1</td>\n",
       "      <td>DC9189</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>216182</td>\n",
       "      <td>1995-01-03 23:45:00</td>\n",
       "      <td>G91.9</td>\n",
       "      <td>DC78</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>458392</td>\n",
       "      <td>1995-01-04 16:27:00</td>\n",
       "      <td>S72.20</td>\n",
       "      <td>DC13004</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>396139</td>\n",
       "      <td>1995-01-06 15:19:00</td>\n",
       "      <td>M19.99</td>\n",
       "      <td>DC7130</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>251126</td>\n",
       "      <td>1995-01-12 17:33:00</td>\n",
       "      <td>M95303</td>\n",
       "      <td>DC13765</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid             diagdate diagcode diaglocalcode diagdept  sex  \\\n",
       "0     68536  1994-08-30 02:37:00    K01.1        DC9189        I    1   \n",
       "1    216182  1995-01-03 23:45:00    G91.9          DC78        I    1   \n",
       "2    458392  1995-01-04 16:27:00   S72.20       DC13004        I    1   \n",
       "3    396139  1995-01-06 15:19:00   M19.99        DC7130        I    0   \n",
       "4    251126  1995-01-12 17:33:00   M95303       DC13765        I    0   \n",
       "\n",
       "   Birthyeargroup  ethnicity MI  \n",
       "0              16          1  0  \n",
       "1               4          1  0  \n",
       "2              10          1  0  \n",
       "3               7          1  0  \n",
       "4              11          1  0  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "####################### Create a new feature base on the diagnosis code: MI (1 = MI, 0 = no) ################\n",
    "\n",
    "# To select rows whose column value is in list \n",
    "code = ['DC13972','DC3641','DC2108','DC8232','DC910','DC7547',\n",
    "        'DC2624','DC2485']\n",
    "condition = df_diag_person.diaglocalcode.isin(code)\n",
    "\n",
    "df_diag_person['MI'] = np.where(condition, '1', '0')\n",
    "df_diag_person.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(424363, 9)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag_person.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DC14546    2\n",
       "Name: diaglocalcode, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the number of entries of a specific diagnostic code \n",
    "df_diag_person['diaglocalcode'][df_diag_person['diaglocalcode'] == \"DC14546\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diag_person['diaglocalcode'].value_counts();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    422766\n",
       "1      1597\n",
       "Name: MI, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag_person['MI'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x10d326550>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAE7CAYAAACG3xazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAaAUlEQVR4nO3df8ye1X3f8fcnNqS0GYHAU0ZtmFHxlppscRKXOMr+yGABQ7qaTCQCTcWLvLhRYErUasXkH9okSOSPlg4poaLDw7AsDiKp8FInrgdEUbTxw2lciCGMJ/yI7fHDwQaaRoEB3/3xHJqbp/fzyz94jm/eL+nSc13fc851zs0f1kfXdR/uVBWSJEnqy5vmewGSJEn6hwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCbpiJbksSQvJjlxUv37SSrJknZ9Y5LPT3OfY5P8aZIfJ/lpkh+16xOnGnOI1v/vk3z3cM4h6chkSJM0Ch4FLn71Isk/B355toOTHA3cDpwBrAKOBd4HPAOceUhXKkmzZEiTNApuBi4ZuF4D3DSH8ZcApwIfrqoHquqVqnq6qj5XVVsAkvxGkm8neTbJziS//ergVv8PA9eveTrWnuh9IsnDbfwXM+E3gD8D3tee3j17YB9f0igypEkaBXcBx7YgtQC4CPhvcxj/r4FvVdVPhzUmOQr4H8BfAb8K/Efgy0n+2Rzm+C3gN4F/AXwUOLeqHgQ+AfzvqnpLVR03h/tJGnGGNEmj4tWnaR8EHgT2zGHsCcAT07SvBN4CXF1VL1bVHcA3GHjFOgtXV9WzVfVj4E5g+RzGSnoDWjjfC5CkQ+Rm4DvAacztVSdMfPfs5Gnafw3YVVWvDNQeBxbNYY4nB85/xkTok6Qp+SRN0kioqseZ2EBwPvD1OQ7/n8C5SX5livb/C5ySZPDfzFP5xdO6v+O1GxX+8Rzmrjn0lfQGYkiTNErWAmdV1d/NcdzNwC7ga0nenuRNSU5I8pkk5wN3M/H06w+SHJXkA8C/ATa18TuAf5vkl5Oc3tYxW08Bi9sOU0n6e4Y0SSOjqn5UVdsPYNwLTGwe+CGwDXgeuAc4Ebi7ql5kIpSdB/wE+BJwSVX9sN3iGuBFJgLXRuDLc5j+DmAn8GSSn8x17ZJGV6p80i5JktQbn6RJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdWjkfnHgxBNPrCVLlsz3MiRJkmb0ve997ydVNTasbeRC2pIlS9i+fc7/myRJkqTXXZLHp2rzdackSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHVo5H67U8MtWf+X870EHSEeu/pD870ESRI+SZMkSeqSIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6tCsQ1qSBUm+n+Qb7fq0JHcnGU/y1SRHt/qb2/V4a18ycI8rWv2hJOcO1Fe12niS9QP1oXNIkiSNurk8SfsU8ODA9ReAa6rqdGA/sLbV1wL7W/2a1o8ky4CLgDOAVcCXWvBbAHwROA9YBlzc+k43hyRJ0kibVUhLshj4EPBf2nWAs4BbW5eNwAXtfHW7prWf3fqvBjZV1QtV9SgwDpzZjvGqeqSqXgQ2AatnmEOSJGmkzfZJ2p8CfwC80q5PAJ6tqpfa9W5gUTtfBOwCaO3Ptf5/X580Zqr6dHNIkiSNtBlDWpLfAp6uqu+9Dus5IEnWJdmeZPvevXvnezmSJEkHbTZP0t4P/HaSx5h4FXkW8J+B45IsbH0WA3va+R7gFIDW/lbgmcH6pDFT1Z+ZZo7XqKrrq2pFVa0YGxubxUeSJEnq24whraquqKrFVbWEiS/+31FV/w64E7iwdVsD3NbON7drWvsdVVWtflHb/XkasBS4B7gXWNp2ch7d5tjcxkw1hyRJ0kg7mP9P2uXA7yUZZ+L7Yze0+g3ACa3+e8B6gKraCdwCPAB8C7i0ql5u3zm7DNjKxO7RW1rf6eaQJEkaaQtn7vILVfVt4Nvt/BEmdmZO7vNz4CNTjL8KuGpIfQuwZUh96BySJEmjzl8ckCRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQzOGtCS/lOSeJH+TZGeSP2r1G5M8mmRHO5a3epJcm2Q8yX1J3j1wrzVJHm7HmoH6e5Lc38ZcmySt/rYk21r/bUmOP/T/CSRJkvozmydpLwBnVdU7geXAqiQrW9t/qqrl7djRaucBS9uxDrgOJgIXcCXwXuBM4MqB0HUd8PGBcatafT1we1UtBW5v15IkSSNvxpBWE37aLo9qR00zZDVwUxt3F3BckpOBc4FtVbWvqvYD25gIfCcDx1bVXVVVwE3ABQP32tjONw7UJUmSRtqsvpOWZEGSHcDTTAStu1vTVe2V5jVJ3txqi4BdA8N3t9p09d1D6gAnVdUT7fxJ4KQp1rcuyfYk2/fu3TubjyRJktS1WYW0qnq5qpYDi4Ezk7wDuAJ4O/CbwNuAyw/bKifWUEzxBK+qrq+qFVW1Ymxs7HAuQ5Ik6XUxp92dVfUscCewqqqeaK80XwD+KxPfMwPYA5wyMGxxq01XXzykDvBUex1K+/v0XNYrSZJ0pJrN7s6xJMe182OADwI/HAhPYeK7Yj9oQzYDl7RdniuB59ory63AOUmObxsGzgG2trbnk6xs97oEuG3gXq/uAl0zUJckSRppC2fR52RgY5IFTIS6W6rqG0nuSDIGBNgBfKL13wKcD4wDPwM+BlBV+5J8Dri39ftsVe1r558EbgSOAb7ZDoCrgVuSrAUeBz56oB9UkiTpSDJjSKuq+4B3DamfNUX/Ai6dom0DsGFIfTvwjiH1Z4CzZ1qjJEnSqPEXByRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOjRjSEvyS0nuSfI3SXYm+aNWPy3J3UnGk3w1ydGt/uZ2Pd7alwzc64pWfyjJuQP1Va02nmT9QH3oHJIkSaNuNk/SXgDOqqp3AsuBVUlWAl8Arqmq04H9wNrWfy2wv9Wvaf1Isgy4CDgDWAV8KcmCJAuALwLnAcuAi1tfpplDkiRppM0Y0mrCT9vlUe0o4Czg1lbfCFzQzle3a1r72UnS6puq6oWqehQYB85sx3hVPVJVLwKbgNVtzFRzSJIkjbRZfSetPfHaATwNbAN+BDxbVS+1LruBRe18EbALoLU/B5wwWJ80Zqr6CdPMIUmSNNJmFdKq6uWqWg4sZuLJ19sP66rmKMm6JNuTbN+7d+98L0eSJOmgzWl3Z1U9C9wJvA84LsnC1rQY2NPO9wCnALT2twLPDNYnjZmq/sw0c0xe1/VVtaKqVoyNjc3lI0mSJHVpNrs7x5Ic186PAT4IPMhEWLuwdVsD3NbON7drWvsdVVWtflHb/XkasBS4B7gXWNp2ch7NxOaCzW3MVHNIkiSNtIUzd+FkYGPbhfkm4Jaq+kaSB4BNST4PfB+4ofW/Abg5yTiwj4nQRVXtTHIL8ADwEnBpVb0MkOQyYCuwANhQVTvbvS6fYg5JkqSRNmNIq6r7gHcNqT/CxPfTJtd/DnxkintdBVw1pL4F2DLbOSRJkkadvzggSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHZgxpSU5JcmeSB5LsTPKpVv/DJHuS7GjH+QNjrkgynuShJOcO1Fe12niS9QP105Lc3epfTXJ0q7+5XY+39iWH8sNLkiT1ajZP0l4Cfr+qlgErgUuTLGtt11TV8nZsAWhtFwFnAKuALyVZkGQB8EXgPGAZcPHAfb7Q7nU6sB9Y2+prgf2tfk3rJ0mSNPJmDGlV9URV/XU7/1vgQWDRNENWA5uq6oWqehQYB85sx3hVPVJVLwKbgNVJApwF3NrGbwQuGLjXxnZ+K3B26y9JkjTS5vSdtPa68V3A3a10WZL7kmxIcnyrLQJ2DQzb3WpT1U8Anq2qlybVX3Ov1v5c6z95XeuSbE+yfe/evXP5SJIkSV2adUhL8hbga8Cnq+p54Drg14HlwBPAHx+WFc5CVV1fVSuqasXY2Nh8LUOSJOmQmVVIS3IUEwHty1X1dYCqeqqqXq6qV4A/Z+J1JsAe4JSB4Ytbbar6M8BxSRZOqr/mXq39ra2/JEnSSJvN7s4ANwAPVtWfDNRPHuj2YeAH7XwzcFHbmXkasBS4B7gXWNp2ch7NxOaCzVVVwJ3AhW38GuC2gXutaecXAne0/pIkSSNt4cxdeD/wO8D9SXa02meY2J25HCjgMeB3AapqZ5JbgAeY2Bl6aVW9DJDkMmArsADYUFU72/0uBzYl+TzwfSZCIe3vzUnGgX1MBDtJkqSRN2NIq6rvAsN2VG6ZZsxVwFVD6luGjauqR/jF69LB+s+Bj8y0RkmSpFHjLw5IkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktShGUNaklOS3JnkgSQ7k3yq1d+WZFuSh9vf41s9Sa5NMp7kviTvHrjXmtb/4SRrBurvSXJ/G3Ntkkw3hyRJ0qibzZO0l4Dfr6plwErg0iTLgPXA7VW1FLi9XQOcByxtxzrgOpgIXMCVwHuBM4ErB0LXdcDHB8atavWp5pAkSRppM4a0qnqiqv66nf8t8CCwCFgNbGzdNgIXtPPVwE014S7guCQnA+cC26pqX1XtB7YBq1rbsVV1V1UVcNOkew2bQ5IkaaTN6TtpSZYA7wLuBk6qqida05PASe18EbBrYNjuVpuuvntInWnmmLyudUm2J9m+d+/euXwkSZKkLs06pCV5C/A14NNV9fxgW3sCVod4ba8x3RxVdX1VraiqFWNjY4dzGZIkSa+LWYW0JEcxEdC+XFVfb+Wn2qtK2t+nW30PcMrA8MWtNl198ZD6dHNIkiSNtNns7gxwA/BgVf3JQNNm4NUdmmuA2wbql7RdniuB59ory63AOUmObxsGzgG2trbnk6xsc10y6V7D5pAkSRppC2fR5/3A7wD3J9nRap8BrgZuSbIWeBz4aGvbApwPjAM/Az4GUFX7knwOuLf1+2xV7WvnnwRuBI4BvtkOpplDkiRppM0Y0qrqu0CmaD57SP8CLp3iXhuADUPq24F3DKk/M2wOSZKkUecvDkiSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdWjGkJZkQ5Knk/xgoPaHSfYk2dGO8wfarkgynuShJOcO1Fe12niS9QP105Lc3epfTXJ0q7+5XY+39iWH6kNLkiT1bjZP0m4EVg2pX1NVy9uxBSDJMuAi4Iw25ktJFiRZAHwROA9YBlzc+gJ8od3rdGA/sLbV1wL7W/2a1k+SJOkNYcaQVlXfAfbN8n6rgU1V9UJVPQqMA2e2Y7yqHqmqF4FNwOokAc4Cbm3jNwIXDNxrYzu/FTi79ZckSRp5B/OdtMuS3Ndehx7faouAXQN9drfaVPUTgGer6qVJ9dfcq7U/1/pLkiSNvAMNadcBvw4sB54A/viQregAJFmXZHuS7Xv37p3PpUiSJB0SBxTSquqpqnq5ql4B/pyJ15kAe4BTBroubrWp6s8AxyVZOKn+mnu19re2/sPWc31VraiqFWNjYwfykSRJkrpyQCEtyckDlx8GXt35uRm4qO3MPA1YCtwD3AssbTs5j2Zic8HmqirgTuDCNn4NcNvAvda08wuBO1p/SZKkkbdwpg5JvgJ8ADgxyW7gSuADSZYDBTwG/C5AVe1McgvwAPAScGlVvdzucxmwFVgAbKiqnW2Ky4FNST4PfB+4odVvAG5OMs7ExoWLDvrTSpIkHSFmDGlVdfGQ8g1Daq/2vwq4akh9C7BlSP0RfvG6dLD+c+AjM61PkiRpFPmLA5IkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdWjGkJZkQ5Knk/xgoPa2JNuSPNz+Ht/qSXJtkvEk9yV598CYNa3/w0nWDNTfk+T+NubaJJluDkmSpDeC2TxJuxFYNam2Hri9qpYCt7drgPOApe1YB1wHE4ELuBJ4L3AmcOVA6LoO+PjAuFUzzCFJkjTyZgxpVfUdYN+k8mpgYzvfCFwwUL+pJtwFHJfkZOBcYFtV7auq/cA2YFVrO7aq7qqqAm6adK9hc0iSJI28A/1O2klV9UQ7fxI4qZ0vAnYN9NvdatPVdw+pTzfHP5BkXZLtSbbv3bv3AD6OJElSXw5640B7AlaHYC0HPEdVXV9VK6pqxdjY2OFciiRJ0uviQEPaU+1VJe3v062+BzhloN/iVpuuvnhIfbo5JEmSRt6BhrTNwKs7NNcAtw3UL2m7PFcCz7VXlluBc5Ic3zYMnANsbW3PJ1nZdnVeMulew+aQJEkaeQtn6pDkK8AHgBOT7GZil+bVwC1J1gKPAx9t3bcA5wPjwM+AjwFU1b4knwPubf0+W1Wvbkb4JBM7SI8BvtkOpplDkiRp5M0Y0qrq4imazh7St4BLp7jPBmDDkPp24B1D6s8Mm0OSJOmNwF8ckCRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnq0EGFtCSPJbk/yY4k21vtbUm2JXm4/T2+1ZPk2iTjSe5L8u6B+6xp/R9Osmag/p52//E2NgezXkmSpCPFoXiS9q+qanlVrWjX64Hbq2opcHu7BjgPWNqOdcB1MBHqgCuB9wJnAle+Guxan48PjFt1CNYrSZLUvcPxunM1sLGdbwQuGKjfVBPuAo5LcjJwLrCtqvZV1X5gG7CqtR1bVXdVVQE3DdxLkiRppB1sSCvgr5J8L8m6Vjupqp5o508CJ7XzRcCugbG7W226+u4hdUmSpJG38CDH/8uq2pPkV4FtSX442FhVlaQOco4ZtYC4DuDUU0893NNJkiQddgf1JK2q9rS/TwN/wcR3yp5qryppf59u3fcApwwMX9xq09UXD6kPW8f1VbWiqlaMjY0dzEeSJEnqwgGHtCS/kuQfvXoOnAP8ANgMvLpDcw1wWzvfDFzSdnmuBJ5rr0W3AuckOb5tGDgH2Nrank+ysu3qvGTgXpIkSSPtYF53ngT8Rfu/YiwE/ntVfSvJvcAtSdYCjwMfbf23AOcD48DPgI8BVNW+JJ8D7m39PltV+9r5J4EbgWOAb7ZDkiRp5B1wSKuqR4B3Dqk/A5w9pF7ApVPcawOwYUh9O/COA12jJEnSkcpfHJAkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUPdh7Qkq5I8lGQ8yfr5Xo8kSdLroeuQlmQB8EXgPGAZcHGSZfO7KkmSpMNv4XwvYAZnAuNV9QhAkk3AauCBeV2VJAmAJev/cr6XoCPEY1d/aL6XcMTpPaQtAnYNXO8G3ju5U5J1wLp2+dMkD70Oa9OR70TgJ/O9iN7kC/O9AumI578tQ/hvy5T+yVQNvYe0Wamq64Hr53sdOrIk2V5VK+Z7HZJGi/+26FDp+jtpwB7glIHrxa0mSZI00noPafcCS5OcluRo4CJg8zyvSZIk6bDr+nVnVb2U5DJgK7AA2FBVO+d5WRodviKXdDj4b4sOiVTVfK9BkiRJk/T+ulOSJOkNyZAmSZLUIUOaJElSh7reOCAdSknezsQvVixqpT3A5qp6cP5WJUnScD5J0xtCksuBTUCAe9oR4CtJ1s/n2iSNriQfm+816Mjl7k69IST5P8AZVfX/JtWPBnZW1dL5WZmkUZbkx1V16nyvQ0cmX3fqjeIV4NeAxyfVT25tknRAktw3VRNw0uu5Fo0WQ5reKD4N3J7kYWBXq50KnA5cNm+rkjQKTgLOBfZPqgf4X6//cjQqDGl6Q6iqbyX5p8CZvHbjwL1V9fL8rUzSCPgG8Jaq2jG5Icm3X//laFT4nTRJkqQOubtTkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUP/H8Q3ebkYPCO3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_diag_person['MI'].value_counts().plot(kind='bar', title='MI Count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>diagdate</th>\n",
       "      <th>diagcode</th>\n",
       "      <th>diaglocalcode</th>\n",
       "      <th>diagdept</th>\n",
       "      <th>sex</th>\n",
       "      <th>Birthyeargroup</th>\n",
       "      <th>ethnicity</th>\n",
       "      <th>MI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>68536</td>\n",
       "      <td>1994-08-30 02:37:00</td>\n",
       "      <td>K01.1</td>\n",
       "      <td>DC9189</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>216182</td>\n",
       "      <td>1995-01-03 23:45:00</td>\n",
       "      <td>G91.9</td>\n",
       "      <td>DC78</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>458392</td>\n",
       "      <td>1995-01-04 16:27:00</td>\n",
       "      <td>S72.20</td>\n",
       "      <td>DC13004</td>\n",
       "      <td>I</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>396139</td>\n",
       "      <td>1995-01-06 15:19:00</td>\n",
       "      <td>M19.99</td>\n",
       "      <td>DC7130</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>251126</td>\n",
       "      <td>1995-01-12 17:33:00</td>\n",
       "      <td>M95303</td>\n",
       "      <td>DC13765</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid             diagdate diagcode diaglocalcode diagdept  sex  \\\n",
       "0     68536  1994-08-30 02:37:00    K01.1        DC9189        I    1   \n",
       "1    216182  1995-01-03 23:45:00    G91.9          DC78        I    1   \n",
       "2    458392  1995-01-04 16:27:00   S72.20       DC13004        I    1   \n",
       "3    396139  1995-01-06 15:19:00   M19.99        DC7130        I    0   \n",
       "4    251126  1995-01-12 17:33:00   M95303       DC13765        I    0   \n",
       "\n",
       "   Birthyeargroup  ethnicity MI  \n",
       "0              16          1  0  \n",
       "1               4          1  0  \n",
       "2              10          1  0  \n",
       "3               7          1  0  \n",
       "4              11          1  0  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diag_person.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Merge To Obtain The Final Dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>personid</th>\n",
       "      <th>RR</th>\n",
       "      <th>PR</th>\n",
       "      <th>QRS</th>\n",
       "      <th>QT</th>\n",
       "      <th>QTc</th>\n",
       "      <th>P_wave_axis</th>\n",
       "      <th>QRS_axis</th>\n",
       "      <th>T_wave_axis</th>\n",
       "      <th>ACCI</th>\n",
       "      <th>diagdate</th>\n",
       "      <th>diagcode</th>\n",
       "      <th>diaglocalcode</th>\n",
       "      <th>diagdept</th>\n",
       "      <th>sex</th>\n",
       "      <th>Birthyeargroup</th>\n",
       "      <th>ethnicity</th>\n",
       "      <th>MI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>270</td>\n",
       "      <td>600.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>318</td>\n",
       "      <td>410</td>\n",
       "      <td>62.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>7</td>\n",
       "      <td>2004-07-19 00:00:00</td>\n",
       "      <td>J02.9</td>\n",
       "      <td>DC1151</td>\n",
       "      <td>O</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>270</td>\n",
       "      <td>594.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>306</td>\n",
       "      <td>396</td>\n",
       "      <td>67.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7</td>\n",
       "      <td>2004-07-19 00:00:00</td>\n",
       "      <td>J02.9</td>\n",
       "      <td>DC1151</td>\n",
       "      <td>O</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>306709</td>\n",
       "      <td>577.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>362</td>\n",
       "      <td>476</td>\n",
       "      <td>17.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>12</td>\n",
       "      <td>2013-06-16 17:15:00</td>\n",
       "      <td>I10.9</td>\n",
       "      <td>DC7617</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>306709</td>\n",
       "      <td>645.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>372</td>\n",
       "      <td>462</td>\n",
       "      <td>99.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>11</td>\n",
       "      <td>2013-06-16 17:15:00</td>\n",
       "      <td>I10.9</td>\n",
       "      <td>DC7617</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>306709</td>\n",
       "      <td>638.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>378</td>\n",
       "      <td>472</td>\n",
       "      <td>17.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>12</td>\n",
       "      <td>2013-06-16 17:15:00</td>\n",
       "      <td>I10.9</td>\n",
       "      <td>DC7617</td>\n",
       "      <td>I</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   personid     RR     PR   QRS   QT  QTc  P_wave_axis  QRS_axis  T_wave_axis  \\\n",
       "0       270  600.0  122.0  78.0  318  410         62.0      86.0         61.0   \n",
       "1       270  594.0  124.0  78.0  306  396         67.0      85.0         60.0   \n",
       "2    306709  577.0  150.0  92.0  362  476         17.0      46.0         70.0   \n",
       "3    306709  645.0  162.0  90.0  372  462         99.0      55.0         82.0   \n",
       "4    306709  638.0  200.0  94.0  378  472         17.0      67.0         84.0   \n",
       "\n",
       "   ACCI             diagdate diagcode diaglocalcode diagdept  sex  \\\n",
       "0     7  2004-07-19 00:00:00    J02.9        DC1151        O    1   \n",
       "1     7  2004-07-19 00:00:00    J02.9        DC1151        O    1   \n",
       "2    12  2013-06-16 17:15:00    I10.9        DC7617        I    0   \n",
       "3    11  2013-06-16 17:15:00    I10.9        DC7617        I    0   \n",
       "4    12  2013-06-16 17:15:00    I10.9        DC7617        I    0   \n",
       "\n",
       "   Birthyeargroup  ethnicity MI  \n",
       "0              21          1  0  \n",
       "1              21          1  0  \n",
       "2               7          1  0  \n",
       "3               7          1  0  \n",
       "4               7          1  0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge the ecg dataset with the combined diagnosis/personal info dataset using the common column (personid)\n",
    "df_1 = pd.merge(df_ECG_full, df_diag_person, on='personid')\n",
    "df_1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(713447, 18)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RR</th>\n",
       "      <th>PR</th>\n",
       "      <th>QRS</th>\n",
       "      <th>QT</th>\n",
       "      <th>QTc</th>\n",
       "      <th>P_wave_axis</th>\n",
       "      <th>QRS_axis</th>\n",
       "      <th>T_wave_axis</th>\n",
       "      <th>ACCI</th>\n",
       "      <th>sex</th>\n",
       "      <th>Birthyeargroup</th>\n",
       "      <th>MI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>600.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>318</td>\n",
       "      <td>410</td>\n",
       "      <td>62.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>594.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>306</td>\n",
       "      <td>396</td>\n",
       "      <td>67.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>577.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>362</td>\n",
       "      <td>476</td>\n",
       "      <td>17.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>645.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>372</td>\n",
       "      <td>462</td>\n",
       "      <td>99.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>638.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>378</td>\n",
       "      <td>472</td>\n",
       "      <td>17.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      RR     PR   QRS   QT  QTc  P_wave_axis  QRS_axis  T_wave_axis  ACCI  \\\n",
       "0  600.0  122.0  78.0  318  410         62.0      86.0         61.0     7   \n",
       "1  594.0  124.0  78.0  306  396         67.0      85.0         60.0     7   \n",
       "2  577.0  150.0  92.0  362  476         17.0      46.0         70.0    12   \n",
       "3  645.0  162.0  90.0  372  462         99.0      55.0         82.0    11   \n",
       "4  638.0  200.0  94.0  378  472         17.0      67.0         84.0    12   \n",
       "\n",
       "   sex  Birthyeargroup MI  \n",
       "0    1              21  0  \n",
       "1    1              21  0  \n",
       "2    0               7  0  \n",
       "3    0               7  0  \n",
       "4    0               7  0  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2 = df_1.drop(columns = ['diagdate', 'diagcode', 'diaglocalcode','diagdept', 'personid', 'ethnicity'])\n",
    "col_names = df_2.columns\n",
    "df_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2['Birthyeargroup'].value_counts();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(713447, 12)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    705052\n",
       "1      8395\n",
       "Name: MI, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_2['MI'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x13e69f6d8>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAE7CAYAAACG3xazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAXxUlEQVR4nO3df6zd9X3f8ecrOHQ0GcGBW4/aMCPFS0qyhYAHjjpNWViMIW3MphaBptlDLF4VUjXqpMbpP9agkcg/S4OUeELBw86yUJY2wk0NrucEVdVm4NIwKJDMNyTE9vjhYH4sQQ0jee+P+3FzuDn33nOJ7fvx4fmQjs73+/58Pt/P5/xz9dL3ez73pKqQJElSX96w2AuQJEnSzzKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5qkk1qS7yZ5OclZM+rfSFJJVrbz25L8wRzXOT3JHyb5XpIfJPl2Oz9rtjHHaP3/JslfHs85JJ2cDGmSxsF3gGuOniT5h8Avjjo4yanAXuCdwDrgdOC9wLPAxcd0pZI0IkOapHHwBWDDwPlGYMcCxm8AzgX+RVU9WlU/qapnqurGqtoFkORXktyT5PkkjyT50NHBrf5vB85fdXes3dH7rST72/jPZtqvAP8JeG+7e/f8a/v4ksaRIU3SONgHnN6C1CnA1cB/WcD4fw7cXVU/GNaY5I3AnwJ/DvwS8NvAF5O8fQFz/Brwj4F/BFwFXFZVjwG/BfzPqnpzVZ2xgOtJGnOGNEnj4ujdtA8AjwGHFjD2TODJOdrXAG8Gbqqql6vqa8BXGXjEOoKbqur5qvoe8HXgggWMlfQ6tGSxFyBJx8gXgL8AzmNhjzph+rtnZ8/R/svAgar6yUDtCWD5AuZ4auD4JaZDnyTNyjtpksZCVT3B9AaCK4A/WeDw/w5cluRNs7T/H+CcJIN/M8/lp3frfsirNyr8vQXMXQvoK+l1xJAmaZxcB7y/qn64wHFfAA4Af5zkHUnekOTMJL+f5ArgXqbvfv1ekjcmeR/w68DtbfyDwL9M8otJ3tbWMaqngRVth6kk/S1DmqSxUVXfrqrJ1zDuR0xvHvgmsAd4EbgPOAu4t6peZjqUXQ58H/gcsKGqvtku8WngZaYD13bgiwuY/mvAI8BTSb6/0LVLGl+p8k67JElSb7yTJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktShsfvFgbPOOqtWrly52MuQJEma1wMPPPD9qpoY1jZ2IW3lypVMTi743yRJkiSdcEmemK3Nx52SJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR2aN6QleXuSBwdeLyb5WJK3JtmTZH97X9r6J8nNSaaSPJTkwoFrbWz99yfZOFC/KMnDbczNSdLqQ+eQJEkad/OGtKr6VlVdUFUXABcBLwFfATYDe6tqFbC3nQNcDqxqr03AVpgOXMAW4BLgYmDLQOjaCnx4YNy6Vp9tDkmSpLG20N/uvBT4dlU9kWQ98L5W3w7cA3wcWA/sqKoC9iU5I8nZre+eqjoCkGQPsC7JPcDpVbWv1XcAVwJ3tWsNm0MLtHLzny32EnSS+O5NH1zsJUiSWPh30q4GvtSOl1XVk+34KWBZO14OHBgYc7DV5qofHFKfa45XSbIpyWSSycOHDy/wI0mSJPVn5JCW5FTgQ8B/m9nW7prVMVzXz5hrjqq6papWV9XqiYmJ47kMSZKkE2Ihd9IuB/6qqp5u50+3x5i092da/RBwzsC4Fa02V33FkPpcc0iSJI21hYS0a/jpo06AncDRHZobgTsH6hvaLs81wAvtkeVuYG2SpW3DwFpgd2t7Mcmatqtzw4xrDZtDkiRprI20cSDJm4APAP9uoHwTcEeS64AngKtafRdwBTDF9E7QawGq6kiSG4H7W78bjm4iAD4C3AacxvSGgbvmmUOSJGmsjRTSquqHwJkzas8yvdtzZt8Crp/lOtuAbUPqk8C7htSHziFJkjTu/MUBSZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUMjhbQkZyT5cpJvJnksyXuTvDXJniT72/vS1jdJbk4yleShJBcOXGdj678/ycaB+kVJHm5jbk6SVh86hyRJ0rgb9U7aZ4C7q+odwLuBx4DNwN6qWgXsbecAlwOr2msTsBWmAxewBbgEuBjYMhC6tgIfHhi3rtVnm0OSJGmszRvSkrwF+KfArQBV9XJVPQ+sB7a3btuBK9vxemBHTdsHnJHkbOAyYE9VHamq54A9wLrWdnpV7auqAnbMuNawOSRJksbaKHfSzgMOA/85yTeSfD7Jm4BlVfVk6/MUsKwdLwcODIw/2Gpz1Q8OqTPHHK+SZFOSySSThw8fHuEjSZIk9W2UkLYEuBDYWlXvAX7IjMeO7Q5YHfvljTZHVd1SVauravXExMTxXIYkSdIJMUpIOwgcrKp72/mXmQ5tT7dHlbT3Z1r7IeCcgfErWm2u+oohdeaYQ5IkaazNG9Kq6ingQJK3t9KlwKPATuDoDs2NwJ3teCewoe3yXAO80B5Z7gbWJlnaNgysBXa3theTrGm7OjfMuNawOSRJksbakhH7/TbwxSSnAo8D1zId8O5Ich3wBHBV67sLuAKYAl5qfamqI0luBO5v/W6oqiPt+CPAbcBpwF3tBXDTLHNIkiSNtZFCWlU9CKwe0nTpkL4FXD/LdbYB24bUJ4F3Dak/O2wOSZKkcecvDkiSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHRgppSb6b5OEkDyaZbLW3JtmTZH97X9rqSXJzkqkkDyW5cOA6G1v//Uk2DtQvatefamMz1xySJEnjbiF30v5ZVV1QVavb+WZgb1WtAva2c4DLgVXttQnYCtOBC9gCXAJcDGwZCF1bgQ8PjFs3zxySJElj7ed53Lke2N6OtwNXDtR31LR9wBlJzgYuA/ZU1ZGqeg7YA6xrbadX1b6qKmDHjGsNm0OSJGmsjRrSCvjzJA8k2dRqy6rqyXb8FLCsHS8HDgyMPdhqc9UPDqnPNYckSdJYWzJiv39SVYeS/BKwJ8k3BxurqpLUsV/eaHO04LgJ4Nxzzz2ey5AkSTohRrqTVlWH2vszwFeY/k7Z0+1RJe39mdb9EHDOwPAVrTZXfcWQOnPMMXN9t1TV6qpaPTExMcpHkiRJ6tq8IS3Jm5L83aPHwFrgr4GdwNEdmhuBO9vxTmBD2+W5BnihPbLcDaxNsrRtGFgL7G5tLyZZ03Z1bphxrWFzSJIkjbVRHncuA77S/ivGEuC/VtXdSe4H7khyHfAEcFXrvwu4ApgCXgKuBaiqI0luBO5v/W6oqiPt+CPAbcBpwF3tBXDTLHNIkiSNtXlDWlU9Drx7SP1Z4NIh9QKun+Va24BtQ+qTwLtGnUOSJGnc+YsDkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUodGDmlJTknyjSRfbefnJbk3yVSSP0pyaqv/Qjufau0rB67xiVb/VpLLBurrWm0qyeaB+tA5JEmSxt1C7qT9DvDYwPmngE9X1duA54DrWv064LlW/3TrR5LzgauBdwLrgM+14HcK8FngcuB84JrWd645JEmSxtpIIS3JCuCDwOfbeYD3A19uXbYDV7bj9e2c1n5p678euL2qflRV3wGmgIvba6qqHq+ql4HbgfXzzCFJkjTWRr2T9ofA7wE/aednAs9X1Svt/CCwvB0vBw4AtPYXWv+/rc8YM1t9rjkkSZLG2rwhLcmvAc9U1QMnYD2vSZJNSSaTTB4+fHixlyNJkvRzG+VO2q8CH0ryXaYfRb4f+AxwRpIlrc8K4FA7PgScA9Da3wI8O1ifMWa2+rNzzPEqVXVLVa2uqtUTExMjfCRJkqS+zRvSquoTVbWiqlYy/cX/r1XVvwK+DvxG67YRuLMd72zntPavVVW1+tVt9+d5wCrgPuB+YFXbyXlqm2NnGzPbHJIkSWPt5/k/aR8HfjfJFNPfH7u11W8Fzmz13wU2A1TVI8AdwKPA3cD1VfXj9p2zjwK7md49ekfrO9cckiRJY23J/F1+qqruAe5px48zvTNzZp+/AX5zlvGfBD45pL4L2DWkPnQOSZKkcecvDkiSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdmjekJfk7Se5L8r+SPJLkP7T6eUnuTTKV5I+SnNrqv9DOp1r7yoFrfaLVv5XksoH6ulabSrJ5oD50DkmSpHE3yp20HwHvr6p3AxcA65KsAT4FfLqq3gY8B1zX+l8HPNfqn279SHI+cDXwTmAd8LkkpyQ5BfgscDlwPnBN68scc0iSJI21eUNaTftBO31jexXwfuDLrb4duLIdr2/ntPZLk6TVb6+qH1XVd4Ap4OL2mqqqx6vqZeB2YH0bM9sckiRJY22k76S1O14PAs8Ae4BvA89X1Suty0FgeTteDhwAaO0vAGcO1meMma1+5hxzzFzfpiSTSSYPHz48ykeSJEnq2kghrap+XFUXACuYvvP1juO6qgWqqluqanVVrZ6YmFjs5UiSJP3cFrS7s6qeB74OvBc4I8mS1rQCONSODwHnALT2twDPDtZnjJmt/uwcc0iSJI21UXZ3TiQ5ox2fBnwAeIzpsPYbrdtG4M52vLOd09q/VlXV6le33Z/nAauA+4D7gVVtJ+epTG8u2NnGzDaHJEnSWFsyfxfOBra3XZhvAO6oqq8meRS4PckfAN8Abm39bwW+kGQKOMJ06KKqHklyB/Ao8ApwfVX9GCDJR4HdwCnAtqp6pF3r47PMIUmSNNbmDWlV9RDwniH1x5n+ftrM+t8AvznLtT4JfHJIfRewa9Q5JEmSxp2/OCBJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1aN6QluScJF9P8miSR5L8Tqu/NcmeJPvb+9JWT5Kbk0wleSjJhQPX2tj670+ycaB+UZKH25ibk2SuOSRJksbdKHfSXgH+fVWdD6wBrk9yPrAZ2FtVq4C97RzgcmBVe20CtsJ04AK2AJcAFwNbBkLXVuDDA+PWtfpsc0iSJI21eUNaVT1ZVX/Vjv8v8BiwHFgPbG/dtgNXtuP1wI6atg84I8nZwGXAnqo6UlXPAXuAda3t9KraV1UF7JhxrWFzSJIkjbUFfSctyUrgPcC9wLKqerI1PQUsa8fLgQMDww622lz1g0PqzDHHzHVtSjKZZPLw4cML+UiSJEldGjmkJXkz8MfAx6rqxcG2dgesjvHaXmWuOarqlqpaXVWrJyYmjucyJEmSToiRQlqSNzId0L5YVX/Syk+3R5W092da/RBwzsDwFa02V33FkPpcc0iSJI21UXZ3BrgVeKyq/uNA007g6A7NjcCdA/UNbZfnGuCF9shyN7A2ydK2YWAtsLu1vZhkTZtrw4xrDZtDkiRprC0Zoc+vAv8aeDjJg632+8BNwB1JrgOeAK5qbbuAK4Ap4CXgWoCqOpLkRuD+1u+GqjrSjj8C3AacBtzVXswxhyRJ0libN6RV1V8CmaX50iH9C7h+lmttA7YNqU8C7xpSf3bYHJIkSePOXxyQJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDhnSJEmSOmRIkyRJ6pAhTZIkqUOGNEmSpA4Z0iRJkjpkSJMkSeqQIU2SJKlDhjRJkqQOGdIkSZI6ZEiTJEnqkCFNkiSpQ4Y0SZKkDs0b0pJsS/JMkr8eqL01yZ4k+9v70lZPkpuTTCV5KMmFA2M2tv77k2wcqF+U5OE25uYkmWsOSZKk14NR7qTdBqybUdsM7K2qVcDedg5wObCqvTYBW2E6cAFbgEuAi4EtA6FrK/DhgXHr5plDkiRp7M0b0qrqL4AjM8rrge3teDtw5UB9R03bB5yR5GzgMmBPVR2pqueAPcC61nZ6Ve2rqgJ2zLjWsDkkSZLG3mv9TtqyqnqyHT8FLGvHy4EDA/0Ottpc9YND6nPNIUmSNPZ+7o0D7Q5YHYO1vOY5kmxKMplk8vDhw8dzKZIkSSfEaw1pT7dHlbT3Z1r9EHDOQL8VrTZXfcWQ+lxz/IyquqWqVlfV6omJidf4kSRJkvrxWkPaTuDoDs2NwJ0D9Q1tl+ca4IX2yHI3sDbJ0rZhYC2wu7W9mGRN29W5Yca1hs0hSZI09pbM1yHJl4D3AWclOcj0Ls2bgDuSXAc8AVzVuu8CrgCmgJeAawGq6kiSG4H7W78bquroZoSPML2D9DTgrvZijjkkSZLG3rwhraqumaXp0iF9C7h+lutsA7YNqU8C7xpSf3bYHJIkSa8H/uKAJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1CFDmiRJUocMaZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkdMqRJkiR1yJAmSZLUIUOaJElShwxpkiRJHTKkSZIkdciQJkmS1KEli72A+SRZB3wGOAX4fFXdtMhLkiQ1Kzf/2WIvQSeJ7970wcVewkmn6ztpSU4BPgtcDpwPXJPk/MVdlSRJ0vHXdUgDLgamqurxqnoZuB1Yv8hrkiRJOu56f9y5HDgwcH4QuGRmpySbgE3t9AdJvnUC1qaT31nA9xd7Eb3JpxZ7BdJJz78tQ/i3ZVZ/f7aG3kPaSKrqFuCWxV6HTi5JJqtq9WKvQ9J48W+LjpXeH3ceAs4ZOF/RapIkSWOt95B2P7AqyXlJTgWuBnYu8pokSZKOu64fd1bVK0k+Cuxm+l9wbKuqRxZ5WRofPiKXdDz4t0XHRKpqsdcgSZKkGXp/3ClJkvS6ZEiTJEnqkCFNkiSpQ11vHJCOpSTvYPoXK5a30iFgZ1U9tnirkiRpOO+k6XUhyceZ/lmxAPe1V4AvJdm8mGuTNJ6SXLvYa9DJzd2del1I8r+Bd1bV/5tRPxV4pKpWLc7KJI2rJN+rqnMXex06efm4U68XPwF+GXhiRv3s1iZJC5bkodmagGUnci0aP4Y0vV58DNibZD9woNXOBd4GfHTRViXpZLcMuAx4bkY9wP848cvRODGk6XWhqu5O8g+Ai3n1xoH7q+rHi7cySSe5rwJvrqoHZzYkuefEL0fjxO+kSZIkdcjdnZIkSR0ypEmSJHXIkCZJktQhQ5okSVKHDGmSJEkd+v+KMLhx3/dX0wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_2['MI'].value_counts().plot(kind='bar', title='MI Count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Robust Scaling & SMOTE "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get training data from dataframe\n",
    "train_data = df_2.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the scaler\n",
    "robust_scaler = RobustScaler().fit(train_data)\n",
    "\n",
    "# Scaling test data using robust scaling\n",
    "train_data_robust = robust_scaler.transform(train_data)\n",
    "\n",
    "# Training data features\n",
    "train_features_robust= train_data_robust[:, 0:11]\n",
    "\n",
    "# 'MI' column values\n",
    "train_target_robust = train_data_robust[:, 11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Minority oversampling using SMOTE\n",
    "over = SMOTE(sampling_strategy=0.25) \n",
    "under = RandomUnderSampler(sampling_strategy=0.5)\n",
    "steps = [('o', over), ('u', under)]\n",
    "pipeline = Pipeline(steps=steps)\n",
    "\n",
    "# Transform the dataset\n",
    "x_resampled, y_resampled = pipeline.fit_resample(train_features_robust, train_target_robust)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split 80-20 train vs test data\n",
    "x_train, x_test, y_train, y_test = train_test_split(x_resampled, \n",
    "                                                    y_resampled, \n",
    "                                                    test_size=0.20, \n",
    "                                                    random_state=0,\n",
    "                                                    shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Exporting Data As CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export train data\n",
    "df_train=pd.DataFrame(data=x_train)\n",
    "df_train.insert(11, column= 'labels', value=y_train)\n",
    "df_train.to_csv('train.csv', index=False, header=False)\n",
    "\n",
    "# Export test data\n",
    "df_train=pd.DataFrame(data=x_test)\n",
    "df_train.insert(11, column= 'labels', value=y_test)\n",
    "df_train.to_csv('test.csv', index=False, header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}