pandas_exercise.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pandas 資料清理使用筆記"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 資料缺值處理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mark\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2717: DtypeWarning: Columns (25,51) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('data/NFL Play by Play 2009-2017 (v4).csv.zip', compression='zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(407688, 102)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>GameID</th>\n",
       "      <th>Drive</th>\n",
       "      <th>qtr</th>\n",
       "      <th>down</th>\n",
       "      <th>time</th>\n",
       "      <th>TimeUnder</th>\n",
       "      <th>TimeSecs</th>\n",
       "      <th>PlayTimeDiff</th>\n",
       "      <th>SideofField</th>\n",
       "      <th>...</th>\n",
       "      <th>yacEPA</th>\n",
       "      <th>Home_WP_pre</th>\n",
       "      <th>Away_WP_pre</th>\n",
       "      <th>Home_WP_post</th>\n",
       "      <th>Away_WP_post</th>\n",
       "      <th>Win_Prob</th>\n",
       "      <th>WPA</th>\n",
       "      <th>airWPA</th>\n",
       "      <th>yacWPA</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15:00</td>\n",
       "      <td>15</td>\n",
       "      <td>3600.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.514325</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.060758</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14:53</td>\n",
       "      <td>15</td>\n",
       "      <td>3593.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>1.146076</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.004655</td>\n",
       "      <td>-0.032244</td>\n",
       "      <td>0.036899</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>14:16</td>\n",
       "      <td>15</td>\n",
       "      <td>3556.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>-0.040295</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>13:35</td>\n",
       "      <td>14</td>\n",
       "      <td>3515.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>-5.031425</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>-0.049576</td>\n",
       "      <td>0.106663</td>\n",
       "      <td>-0.156239</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13:27</td>\n",
       "      <td>14</td>\n",
       "      <td>3507.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.558929</td>\n",
       "      <td>0.441071</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.097712</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13:16</td>\n",
       "      <td>14</td>\n",
       "      <td>3496.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.558929</td>\n",
       "      <td>0.441071</td>\n",
       "      <td>0.578453</td>\n",
       "      <td>0.421547</td>\n",
       "      <td>0.441071</td>\n",
       "      <td>-0.019524</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12:40</td>\n",
       "      <td>13</td>\n",
       "      <td>3460.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.163935</td>\n",
       "      <td>0.578453</td>\n",
       "      <td>0.421547</td>\n",
       "      <td>0.582881</td>\n",
       "      <td>0.417119</td>\n",
       "      <td>0.421547</td>\n",
       "      <td>-0.004427</td>\n",
       "      <td>-0.010456</td>\n",
       "      <td>0.006029</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>12:11</td>\n",
       "      <td>13</td>\n",
       "      <td>3431.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.582881</td>\n",
       "      <td>0.417119</td>\n",
       "      <td>0.617544</td>\n",
       "      <td>0.382456</td>\n",
       "      <td>0.417119</td>\n",
       "      <td>-0.034663</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11:34</td>\n",
       "      <td>12</td>\n",
       "      <td>3394.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.617544</td>\n",
       "      <td>0.382456</td>\n",
       "      <td>0.591489</td>\n",
       "      <td>0.408511</td>\n",
       "      <td>0.382456</td>\n",
       "      <td>0.026054</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11:24</td>\n",
       "      <td>12</td>\n",
       "      <td>3384.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.541602</td>\n",
       "      <td>0.591489</td>\n",
       "      <td>0.408511</td>\n",
       "      <td>0.585405</td>\n",
       "      <td>0.414595</td>\n",
       "      <td>0.591489</td>\n",
       "      <td>-0.006084</td>\n",
       "      <td>-0.024526</td>\n",
       "      <td>0.018442</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10:48</td>\n",
       "      <td>11</td>\n",
       "      <td>3348.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.807459</td>\n",
       "      <td>0.585405</td>\n",
       "      <td>0.414595</td>\n",
       "      <td>0.615159</td>\n",
       "      <td>0.384841</td>\n",
       "      <td>0.585405</td>\n",
       "      <td>0.029754</td>\n",
       "      <td>-0.030157</td>\n",
       "      <td>0.059911</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10:21</td>\n",
       "      <td>11</td>\n",
       "      <td>3321.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.615159</td>\n",
       "      <td>0.384841</td>\n",
       "      <td>0.597822</td>\n",
       "      <td>0.402178</td>\n",
       "      <td>0.615159</td>\n",
       "      <td>-0.017337</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>09:43</td>\n",
       "      <td>10</td>\n",
       "      <td>3283.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.229360</td>\n",
       "      <td>0.597822</td>\n",
       "      <td>0.402178</td>\n",
       "      <td>0.607521</td>\n",
       "      <td>0.392479</td>\n",
       "      <td>0.597822</td>\n",
       "      <td>0.009699</td>\n",
       "      <td>-0.031783</td>\n",
       "      <td>0.041482</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>09:01</td>\n",
       "      <td>10</td>\n",
       "      <td>3241.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.607521</td>\n",
       "      <td>0.392479</td>\n",
       "      <td>0.514075</td>\n",
       "      <td>0.485925</td>\n",
       "      <td>0.607521</td>\n",
       "      <td>-0.093445</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>08:25</td>\n",
       "      <td>9</td>\n",
       "      <td>3205.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.514075</td>\n",
       "      <td>0.485925</td>\n",
       "      <td>0.521073</td>\n",
       "      <td>0.478927</td>\n",
       "      <td>0.514075</td>\n",
       "      <td>0.006997</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>08:20</td>\n",
       "      <td>9</td>\n",
       "      <td>3200.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.521073</td>\n",
       "      <td>0.478927</td>\n",
       "      <td>0.399273</td>\n",
       "      <td>0.600727</td>\n",
       "      <td>0.478927</td>\n",
       "      <td>0.121800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>07:57</td>\n",
       "      <td>8</td>\n",
       "      <td>3177.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>0.137531</td>\n",
       "      <td>0.399273</td>\n",
       "      <td>0.600727</td>\n",
       "      <td>0.368797</td>\n",
       "      <td>0.631203</td>\n",
       "      <td>0.600727</td>\n",
       "      <td>0.030476</td>\n",
       "      <td>0.024894</td>\n",
       "      <td>0.005582</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>07:24</td>\n",
       "      <td>8</td>\n",
       "      <td>3144.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.368797</td>\n",
       "      <td>0.631203</td>\n",
       "      <td>0.376194</td>\n",
       "      <td>0.623806</td>\n",
       "      <td>0.631203</td>\n",
       "      <td>-0.007397</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>06:57</td>\n",
       "      <td>7</td>\n",
       "      <td>3117.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.273617</td>\n",
       "      <td>0.376194</td>\n",
       "      <td>0.623806</td>\n",
       "      <td>0.400657</td>\n",
       "      <td>0.599343</td>\n",
       "      <td>0.623806</td>\n",
       "      <td>-0.024463</td>\n",
       "      <td>-0.015287</td>\n",
       "      <td>-0.009176</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>06:52</td>\n",
       "      <td>7</td>\n",
       "      <td>3112.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>0.037212</td>\n",
       "      <td>0.400657</td>\n",
       "      <td>0.599343</td>\n",
       "      <td>0.430491</td>\n",
       "      <td>0.569509</td>\n",
       "      <td>0.599343</td>\n",
       "      <td>-0.029834</td>\n",
       "      <td>-0.030982</td>\n",
       "      <td>0.001148</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>06:48</td>\n",
       "      <td>7</td>\n",
       "      <td>3108.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.430491</td>\n",
       "      <td>0.569509</td>\n",
       "      <td>0.528421</td>\n",
       "      <td>0.471579</td>\n",
       "      <td>0.569509</td>\n",
       "      <td>-0.097930</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>06:44</td>\n",
       "      <td>7</td>\n",
       "      <td>3104.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.528421</td>\n",
       "      <td>0.471579</td>\n",
       "      <td>0.511082</td>\n",
       "      <td>0.488918</td>\n",
       "      <td>0.528421</td>\n",
       "      <td>-0.017339</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>06:09</td>\n",
       "      <td>7</td>\n",
       "      <td>3069.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.511082</td>\n",
       "      <td>0.488918</td>\n",
       "      <td>0.491891</td>\n",
       "      <td>0.508109</td>\n",
       "      <td>0.511082</td>\n",
       "      <td>-0.019192</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>05:27</td>\n",
       "      <td>6</td>\n",
       "      <td>3027.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.491891</td>\n",
       "      <td>0.508109</td>\n",
       "      <td>0.413994</td>\n",
       "      <td>0.586006</td>\n",
       "      <td>0.491891</td>\n",
       "      <td>-0.077897</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>05:02</td>\n",
       "      <td>6</td>\n",
       "      <td>3002.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.413994</td>\n",
       "      <td>0.586006</td>\n",
       "      <td>0.475251</td>\n",
       "      <td>0.524749</td>\n",
       "      <td>0.413994</td>\n",
       "      <td>0.061257</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>04:49</td>\n",
       "      <td>5</td>\n",
       "      <td>2989.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.504306</td>\n",
       "      <td>0.475251</td>\n",
       "      <td>0.524749</td>\n",
       "      <td>0.448998</td>\n",
       "      <td>0.551002</td>\n",
       "      <td>0.524749</td>\n",
       "      <td>0.026253</td>\n",
       "      <td>-0.018135</td>\n",
       "      <td>0.044387</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>04:27</td>\n",
       "      <td>5</td>\n",
       "      <td>2967.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.448998</td>\n",
       "      <td>0.551002</td>\n",
       "      <td>0.470567</td>\n",
       "      <td>0.529433</td>\n",
       "      <td>0.551002</td>\n",
       "      <td>-0.021568</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>04:07</td>\n",
       "      <td>5</td>\n",
       "      <td>2947.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.315907</td>\n",
       "      <td>0.470567</td>\n",
       "      <td>0.529433</td>\n",
       "      <td>0.454633</td>\n",
       "      <td>0.545367</td>\n",
       "      <td>0.529433</td>\n",
       "      <td>0.015933</td>\n",
       "      <td>0.007462</td>\n",
       "      <td>0.008471</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>03:45</td>\n",
       "      <td>4</td>\n",
       "      <td>2925.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.454633</td>\n",
       "      <td>0.545367</td>\n",
       "      <td>0.464664</td>\n",
       "      <td>0.535336</td>\n",
       "      <td>0.545367</td>\n",
       "      <td>-0.010031</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>03:06</td>\n",
       "      <td>4</td>\n",
       "      <td>2886.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.201069</td>\n",
       "      <td>0.464664</td>\n",
       "      <td>0.535336</td>\n",
       "      <td>0.422034</td>\n",
       "      <td>0.577966</td>\n",
       "      <td>0.535336</td>\n",
       "      <td>0.042630</td>\n",
       "      <td>0.038782</td>\n",
       "      <td>0.003848</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>01:38</td>\n",
       "      <td>2</td>\n",
       "      <td>1898.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>0.218866</td>\n",
       "      <td>0.537307</td>\n",
       "      <td>0.462693</td>\n",
       "      <td>0.589619</td>\n",
       "      <td>0.410381</td>\n",
       "      <td>0.537307</td>\n",
       "      <td>0.052312</td>\n",
       "      <td>0.041118</td>\n",
       "      <td>0.011194</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>01:29</td>\n",
       "      <td>2</td>\n",
       "      <td>1889.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>01:29</td>\n",
       "      <td>2</td>\n",
       "      <td>1889.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.589619</td>\n",
       "      <td>0.410381</td>\n",
       "      <td>0.738386</td>\n",
       "      <td>0.261614</td>\n",
       "      <td>0.589619</td>\n",
       "      <td>0.148767</td>\n",
       "      <td>0.148767</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>01:22</td>\n",
       "      <td>2</td>\n",
       "      <td>1882.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.738386</td>\n",
       "      <td>0.261614</td>\n",
       "      <td>0.715750</td>\n",
       "      <td>0.284250</td>\n",
       "      <td>0.738386</td>\n",
       "      <td>-0.022636</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>01:22</td>\n",
       "      <td>2</td>\n",
       "      <td>1882.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.715750</td>\n",
       "      <td>0.284250</td>\n",
       "      <td>0.705351</td>\n",
       "      <td>0.294649</td>\n",
       "      <td>0.284250</td>\n",
       "      <td>0.010400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>01:15</td>\n",
       "      <td>2</td>\n",
       "      <td>1875.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.988674</td>\n",
       "      <td>0.705351</td>\n",
       "      <td>0.294649</td>\n",
       "      <td>0.713310</td>\n",
       "      <td>0.286690</td>\n",
       "      <td>0.294649</td>\n",
       "      <td>-0.007959</td>\n",
       "      <td>0.025911</td>\n",
       "      <td>-0.033870</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>01:09</td>\n",
       "      <td>2</td>\n",
       "      <td>1869.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.845155</td>\n",
       "      <td>0.713310</td>\n",
       "      <td>0.286690</td>\n",
       "      <td>0.617306</td>\n",
       "      <td>0.382694</td>\n",
       "      <td>0.286690</td>\n",
       "      <td>0.096004</td>\n",
       "      <td>0.062619</td>\n",
       "      <td>0.033385</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:54</td>\n",
       "      <td>1</td>\n",
       "      <td>1854.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>00:53</td>\n",
       "      <td>1</td>\n",
       "      <td>1853.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.617306</td>\n",
       "      <td>0.382694</td>\n",
       "      <td>0.499662</td>\n",
       "      <td>0.500338</td>\n",
       "      <td>0.382694</td>\n",
       "      <td>0.117644</td>\n",
       "      <td>0.117644</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:48</td>\n",
       "      <td>1</td>\n",
       "      <td>1848.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.499662</td>\n",
       "      <td>0.500338</td>\n",
       "      <td>0.519191</td>\n",
       "      <td>0.480809</td>\n",
       "      <td>0.500338</td>\n",
       "      <td>-0.019529</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:48</td>\n",
       "      <td>1</td>\n",
       "      <td>1848.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.519191</td>\n",
       "      <td>0.480809</td>\n",
       "      <td>0.525983</td>\n",
       "      <td>0.474017</td>\n",
       "      <td>0.519191</td>\n",
       "      <td>0.006792</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>00:41</td>\n",
       "      <td>1</td>\n",
       "      <td>1841.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>0.108507</td>\n",
       "      <td>0.525983</td>\n",
       "      <td>0.474017</td>\n",
       "      <td>0.527188</td>\n",
       "      <td>0.472812</td>\n",
       "      <td>0.525983</td>\n",
       "      <td>0.001205</td>\n",
       "      <td>-0.002174</td>\n",
       "      <td>0.003380</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>00:35</td>\n",
       "      <td>1</td>\n",
       "      <td>1835.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.527188</td>\n",
       "      <td>0.472812</td>\n",
       "      <td>0.552483</td>\n",
       "      <td>0.447517</td>\n",
       "      <td>0.527188</td>\n",
       "      <td>0.025295</td>\n",
       "      <td>0.025295</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:27</td>\n",
       "      <td>1</td>\n",
       "      <td>1827.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>00:27</td>\n",
       "      <td>1</td>\n",
       "      <td>1827.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.509636</td>\n",
       "      <td>0.552483</td>\n",
       "      <td>0.447517</td>\n",
       "      <td>0.569221</td>\n",
       "      <td>0.430779</td>\n",
       "      <td>0.552483</td>\n",
       "      <td>0.016738</td>\n",
       "      <td>0.033410</td>\n",
       "      <td>-0.016672</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>00:21</td>\n",
       "      <td>1</td>\n",
       "      <td>1821.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002546</td>\n",
       "      <td>0.569221</td>\n",
       "      <td>0.430779</td>\n",
       "      <td>0.560476</td>\n",
       "      <td>0.439524</td>\n",
       "      <td>0.569221</td>\n",
       "      <td>-0.008745</td>\n",
       "      <td>-0.008820</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>00:16</td>\n",
       "      <td>1</td>\n",
       "      <td>1816.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.560476</td>\n",
       "      <td>0.439524</td>\n",
       "      <td>0.539795</td>\n",
       "      <td>0.460205</td>\n",
       "      <td>0.560476</td>\n",
       "      <td>-0.020682</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:07</td>\n",
       "      <td>1</td>\n",
       "      <td>1807.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>00:07</td>\n",
       "      <td>1</td>\n",
       "      <td>1807.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.539795</td>\n",
       "      <td>0.460205</td>\n",
       "      <td>0.532696</td>\n",
       "      <td>0.467304</td>\n",
       "      <td>0.539795</td>\n",
       "      <td>-0.007099</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>00:01</td>\n",
       "      <td>1</td>\n",
       "      <td>1801.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>-6.235342</td>\n",
       "      <td>0.532696</td>\n",
       "      <td>0.467304</td>\n",
       "      <td>0.532696</td>\n",
       "      <td>0.467304</td>\n",
       "      <td>0.532696</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.532696</td>\n",
       "      <td>0.532696</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15:00</td>\n",
       "      <td>15</td>\n",
       "      <td>1800.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.497962</td>\n",
       "      <td>0.502038</td>\n",
       "      <td>0.494973</td>\n",
       "      <td>0.505027</td>\n",
       "      <td>0.502038</td>\n",
       "      <td>0.002988</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14:55</td>\n",
       "      <td>15</td>\n",
       "      <td>1795.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.494973</td>\n",
       "      <td>0.505027</td>\n",
       "      <td>0.478256</td>\n",
       "      <td>0.521744</td>\n",
       "      <td>0.505027</td>\n",
       "      <td>0.016717</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>14:20</td>\n",
       "      <td>15</td>\n",
       "      <td>1760.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.478256</td>\n",
       "      <td>0.521744</td>\n",
       "      <td>0.427334</td>\n",
       "      <td>0.572666</td>\n",
       "      <td>0.521744</td>\n",
       "      <td>0.050923</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13:38</td>\n",
       "      <td>14</td>\n",
       "      <td>1718.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.427334</td>\n",
       "      <td>0.572666</td>\n",
       "      <td>0.425697</td>\n",
       "      <td>0.574303</td>\n",
       "      <td>0.572666</td>\n",
       "      <td>0.001637</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13:04</td>\n",
       "      <td>14</td>\n",
       "      <td>1684.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.204002</td>\n",
       "      <td>0.425697</td>\n",
       "      <td>0.574303</td>\n",
       "      <td>0.398391</td>\n",
       "      <td>0.601609</td>\n",
       "      <td>0.574303</td>\n",
       "      <td>0.027305</td>\n",
       "      <td>-0.013627</td>\n",
       "      <td>0.040932</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12:41</td>\n",
       "      <td>13</td>\n",
       "      <td>1661.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.398391</td>\n",
       "      <td>0.601609</td>\n",
       "      <td>0.417641</td>\n",
       "      <td>0.582359</td>\n",
       "      <td>0.601609</td>\n",
       "      <td>-0.019249</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12:06</td>\n",
       "      <td>13</td>\n",
       "      <td>1626.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.417641</td>\n",
       "      <td>0.582359</td>\n",
       "      <td>0.574091</td>\n",
       "      <td>0.425909</td>\n",
       "      <td>0.582359</td>\n",
       "      <td>-0.156450</td>\n",
       "      <td>-0.156450</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11:57</td>\n",
       "      <td>12</td>\n",
       "      <td>1617.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>PIT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.574091</td>\n",
       "      <td>0.425909</td>\n",
       "      <td>0.596120</td>\n",
       "      <td>0.403880</td>\n",
       "      <td>0.574091</td>\n",
       "      <td>0.022029</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>11:20</td>\n",
       "      <td>12</td>\n",
       "      <td>1580.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.596120</td>\n",
       "      <td>0.403880</td>\n",
       "      <td>0.587230</td>\n",
       "      <td>0.412770</td>\n",
       "      <td>0.596120</td>\n",
       "      <td>-0.008890</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10:37</td>\n",
       "      <td>11</td>\n",
       "      <td>1537.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>TEN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.587230</td>\n",
       "      <td>0.412770</td>\n",
       "      <td>0.515547</td>\n",
       "      <td>0.484453</td>\n",
       "      <td>0.587230</td>\n",
       "      <td>-0.071683</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Date      GameID  Drive  qtr  down   time  TimeUnder  TimeSecs  \\\n",
       "0   2009-09-10  2009091000      1    1   NaN  15:00         15    3600.0   \n",
       "1   2009-09-10  2009091000      1    1   1.0  14:53         15    3593.0   \n",
       "2   2009-09-10  2009091000      1    1   2.0  14:16         15    3556.0   \n",
       "3   2009-09-10  2009091000      1    1   3.0  13:35         14    3515.0   \n",
       "4   2009-09-10  2009091000      1    1   4.0  13:27         14    3507.0   \n",
       "5   2009-09-10  2009091000      2    1   1.0  13:16         14    3496.0   \n",
       "6   2009-09-10  2009091000      2    1   2.0  12:40         13    3460.0   \n",
       "7   2009-09-10  2009091000      2    1   3.0  12:11         13    3431.0   \n",
       "8   2009-09-10  2009091000      2    1   4.0  11:34         12    3394.0   \n",
       "9   2009-09-10  2009091000      3    1   1.0  11:24         12    3384.0   \n",
       "10  2009-09-10  2009091000      3    1   2.0  10:48         11    3348.0   \n",
       "11  2009-09-10  2009091000      3    1   1.0  10:21         11    3321.0   \n",
       "12  2009-09-10  2009091000      3    1   2.0  09:43         10    3283.0   \n",
       "13  2009-09-10  2009091000      3    1   3.0  09:01         10    3241.0   \n",
       "14  2009-09-10  2009091000      3    1   4.0  08:25          9    3205.0   \n",
       "15  2009-09-10  2009091000      4    1   1.0  08:20          9    3200.0   \n",
       "16  2009-09-10  2009091000      4    1   1.0  07:57          8    3177.0   \n",
       "17  2009-09-10  2009091000      4    1   1.0  07:24          8    3144.0   \n",
       "18  2009-09-10  2009091000      4    1   2.0  06:57          7    3117.0   \n",
       "19  2009-09-10  2009091000      4    1   3.0  06:52          7    3112.0   \n",
       "20  2009-09-10  2009091000      4    1   4.0  06:48          7    3108.0   \n",
       "21  2009-09-10  2009091000      5    1   1.0  06:44          7    3104.0   \n",
       "22  2009-09-10  2009091000      5    1   2.0  06:09          7    3069.0   \n",
       "23  2009-09-10  2009091000      5    1   3.0  05:27          6    3027.0   \n",
       "24  2009-09-10  2009091000      5    1   4.0  05:02          6    3002.0   \n",
       "25  2009-09-10  2009091000      6    1   1.0  04:49          5    2989.0   \n",
       "26  2009-09-10  2009091000      6    1   1.0  04:27          5    2967.0   \n",
       "27  2009-09-10  2009091000      6    1   1.0  04:07          5    2947.0   \n",
       "28  2009-09-10  2009091000      6    1   2.0  03:45          4    2925.0   \n",
       "29  2009-09-10  2009091000      6    1   3.0  03:06          4    2886.0   \n",
       "..         ...         ...    ...  ...   ...    ...        ...       ...   \n",
       "70  2009-09-10  2009091000     11    2   2.0  01:38          2    1898.0   \n",
       "71  2009-09-10  2009091000     11    2   NaN  01:29          2    1889.0   \n",
       "72  2009-09-10  2009091000     11    2   1.0  01:29          2    1889.0   \n",
       "73  2009-09-10  2009091000     11    2   NaN  01:22          2    1882.0   \n",
       "74  2009-09-10  2009091000     12    2   NaN  01:22          2    1882.0   \n",
       "75  2009-09-10  2009091000     12    2   1.0  01:15          2    1875.0   \n",
       "76  2009-09-10  2009091000     12    2   2.0  01:09          2    1869.0   \n",
       "77  2009-09-10  2009091000     12    2   NaN  00:54          1    1854.0   \n",
       "78  2009-09-10  2009091000     12    2   1.0  00:53          1    1853.0   \n",
       "79  2009-09-10  2009091000     12    2   NaN  00:48          1    1848.0   \n",
       "80  2009-09-10  2009091000     13    2   NaN  00:48          1    1848.0   \n",
       "81  2009-09-10  2009091000     13    2   1.0  00:41          1    1841.0   \n",
       "82  2009-09-10  2009091000     13    2   2.0  00:35          1    1835.0   \n",
       "83  2009-09-10  2009091000     13    2   NaN  00:27          1    1827.0   \n",
       "84  2009-09-10  2009091000     13    2   1.0  00:27          1    1827.0   \n",
       "85  2009-09-10  2009091000     13    2   1.0  00:21          1    1821.0   \n",
       "86  2009-09-10  2009091000     13    2   2.0  00:16          1    1816.0   \n",
       "87  2009-09-10  2009091000     13    2   NaN  00:07          1    1807.0   \n",
       "88  2009-09-10  2009091000     13    2   3.0  00:07          1    1807.0   \n",
       "89  2009-09-10  2009091000     13    2   3.0  00:01          1    1801.0   \n",
       "90  2009-09-10  2009091000     14    3   NaN  15:00         15    1800.0   \n",
       "91  2009-09-10  2009091000     14    3   1.0  14:55         15    1795.0   \n",
       "92  2009-09-10  2009091000     14    3   2.0  14:20         15    1760.0   \n",
       "93  2009-09-10  2009091000     14    3   1.0  13:38         14    1718.0   \n",
       "94  2009-09-10  2009091000     14    3   2.0  13:04         14    1684.0   \n",
       "95  2009-09-10  2009091000     14    3   1.0  12:41         13    1661.0   \n",
       "96  2009-09-10  2009091000     14    3   2.0  12:06         13    1626.0   \n",
       "97  2009-09-10  2009091000     15    3   1.0  11:57         12    1617.0   \n",
       "98  2009-09-10  2009091000     15    3   2.0  11:20         12    1580.0   \n",
       "99  2009-09-10  2009091000     15    3   3.0  10:37         11    1537.0   \n",
       "\n",
       "    PlayTimeDiff SideofField   ...      yacEPA  Home_WP_pre  Away_WP_pre  \\\n",
       "0            0.0         TEN   ...         NaN     0.485675     0.514325   \n",
       "1            7.0         PIT   ...    1.146076     0.546433     0.453567   \n",
       "2           37.0         PIT   ...         NaN     0.551088     0.448912   \n",
       "3           41.0         PIT   ...   -5.031425     0.510793     0.489207   \n",
       "4            8.0         PIT   ...         NaN     0.461217     0.538783   \n",
       "5           11.0         TEN   ...         NaN     0.558929     0.441071   \n",
       "6           36.0         TEN   ...    0.163935     0.578453     0.421547   \n",
       "7           29.0         TEN   ...         NaN     0.582881     0.417119   \n",
       "8           37.0         TEN   ...         NaN     0.617544     0.382456   \n",
       "9           10.0         TEN   ...    0.541602     0.591489     0.408511   \n",
       "10          36.0         TEN   ...    1.807459     0.585405     0.414595   \n",
       "11          27.0         TEN   ...         NaN     0.615159     0.384841   \n",
       "12          38.0         TEN   ...    1.229360     0.597822     0.402178   \n",
       "13          42.0         TEN   ...         NaN     0.607521     0.392479   \n",
       "14          36.0         TEN   ...         NaN     0.514075     0.485925   \n",
       "15           5.0         TEN   ...         NaN     0.521073     0.478927   \n",
       "16          23.0         PIT   ...    0.137531     0.399273     0.600727   \n",
       "17          33.0         PIT   ...         NaN     0.368797     0.631203   \n",
       "18          27.0         PIT   ...   -0.273617     0.376194     0.623806   \n",
       "19           5.0         PIT   ...    0.037212     0.400657     0.599343   \n",
       "20           4.0         PIT   ...         NaN     0.430491     0.569509   \n",
       "21           4.0         PIT   ...         NaN     0.528421     0.471579   \n",
       "22          35.0         PIT   ...         NaN     0.511082     0.488918   \n",
       "23          42.0         PIT   ...         NaN     0.491891     0.508109   \n",
       "24          25.0         PIT   ...         NaN     0.413994     0.586006   \n",
       "25          13.0         TEN   ...    1.504306     0.475251     0.524749   \n",
       "26          22.0         TEN   ...         NaN     0.448998     0.551002   \n",
       "27          20.0         TEN   ...    0.315907     0.470567     0.529433   \n",
       "28          22.0         TEN   ...         NaN     0.454633     0.545367   \n",
       "29          39.0         TEN   ...    0.201069     0.464664     0.535336   \n",
       "..           ...         ...   ...         ...          ...          ...   \n",
       "70          22.0         PIT   ...    0.218866     0.537307     0.462693   \n",
       "71           9.0         PIT   ...         NaN          NaN          NaN   \n",
       "72           0.0         TEN   ...    0.000000     0.589619     0.410381   \n",
       "73           7.0         TEN   ...         NaN     0.738386     0.261614   \n",
       "74           0.0         PIT   ...         NaN     0.715750     0.284250   \n",
       "75           7.0         TEN   ...   -0.988674     0.705351     0.294649   \n",
       "76           6.0         TEN   ...    0.845155     0.713310     0.286690   \n",
       "77          15.0         TEN   ...         NaN          NaN          NaN   \n",
       "78           1.0         PIT   ...    0.000000     0.617306     0.382694   \n",
       "79           5.0         PIT   ...         NaN     0.499662     0.500338   \n",
       "80           0.0         TEN   ...         NaN     0.519191     0.480809   \n",
       "81           7.0         PIT   ...    0.108507     0.525983     0.474017   \n",
       "82           6.0         PIT   ...    0.000000     0.527188     0.472812   \n",
       "83           8.0         PIT   ...         NaN          NaN          NaN   \n",
       "84           0.0         PIT   ...   -0.509636     0.552483     0.447517   \n",
       "85           6.0         TEN   ...    0.002546     0.569221     0.430779   \n",
       "86           5.0         TEN   ...         NaN     0.560476     0.439524   \n",
       "87           9.0         TEN   ...         NaN          NaN          NaN   \n",
       "88           0.0         PIT   ...         NaN     0.539795     0.460205   \n",
       "89           6.0         PIT   ...   -6.235342     0.532696     0.467304   \n",
       "90           1.0         PIT   ...         NaN     0.497962     0.502038   \n",
       "91           5.0         TEN   ...         NaN     0.494973     0.505027   \n",
       "92          35.0         TEN   ...         NaN     0.478256     0.521744   \n",
       "93          42.0         TEN   ...         NaN     0.427334     0.572666   \n",
       "94          34.0         TEN   ...    1.204002     0.425697     0.574303   \n",
       "95          23.0         PIT   ...         NaN     0.398391     0.601609   \n",
       "96          35.0         PIT   ...    0.000000     0.417641     0.582359   \n",
       "97           9.0         PIT   ...         NaN     0.574091     0.425909   \n",
       "98          37.0         TEN   ...         NaN     0.596120     0.403880   \n",
       "99          43.0         TEN   ...         NaN     0.587230     0.412770   \n",
       "\n",
       "    Home_WP_post  Away_WP_post  Win_Prob       WPA    airWPA    yacWPA  Season  \n",
       "0       0.546433      0.453567  0.485675  0.060758       NaN       NaN    2009  \n",
       "1       0.551088      0.448912  0.546433  0.004655 -0.032244  0.036899    2009  \n",
       "2       0.510793      0.489207  0.551088 -0.040295       NaN       NaN    2009  \n",
       "3       0.461217      0.538783  0.510793 -0.049576  0.106663 -0.156239    2009  \n",
       "4       0.558929      0.441071  0.461217  0.097712       NaN       NaN    2009  \n",
       "5       0.578453      0.421547  0.441071 -0.019524       NaN       NaN    2009  \n",
       "6       0.582881      0.417119  0.421547 -0.004427 -0.010456  0.006029    2009  \n",
       "7       0.617544      0.382456  0.417119 -0.034663       NaN       NaN    2009  \n",
       "8       0.591489      0.408511  0.382456  0.026054       NaN       NaN    2009  \n",
       "9       0.585405      0.414595  0.591489 -0.006084 -0.024526  0.018442    2009  \n",
       "10      0.615159      0.384841  0.585405  0.029754 -0.030157  0.059911    2009  \n",
       "11      0.597822      0.402178  0.615159 -0.017337       NaN       NaN    2009  \n",
       "12      0.607521      0.392479  0.597822  0.009699 -0.031783  0.041482    2009  \n",
       "13      0.514075      0.485925  0.607521 -0.093445       NaN       NaN    2009  \n",
       "14      0.521073      0.478927  0.514075  0.006997       NaN       NaN    2009  \n",
       "15      0.399273      0.600727  0.478927  0.121800       NaN       NaN    2009  \n",
       "16      0.368797      0.631203  0.600727  0.030476  0.024894  0.005582    2009  \n",
       "17      0.376194      0.623806  0.631203 -0.007397       NaN       NaN    2009  \n",
       "18      0.400657      0.599343  0.623806 -0.024463 -0.015287 -0.009176    2009  \n",
       "19      0.430491      0.569509  0.599343 -0.029834 -0.030982  0.001148    2009  \n",
       "20      0.528421      0.471579  0.569509 -0.097930       NaN       NaN    2009  \n",
       "21      0.511082      0.488918  0.528421 -0.017339       NaN       NaN    2009  \n",
       "22      0.491891      0.508109  0.511082 -0.019192       NaN       NaN    2009  \n",
       "23      0.413994      0.586006  0.491891 -0.077897       NaN       NaN    2009  \n",
       "24      0.475251      0.524749  0.413994  0.061257       NaN       NaN    2009  \n",
       "25      0.448998      0.551002  0.524749  0.026253 -0.018135  0.044387    2009  \n",
       "26      0.470567      0.529433  0.551002 -0.021568       NaN       NaN    2009  \n",
       "27      0.454633      0.545367  0.529433  0.015933  0.007462  0.008471    2009  \n",
       "28      0.464664      0.535336  0.545367 -0.010031       NaN       NaN    2009  \n",
       "29      0.422034      0.577966  0.535336  0.042630  0.038782  0.003848    2009  \n",
       "..           ...           ...       ...       ...       ...       ...     ...  \n",
       "70      0.589619      0.410381  0.537307  0.052312  0.041118  0.011194    2009  \n",
       "71           NaN           NaN       NaN  0.000000       NaN       NaN    2009  \n",
       "72      0.738386      0.261614  0.589619  0.148767  0.148767  0.000000    2009  \n",
       "73      0.715750      0.284250  0.738386 -0.022636       NaN       NaN    2009  \n",
       "74      0.705351      0.294649  0.284250  0.010400       NaN       NaN    2009  \n",
       "75      0.713310      0.286690  0.294649 -0.007959  0.025911 -0.033870    2009  \n",
       "76      0.617306      0.382694  0.286690  0.096004  0.062619  0.033385    2009  \n",
       "77           NaN           NaN       NaN  0.000000       NaN       NaN    2009  \n",
       "78      0.499662      0.500338  0.382694  0.117644  0.117644  0.000000    2009  \n",
       "79      0.519191      0.480809  0.500338 -0.019529       NaN       NaN    2009  \n",
       "80      0.525983      0.474017  0.519191  0.006792       NaN       NaN    2009  \n",
       "81      0.527188      0.472812  0.525983  0.001205 -0.002174  0.003380    2009  \n",
       "82      0.552483      0.447517  0.527188  0.025295  0.025295  0.000000    2009  \n",
       "83           NaN           NaN       NaN  0.000000       NaN       NaN    2009  \n",
       "84      0.569221      0.430779  0.552483  0.016738  0.033410 -0.016672    2009  \n",
       "85      0.560476      0.439524  0.569221 -0.008745 -0.008820  0.000076    2009  \n",
       "86      0.539795      0.460205  0.560476 -0.020682       NaN       NaN    2009  \n",
       "87           NaN           NaN       NaN  0.000000       NaN       NaN    2009  \n",
       "88      0.532696      0.467304  0.539795 -0.007099       NaN       NaN    2009  \n",
       "89      0.532696      0.467304  0.532696  0.000000 -0.532696  0.532696    2009  \n",
       "90      0.494973      0.505027  0.502038  0.002988       NaN       NaN    2009  \n",
       "91      0.478256      0.521744  0.505027  0.016717       NaN       NaN    2009  \n",
       "92      0.427334      0.572666  0.521744  0.050923       NaN       NaN    2009  \n",
       "93      0.425697      0.574303  0.572666  0.001637       NaN       NaN    2009  \n",
       "94      0.398391      0.601609  0.574303  0.027305 -0.013627  0.040932    2009  \n",
       "95      0.417641      0.582359  0.601609 -0.019249       NaN       NaN    2009  \n",
       "96      0.574091      0.425909  0.582359 -0.156450 -0.156450  0.000000    2009  \n",
       "97      0.596120      0.403880  0.574091  0.022029       NaN       NaN    2009  \n",
       "98      0.587230      0.412770  0.596120 -0.008890       NaN       NaN    2009  \n",
       "99      0.515547      0.484453  0.587230 -0.071683       NaN       NaN    2009  \n",
       "\n",
       "[100 rows x 102 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 統計所有欄位的 NA 值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Date                0\n",
       "GameID              0\n",
       "Drive               0\n",
       "qtr                 0\n",
       "down            61154\n",
       "time              224\n",
       "TimeUnder           0\n",
       "TimeSecs          224\n",
       "PlayTimeDiff      444\n",
       "SideofField       528\n",
       "dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the number of missing data points per column\n",
    "missing_values_count = df.isnull().sum()\n",
    "missing_values_count[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 去除 NA 值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>GameID</th>\n",
       "      <th>Drive</th>\n",
       "      <th>qtr</th>\n",
       "      <th>down</th>\n",
       "      <th>time</th>\n",
       "      <th>TimeUnder</th>\n",
       "      <th>TimeSecs</th>\n",
       "      <th>PlayTimeDiff</th>\n",
       "      <th>SideofField</th>\n",
       "      <th>...</th>\n",
       "      <th>yacEPA</th>\n",
       "      <th>Home_WP_pre</th>\n",
       "      <th>Away_WP_pre</th>\n",
       "      <th>Home_WP_post</th>\n",
       "      <th>Away_WP_post</th>\n",
       "      <th>Win_Prob</th>\n",
       "      <th>WPA</th>\n",
       "      <th>airWPA</th>\n",
       "      <th>yacWPA</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Date, GameID, Drive, qtr, down, time, TimeUnder, TimeSecs, PlayTimeDiff, SideofField, yrdln, yrdline100, ydstogo, ydsnet, GoalToGo, FirstDown, posteam, DefensiveTeam, desc, PlayAttempted, Yards.Gained, sp, Touchdown, ExPointResult, TwoPointConv, DefTwoPoint, Safety, Onsidekick, PuntResult, PlayType, Passer, Passer_ID, PassAttempt, PassOutcome, PassLength, AirYards, YardsAfterCatch, QBHit, PassLocation, InterceptionThrown, Interceptor, Rusher, Rusher_ID, RushAttempt, RunLocation, RunGap, Receiver, Receiver_ID, Reception, ReturnResult, Returner, BlockingPlayer, Tackler1, Tackler2, FieldGoalResult, FieldGoalDistance, Fumble, RecFumbTeam, RecFumbPlayer, Sack, Challenge.Replay, ChalReplayResult, Accepted.Penalty, PenalizedTeam, PenaltyType, PenalizedPlayer, Penalty.Yards, PosTeamScore, DefTeamScore, ScoreDiff, AbsScoreDiff, HomeTeam, AwayTeam, Timeout_Indicator, Timeout_Team, posteam_timeouts_pre, HomeTimeouts_Remaining_Pre, AwayTimeouts_Remaining_Pre, HomeTimeouts_Remaining_Post, AwayTimeouts_Remaining_Post, No_Score_Prob, Opp_Field_Goal_Prob, Opp_Safety_Prob, Opp_Touchdown_Prob, Field_Goal_Prob, Safety_Prob, Touchdown_Prob, ExPoint_Prob, TwoPoint_Prob, ExpPts, EPA, airEPA, yacEPA, Home_WP_pre, Away_WP_pre, Home_WP_post, Away_WP_post, Win_Prob, WPA, airWPA, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 102 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove all the rows that contain a missing value\n",
    "df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>GameID</th>\n",
       "      <th>Drive</th>\n",
       "      <th>qtr</th>\n",
       "      <th>TimeUnder</th>\n",
       "      <th>ydstogo</th>\n",
       "      <th>ydsnet</th>\n",
       "      <th>PlayAttempted</th>\n",
       "      <th>Yards.Gained</th>\n",
       "      <th>sp</th>\n",
       "      <th>...</th>\n",
       "      <th>Timeout_Indicator</th>\n",
       "      <th>Timeout_Team</th>\n",
       "      <th>posteam_timeouts_pre</th>\n",
       "      <th>HomeTimeouts_Remaining_Pre</th>\n",
       "      <th>AwayTimeouts_Remaining_Pre</th>\n",
       "      <th>HomeTimeouts_Remaining_Post</th>\n",
       "      <th>AwayTimeouts_Remaining_Post</th>\n",
       "      <th>ExPoint_Prob</th>\n",
       "      <th>TwoPoint_Prob</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-3</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2009-09-10</td>\n",
       "      <td>2009091000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date      GameID  Drive  qtr  TimeUnder  ydstogo  ydsnet  \\\n",
       "0  2009-09-10  2009091000      1    1         15        0       0   \n",
       "1  2009-09-10  2009091000      1    1         15       10       5   \n",
       "2  2009-09-10  2009091000      1    1         15        5       2   \n",
       "3  2009-09-10  2009091000      1    1         14        8       2   \n",
       "4  2009-09-10  2009091000      1    1         14        8       2   \n",
       "\n",
       "   PlayAttempted  Yards.Gained  sp   ...    Timeout_Indicator  Timeout_Team  \\\n",
       "0              1            39   0   ...                    0          None   \n",
       "1              1             5   0   ...                    0          None   \n",
       "2              1            -3   0   ...                    0          None   \n",
       "3              1             0   0   ...                    0          None   \n",
       "4              1             0   0   ...                    0          None   \n",
       "\n",
       "   posteam_timeouts_pre HomeTimeouts_Remaining_Pre AwayTimeouts_Remaining_Pre  \\\n",
       "0                     3                          3                          3   \n",
       "1                     3                          3                          3   \n",
       "2                     3                          3                          3   \n",
       "3                     3                          3                          3   \n",
       "4                     3                          3                          3   \n",
       "\n",
       "   HomeTimeouts_Remaining_Post  AwayTimeouts_Remaining_Post  ExPoint_Prob  \\\n",
       "0                            3                            3           0.0   \n",
       "1                            3                            3           0.0   \n",
       "2                            3                            3           0.0   \n",
       "3                            3                            3           0.0   \n",
       "4                            3                            3           0.0   \n",
       "\n",
       "   TwoPoint_Prob  Season  \n",
       "0            0.0    2009  \n",
       "1            0.0    2009  \n",
       "2            0.0    2009  \n",
       "3            0.0    2009  \n",
       "4            0.0    2009  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove all columns with at least one missing value\n",
    "columns_with_na_dropped = df.dropna(axis=1)\n",
    "columns_with_na_dropped.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns in original dataset: 102 \n",
      "\n",
      "Columns with na's dropped: 41\n"
     ]
    }
   ],
   "source": [
    "# just how much data did we lose?\n",
    "print(\"Columns in original dataset: %d \\n\" % df.shape[1])\n",
    "print(\"Columns with na's dropped: %d\" % columns_with_na_dropped.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 資料自動填補值的操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EPA</th>\n",
       "      <th>airEPA</th>\n",
       "      <th>yacEPA</th>\n",
       "      <th>Home_WP_pre</th>\n",
       "      <th>Away_WP_pre</th>\n",
       "      <th>Home_WP_post</th>\n",
       "      <th>Away_WP_post</th>\n",
       "      <th>Win_Prob</th>\n",
       "      <th>WPA</th>\n",
       "      <th>airWPA</th>\n",
       "      <th>yacWPA</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.014474</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.514325</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.060758</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.077907</td>\n",
       "      <td>-1.068169</td>\n",
       "      <td>1.146076</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.004655</td>\n",
       "      <td>-0.032244</td>\n",
       "      <td>0.036899</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.402760</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>-0.040295</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.712583</td>\n",
       "      <td>3.318841</td>\n",
       "      <td>-5.031425</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>-0.049576</td>\n",
       "      <td>0.106663</td>\n",
       "      <td>-0.156239</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.097796</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.558929</td>\n",
       "      <td>0.441071</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.097712</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        EPA    airEPA    yacEPA  Home_WP_pre  Away_WP_pre  Home_WP_post  \\\n",
       "0  2.014474       NaN       NaN     0.485675     0.514325      0.546433   \n",
       "1  0.077907 -1.068169  1.146076     0.546433     0.453567      0.551088   \n",
       "2 -1.402760       NaN       NaN     0.551088     0.448912      0.510793   \n",
       "3 -1.712583  3.318841 -5.031425     0.510793     0.489207      0.461217   \n",
       "4  2.097796       NaN       NaN     0.461217     0.538783      0.558929   \n",
       "\n",
       "   Away_WP_post  Win_Prob       WPA    airWPA    yacWPA  Season  \n",
       "0      0.453567  0.485675  0.060758       NaN       NaN    2009  \n",
       "1      0.448912  0.546433  0.004655 -0.032244  0.036899    2009  \n",
       "2      0.489207  0.551088 -0.040295       NaN       NaN    2009  \n",
       "3      0.538783  0.510793 -0.049576  0.106663 -0.156239    2009  \n",
       "4      0.441071  0.461217  0.097712       NaN       NaN    2009  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get a small subset of the NFL dataset\n",
    "subset_df = df.loc[:, 'EPA':'Season'].head()\n",
    "subset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EPA</th>\n",
       "      <th>airEPA</th>\n",
       "      <th>yacEPA</th>\n",
       "      <th>Home_WP_pre</th>\n",
       "      <th>Away_WP_pre</th>\n",
       "      <th>Home_WP_post</th>\n",
       "      <th>Away_WP_post</th>\n",
       "      <th>Win_Prob</th>\n",
       "      <th>WPA</th>\n",
       "      <th>airWPA</th>\n",
       "      <th>yacWPA</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.014474</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.514325</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.060758</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.077907</td>\n",
       "      <td>-1.068169</td>\n",
       "      <td>1.146076</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.004655</td>\n",
       "      <td>-0.032244</td>\n",
       "      <td>0.036899</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.402760</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>-0.040295</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.712583</td>\n",
       "      <td>3.318841</td>\n",
       "      <td>-5.031425</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>-0.049576</td>\n",
       "      <td>0.106663</td>\n",
       "      <td>-0.156239</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.097796</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.558929</td>\n",
       "      <td>0.441071</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.097712</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        EPA    airEPA    yacEPA  Home_WP_pre  Away_WP_pre  Home_WP_post  \\\n",
       "0  2.014474  0.000000  0.000000     0.485675     0.514325      0.546433   \n",
       "1  0.077907 -1.068169  1.146076     0.546433     0.453567      0.551088   \n",
       "2 -1.402760  0.000000  0.000000     0.551088     0.448912      0.510793   \n",
       "3 -1.712583  3.318841 -5.031425     0.510793     0.489207      0.461217   \n",
       "4  2.097796  0.000000  0.000000     0.461217     0.538783      0.558929   \n",
       "\n",
       "   Away_WP_post  Win_Prob       WPA    airWPA    yacWPA  Season  \n",
       "0      0.453567  0.485675  0.060758  0.000000  0.000000    2009  \n",
       "1      0.448912  0.546433  0.004655 -0.032244  0.036899    2009  \n",
       "2      0.489207  0.551088 -0.040295  0.000000  0.000000    2009  \n",
       "3      0.538783  0.510793 -0.049576  0.106663 -0.156239    2009  \n",
       "4      0.441071  0.461217  0.097712  0.000000  0.000000    2009  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace all NA's with 0\n",
    "subset_df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EPA</th>\n",
       "      <th>airEPA</th>\n",
       "      <th>yacEPA</th>\n",
       "      <th>Home_WP_pre</th>\n",
       "      <th>Away_WP_pre</th>\n",
       "      <th>Home_WP_post</th>\n",
       "      <th>Away_WP_post</th>\n",
       "      <th>Win_Prob</th>\n",
       "      <th>WPA</th>\n",
       "      <th>airWPA</th>\n",
       "      <th>yacWPA</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.014474</td>\n",
       "      <td>-1.068169</td>\n",
       "      <td>1.146076</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.514325</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.485675</td>\n",
       "      <td>0.060758</td>\n",
       "      <td>-0.032244</td>\n",
       "      <td>0.036899</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.077907</td>\n",
       "      <td>-1.068169</td>\n",
       "      <td>1.146076</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.453567</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.546433</td>\n",
       "      <td>0.004655</td>\n",
       "      <td>-0.032244</td>\n",
       "      <td>0.036899</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.402760</td>\n",
       "      <td>3.318841</td>\n",
       "      <td>-5.031425</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>0.448912</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.551088</td>\n",
       "      <td>-0.040295</td>\n",
       "      <td>0.106663</td>\n",
       "      <td>-0.156239</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.712583</td>\n",
       "      <td>3.318841</td>\n",
       "      <td>-5.031425</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>0.489207</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.510793</td>\n",
       "      <td>-0.049576</td>\n",
       "      <td>0.106663</td>\n",
       "      <td>-0.156239</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.097796</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.538783</td>\n",
       "      <td>0.558929</td>\n",
       "      <td>0.441071</td>\n",
       "      <td>0.461217</td>\n",
       "      <td>0.097712</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        EPA    airEPA    yacEPA  Home_WP_pre  Away_WP_pre  Home_WP_post  \\\n",
       "0  2.014474 -1.068169  1.146076     0.485675     0.514325      0.546433   \n",
       "1  0.077907 -1.068169  1.146076     0.546433     0.453567      0.551088   \n",
       "2 -1.402760  3.318841 -5.031425     0.551088     0.448912      0.510793   \n",
       "3 -1.712583  3.318841 -5.031425     0.510793     0.489207      0.461217   \n",
       "4  2.097796  0.000000  0.000000     0.461217     0.538783      0.558929   \n",
       "\n",
       "   Away_WP_post  Win_Prob       WPA    airWPA    yacWPA  Season  \n",
       "0      0.453567  0.485675  0.060758 -0.032244  0.036899    2009  \n",
       "1      0.448912  0.546433  0.004655 -0.032244  0.036899    2009  \n",
       "2      0.489207  0.551088 -0.040295  0.106663 -0.156239    2009  \n",
       "3      0.538783  0.510793 -0.049576  0.106663 -0.156239    2009  \n",
       "4      0.441071  0.461217  0.097712  0.000000  0.000000    2009  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace all NA's the value that comes directly after it in the same column, \n",
    "# then replace all the reamining na's with 0\n",
    "subset_df.fillna(method = 'bfill', axis=0).fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 時間日期處理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1693, 23)"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('data/catalog.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>continent_code</th>\n",
       "      <th>country_name</th>\n",
       "      <th>country_code</th>\n",
       "      <th>state/province</th>\n",
       "      <th>population</th>\n",
       "      <th>city/town</th>\n",
       "      <th>distance</th>\n",
       "      <th>...</th>\n",
       "      <th>geolocation</th>\n",
       "      <th>hazard_type</th>\n",
       "      <th>landslide_type</th>\n",
       "      <th>landslide_size</th>\n",
       "      <th>trigger</th>\n",
       "      <th>storm_name</th>\n",
       "      <th>injuries</th>\n",
       "      <th>fatalities</th>\n",
       "      <th>source_name</th>\n",
       "      <th>source_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34</td>\n",
       "      <td>3/2/07</td>\n",
       "      <td>Night</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Virginia</td>\n",
       "      <td>16000</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>3.40765</td>\n",
       "      <td>...</td>\n",
       "      <td>(38.600900000000003, -77.268199999999993)</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NBC 4 news</td>\n",
       "      <td>http://www.nbc4.com/news/11186871/detail.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>42</td>\n",
       "      <td>3/22/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Ohio</td>\n",
       "      <td>17288</td>\n",
       "      <td>New Philadelphia</td>\n",
       "      <td>3.33522</td>\n",
       "      <td>...</td>\n",
       "      <td>(40.517499999999998, -81.430499999999995)</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canton Rep.com</td>\n",
       "      <td>http://www.cantonrep.com/index.php?ID=345054&amp;C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>56</td>\n",
       "      <td>4/6/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>15930</td>\n",
       "      <td>Wilkinsburg</td>\n",
       "      <td>2.91977</td>\n",
       "      <td>...</td>\n",
       "      <td>(40.4377, -79.915999999999997)</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Pittsburgh Channel.com</td>\n",
       "      <td>https://web.archive.org/web/20080423132842/htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>59</td>\n",
       "      <td>4/14/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canada</td>\n",
       "      <td>CA</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>42786</td>\n",
       "      <td>Châteauguay</td>\n",
       "      <td>2.98682</td>\n",
       "      <td>...</td>\n",
       "      <td>(45.322600000000001, -73.777100000000004)</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Riverbank collapse</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Le Soleil</td>\n",
       "      <td>http://www.hebdos.net/lsc/edition162007/articl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>61</td>\n",
       "      <td>4/15/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Kentucky</td>\n",
       "      <td>6903</td>\n",
       "      <td>Pikeville</td>\n",
       "      <td>5.66542</td>\n",
       "      <td>...</td>\n",
       "      <td>(37.432499999999997, -82.493099999999998)</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Downpour</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Matthew Crawford (KGS)</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id     date   time continent_code   country_name country_code  \\\n",
       "0  34   3/2/07  Night            NaN  United States           US   \n",
       "1  42  3/22/07    NaN            NaN  United States           US   \n",
       "2  56   4/6/07    NaN            NaN  United States           US   \n",
       "3  59  4/14/07    NaN            NaN         Canada           CA   \n",
       "4  61  4/15/07    NaN            NaN  United States           US   \n",
       "\n",
       "  state/province  population         city/town  distance  \\\n",
       "0       Virginia       16000       Cherry Hill   3.40765   \n",
       "1           Ohio       17288  New Philadelphia   3.33522   \n",
       "2   Pennsylvania       15930       Wilkinsburg   2.91977   \n",
       "3         Quebec       42786       Châteauguay   2.98682   \n",
       "4       Kentucky        6903         Pikeville   5.66542   \n",
       "\n",
       "                         ...                          \\\n",
       "0                        ...                           \n",
       "1                        ...                           \n",
       "2                        ...                           \n",
       "3                        ...                           \n",
       "4                        ...                           \n",
       "\n",
       "                                 geolocation  hazard_type      landslide_type  \\\n",
       "0  (38.600900000000003, -77.268199999999993)    Landslide           Landslide   \n",
       "1  (40.517499999999998, -81.430499999999995)    Landslide           Landslide   \n",
       "2             (40.4377, -79.915999999999997)    Landslide           Landslide   \n",
       "3  (45.322600000000001, -73.777100000000004)    Landslide  Riverbank collapse   \n",
       "4  (37.432499999999997, -82.493099999999998)    Landslide           Landslide   \n",
       "\n",
       "  landslide_size   trigger storm_name injuries fatalities  \\\n",
       "0          Small      Rain        NaN      NaN        NaN   \n",
       "1          Small      Rain        NaN      NaN        NaN   \n",
       "2          Small      Rain        NaN      NaN        NaN   \n",
       "3          Small      Rain        NaN      NaN        NaN   \n",
       "4          Small  Downpour        NaN      NaN        0.0   \n",
       "\n",
       "                  source_name  \\\n",
       "0                  NBC 4 news   \n",
       "1              Canton Rep.com   \n",
       "2  The Pittsburgh Channel.com   \n",
       "3                   Le Soleil   \n",
       "4      Matthew Crawford (KGS)   \n",
       "\n",
       "                                         source_link  \n",
       "0      http://www.nbc4.com/news/11186871/detail.html  \n",
       "1  http://www.cantonrep.com/index.php?ID=345054&C...  \n",
       "2  https://web.archive.org/web/20080423132842/htt...  \n",
       "3  http://www.hebdos.net/lsc/edition162007/articl...  \n",
       "4                                                NaN  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 時間日期欄位轉換"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     3/2/07\n",
      "1    3/22/07\n",
      "2     4/6/07\n",
      "3    4/14/07\n",
      "4    4/15/07\n",
      "5    4/20/07\n",
      "6    4/24/07\n",
      "7    5/21/07\n",
      "8    6/27/07\n",
      "9    6/27/07\n",
      "Name: date, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# print the first few rows of the date column\n",
    "print(df['date'].head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('O')"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['date'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# convert to datetime format\n",
    "df['date_parsed'] = pd.to_datetime(df['date'], format = \"%m/%d/%y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0   2007-03-02\n",
      "1   2007-03-22\n",
      "2   2007-04-06\n",
      "3   2007-04-14\n",
      "4   2007-04-15\n",
      "5   2007-04-20\n",
      "6   2007-04-24\n",
      "7   2007-05-21\n",
      "8   2007-06-27\n",
      "9   2007-06-27\n",
      "Name: date_parsed, dtype: datetime64[ns]\n"
     ]
    }
   ],
   "source": [
    "print(df['date_parsed'].head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>continent_code</th>\n",
       "      <th>country_name</th>\n",
       "      <th>country_code</th>\n",
       "      <th>state/province</th>\n",
       "      <th>population</th>\n",
       "      <th>city/town</th>\n",
       "      <th>distance</th>\n",
       "      <th>...</th>\n",
       "      <th>hazard_type</th>\n",
       "      <th>landslide_type</th>\n",
       "      <th>landslide_size</th>\n",
       "      <th>trigger</th>\n",
       "      <th>storm_name</th>\n",
       "      <th>injuries</th>\n",
       "      <th>fatalities</th>\n",
       "      <th>source_name</th>\n",
       "      <th>source_link</th>\n",
       "      <th>date_parsed</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_parsed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2007-03-02</th>\n",
       "      <td>34</td>\n",
       "      <td>3/2/07</td>\n",
       "      <td>Night</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Virginia</td>\n",
       "      <td>16000</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>3.40765</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NBC 4 news</td>\n",
       "      <td>http://www.nbc4.com/news/11186871/detail.html</td>\n",
       "      <td>2007-03-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-22</th>\n",
       "      <td>42</td>\n",
       "      <td>3/22/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Ohio</td>\n",
       "      <td>17288</td>\n",
       "      <td>New Philadelphia</td>\n",
       "      <td>3.33522</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canton Rep.com</td>\n",
       "      <td>http://www.cantonrep.com/index.php?ID=345054&amp;C...</td>\n",
       "      <td>2007-03-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-06</th>\n",
       "      <td>56</td>\n",
       "      <td>4/6/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>15930</td>\n",
       "      <td>Wilkinsburg</td>\n",
       "      <td>2.91977</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Pittsburgh Channel.com</td>\n",
       "      <td>https://web.archive.org/web/20080423132842/htt...</td>\n",
       "      <td>2007-04-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-13</th>\n",
       "      <td>7399</td>\n",
       "      <td>4/13/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>3165</td>\n",
       "      <td>Homestead</td>\n",
       "      <td>3.74490</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Medium</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>WTAE</td>\n",
       "      <td>https://web.archive.org/web/20080423132842/htt...</td>\n",
       "      <td>2007-04-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-14</th>\n",
       "      <td>59</td>\n",
       "      <td>4/14/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canada</td>\n",
       "      <td>CA</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>42786</td>\n",
       "      <td>Châteauguay</td>\n",
       "      <td>2.98682</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Riverbank collapse</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Le Soleil</td>\n",
       "      <td>http://www.hebdos.net/lsc/edition162007/articl...</td>\n",
       "      <td>2007-04-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-15</th>\n",
       "      <td>61</td>\n",
       "      <td>4/15/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Kentucky</td>\n",
       "      <td>6903</td>\n",
       "      <td>Pikeville</td>\n",
       "      <td>5.66542</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Downpour</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Matthew Crawford (KGS)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007-04-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-20</th>\n",
       "      <td>64</td>\n",
       "      <td>4/20/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Kentucky</td>\n",
       "      <td>6903</td>\n",
       "      <td>Pikeville</td>\n",
       "      <td>0.23715</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Applalachain news-express</td>\n",
       "      <td>http://www.news-expressky.com/articles/2007/04...</td>\n",
       "      <td>2007-04-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-24</th>\n",
       "      <td>67</td>\n",
       "      <td>4/24/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>South Dakota</td>\n",
       "      <td>2540</td>\n",
       "      <td>Dakota Dunes</td>\n",
       "      <td>2.48033</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sioux City Journnal</td>\n",
       "      <td>http://www.siouxcityjournal.com/articles/2007/...</td>\n",
       "      <td>2007-04-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-21</th>\n",
       "      <td>77</td>\n",
       "      <td>5/21/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SA</td>\n",
       "      <td>Colombia</td>\n",
       "      <td>CO</td>\n",
       "      <td>Risaralda</td>\n",
       "      <td>440118</td>\n",
       "      <td>Pereira</td>\n",
       "      <td>0.62022</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Mudslide</td>\n",
       "      <td>Large</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.0</td>\n",
       "      <td>Reuters - AlertNet.org</td>\n",
       "      <td>http://www.reuters.com/news/video/videoStory?v...</td>\n",
       "      <td>2007-05-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-06-01</th>\n",
       "      <td>6253</td>\n",
       "      <td>6/1/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Colorado</td>\n",
       "      <td>1864</td>\n",
       "      <td>Granby</td>\n",
       "      <td>5.53226</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Medium</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Sky-Hi News</td>\n",
       "      <td>http://www.skyhidailynews.com/news/13393638-11...</td>\n",
       "      <td>2007-06-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id     date   time continent_code   country_name country_code  \\\n",
       "date_parsed                                                                    \n",
       "2007-03-02     34   3/2/07  Night            NaN  United States           US   \n",
       "2007-03-22     42  3/22/07    NaN            NaN  United States           US   \n",
       "2007-04-06     56   4/6/07    NaN            NaN  United States           US   \n",
       "2007-04-13   7399  4/13/07    NaN            NaN  United States           US   \n",
       "2007-04-14     59  4/14/07    NaN            NaN         Canada           CA   \n",
       "2007-04-15     61  4/15/07    NaN            NaN  United States           US   \n",
       "2007-04-20     64  4/20/07    NaN            NaN  United States           US   \n",
       "2007-04-24     67  4/24/07    NaN            NaN  United States           US   \n",
       "2007-05-21     77  5/21/07    NaN             SA       Colombia           CO   \n",
       "2007-06-01   6253   6/1/07    NaN            NaN  United States           US   \n",
       "\n",
       "            state/province  population         city/town  distance  \\\n",
       "date_parsed                                                          \n",
       "2007-03-02        Virginia       16000       Cherry Hill   3.40765   \n",
       "2007-03-22            Ohio       17288  New Philadelphia   3.33522   \n",
       "2007-04-06    Pennsylvania       15930       Wilkinsburg   2.91977   \n",
       "2007-04-13    Pennsylvania        3165         Homestead   3.74490   \n",
       "2007-04-14          Quebec       42786       Châteauguay   2.98682   \n",
       "2007-04-15        Kentucky        6903         Pikeville   5.66542   \n",
       "2007-04-20        Kentucky        6903         Pikeville   0.23715   \n",
       "2007-04-24    South Dakota        2540      Dakota Dunes   2.48033   \n",
       "2007-05-21       Risaralda      440118           Pereira   0.62022   \n",
       "2007-06-01        Colorado        1864            Granby   5.53226   \n",
       "\n",
       "                ...      hazard_type      landslide_type  landslide_size  \\\n",
       "date_parsed     ...                                                        \n",
       "2007-03-02      ...        Landslide           Landslide           Small   \n",
       "2007-03-22      ...        Landslide           Landslide           Small   \n",
       "2007-04-06      ...        Landslide           Landslide           Small   \n",
       "2007-04-13      ...        Landslide           Landslide          Medium   \n",
       "2007-04-14      ...        Landslide  Riverbank collapse           Small   \n",
       "2007-04-15      ...        Landslide           Landslide           Small   \n",
       "2007-04-20      ...        Landslide           Landslide           Small   \n",
       "2007-04-24      ...        Landslide           Landslide           Small   \n",
       "2007-05-21      ...        Landslide            Mudslide           Large   \n",
       "2007-06-01      ...        Landslide           Landslide          Medium   \n",
       "\n",
       "              trigger storm_name injuries fatalities  \\\n",
       "date_parsed                                            \n",
       "2007-03-02       Rain        NaN      NaN        NaN   \n",
       "2007-03-22       Rain        NaN      NaN        NaN   \n",
       "2007-04-06       Rain        NaN      NaN        NaN   \n",
       "2007-04-13    Unknown        NaN      0.0        0.0   \n",
       "2007-04-14       Rain        NaN      NaN        NaN   \n",
       "2007-04-15   Downpour        NaN      NaN        0.0   \n",
       "2007-04-20       Rain        NaN      NaN        NaN   \n",
       "2007-04-24       Rain        NaN      NaN        NaN   \n",
       "2007-05-21       Rain        NaN      NaN       13.0   \n",
       "2007-06-01    Unknown        NaN      0.0        0.0   \n",
       "\n",
       "                            source_name  \\\n",
       "date_parsed                               \n",
       "2007-03-02                   NBC 4 news   \n",
       "2007-03-22               Canton Rep.com   \n",
       "2007-04-06   The Pittsburgh Channel.com   \n",
       "2007-04-13                         WTAE   \n",
       "2007-04-14                    Le Soleil   \n",
       "2007-04-15       Matthew Crawford (KGS)   \n",
       "2007-04-20    Applalachain news-express   \n",
       "2007-04-24          Sioux City Journnal   \n",
       "2007-05-21       Reuters - AlertNet.org   \n",
       "2007-06-01                  Sky-Hi News   \n",
       "\n",
       "                                                   source_link  date_parsed  \n",
       "date_parsed                                                                  \n",
       "2007-03-02       http://www.nbc4.com/news/11186871/detail.html   2007-03-02  \n",
       "2007-03-22   http://www.cantonrep.com/index.php?ID=345054&C...   2007-03-22  \n",
       "2007-04-06   https://web.archive.org/web/20080423132842/htt...   2007-04-06  \n",
       "2007-04-13   https://web.archive.org/web/20080423132842/htt...   2007-04-13  \n",
       "2007-04-14   http://www.hebdos.net/lsc/edition162007/articl...   2007-04-14  \n",
       "2007-04-15                                                 NaN   2007-04-15  \n",
       "2007-04-20   http://www.news-expressky.com/articles/2007/04...   2007-04-20  \n",
       "2007-04-24   http://www.siouxcityjournal.com/articles/2007/...   2007-04-24  \n",
       "2007-05-21   http://www.reuters.com/news/video/videoStory?v...   2007-05-21  \n",
       "2007-06-01   http://www.skyhidailynews.com/news/13393638-11...   2007-06-01  \n",
       "\n",
       "[10 rows x 24 columns]"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set index to date\n",
    "df = df.set_index(df['date_parsed'])\n",
    "df =df.sort_index()\n",
    "df = df[df.index.notnull()]\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>continent_code</th>\n",
       "      <th>country_name</th>\n",
       "      <th>country_code</th>\n",
       "      <th>state/province</th>\n",
       "      <th>population</th>\n",
       "      <th>city/town</th>\n",
       "      <th>distance</th>\n",
       "      <th>...</th>\n",
       "      <th>hazard_type</th>\n",
       "      <th>landslide_type</th>\n",
       "      <th>landslide_size</th>\n",
       "      <th>trigger</th>\n",
       "      <th>storm_name</th>\n",
       "      <th>injuries</th>\n",
       "      <th>fatalities</th>\n",
       "      <th>source_name</th>\n",
       "      <th>source_link</th>\n",
       "      <th>date_parsed</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_parsed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2007-03-02</th>\n",
       "      <td>34</td>\n",
       "      <td>3/2/07</td>\n",
       "      <td>Night</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Virginia</td>\n",
       "      <td>16000</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>3.40765</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NBC 4 news</td>\n",
       "      <td>http://www.nbc4.com/news/11186871/detail.html</td>\n",
       "      <td>2007-03-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-22</th>\n",
       "      <td>42</td>\n",
       "      <td>3/22/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Ohio</td>\n",
       "      <td>17288</td>\n",
       "      <td>New Philadelphia</td>\n",
       "      <td>3.33522</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canton Rep.com</td>\n",
       "      <td>http://www.cantonrep.com/index.php?ID=345054&amp;C...</td>\n",
       "      <td>2007-03-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-06</th>\n",
       "      <td>56</td>\n",
       "      <td>4/6/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>15930</td>\n",
       "      <td>Wilkinsburg</td>\n",
       "      <td>2.91977</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Pittsburgh Channel.com</td>\n",
       "      <td>https://web.archive.org/web/20080423132842/htt...</td>\n",
       "      <td>2007-04-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-13</th>\n",
       "      <td>7399</td>\n",
       "      <td>4/13/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Pennsylvania</td>\n",
       "      <td>3165</td>\n",
       "      <td>Homestead</td>\n",
       "      <td>3.74490</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Medium</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>WTAE</td>\n",
       "      <td>https://web.archive.org/web/20080423132842/htt...</td>\n",
       "      <td>2007-04-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-14</th>\n",
       "      <td>59</td>\n",
       "      <td>4/14/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canada</td>\n",
       "      <td>CA</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>42786</td>\n",
       "      <td>Châteauguay</td>\n",
       "      <td>2.98682</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Riverbank collapse</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Le Soleil</td>\n",
       "      <td>http://www.hebdos.net/lsc/edition162007/articl...</td>\n",
       "      <td>2007-04-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-15</th>\n",
       "      <td>61</td>\n",
       "      <td>4/15/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Kentucky</td>\n",
       "      <td>6903</td>\n",
       "      <td>Pikeville</td>\n",
       "      <td>5.66542</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Downpour</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Matthew Crawford (KGS)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007-04-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-20</th>\n",
       "      <td>64</td>\n",
       "      <td>4/20/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>Kentucky</td>\n",
       "      <td>6903</td>\n",
       "      <td>Pikeville</td>\n",
       "      <td>0.23715</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Applalachain news-express</td>\n",
       "      <td>http://www.news-expressky.com/articles/2007/04...</td>\n",
       "      <td>2007-04-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-24</th>\n",
       "      <td>67</td>\n",
       "      <td>4/24/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>United States</td>\n",
       "      <td>US</td>\n",
       "      <td>South Dakota</td>\n",
       "      <td>2540</td>\n",
       "      <td>Dakota Dunes</td>\n",
       "      <td>2.48033</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Small</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sioux City Journnal</td>\n",
       "      <td>http://www.siouxcityjournal.com/articles/2007/...</td>\n",
       "      <td>2007-04-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-21</th>\n",
       "      <td>77</td>\n",
       "      <td>5/21/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SA</td>\n",
       "      <td>Colombia</td>\n",
       "      <td>CO</td>\n",
       "      <td>Risaralda</td>\n",
       "      <td>440118</td>\n",
       "      <td>Pereira</td>\n",
       "      <td>0.62022</td>\n",
       "      <td>...</td>\n",
       "      <td>Landslide</td>\n",
       "      <td>Mudslide</td>\n",
       "      <td>Large</td>\n",
       "      <td>Rain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13.0</td>\n",
       "      <td>Reuters - AlertNet.org</td>\n",
       "      <td>http://www.reuters.com/news/video/videoStory?v...</td>\n",
       "      <td>2007-05-21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id     date   time continent_code   country_name country_code  \\\n",
       "date_parsed                                                                    \n",
       "2007-03-02     34   3/2/07  Night            NaN  United States           US   \n",
       "2007-03-22     42  3/22/07    NaN            NaN  United States           US   \n",
       "2007-04-06     56   4/6/07    NaN            NaN  United States           US   \n",
       "2007-04-13   7399  4/13/07    NaN            NaN  United States           US   \n",
       "2007-04-14     59  4/14/07    NaN            NaN         Canada           CA   \n",
       "2007-04-15     61  4/15/07    NaN            NaN  United States           US   \n",
       "2007-04-20     64  4/20/07    NaN            NaN  United States           US   \n",
       "2007-04-24     67  4/24/07    NaN            NaN  United States           US   \n",
       "2007-05-21     77  5/21/07    NaN             SA       Colombia           CO   \n",
       "\n",
       "            state/province  population         city/town  distance  \\\n",
       "date_parsed                                                          \n",
       "2007-03-02        Virginia       16000       Cherry Hill   3.40765   \n",
       "2007-03-22            Ohio       17288  New Philadelphia   3.33522   \n",
       "2007-04-06    Pennsylvania       15930       Wilkinsburg   2.91977   \n",
       "2007-04-13    Pennsylvania        3165         Homestead   3.74490   \n",
       "2007-04-14          Quebec       42786       Châteauguay   2.98682   \n",
       "2007-04-15        Kentucky        6903         Pikeville   5.66542   \n",
       "2007-04-20        Kentucky        6903         Pikeville   0.23715   \n",
       "2007-04-24    South Dakota        2540      Dakota Dunes   2.48033   \n",
       "2007-05-21       Risaralda      440118           Pereira   0.62022   \n",
       "\n",
       "                ...      hazard_type      landslide_type  landslide_size  \\\n",
       "date_parsed     ...                                                        \n",
       "2007-03-02      ...        Landslide           Landslide           Small   \n",
       "2007-03-22      ...        Landslide           Landslide           Small   \n",
       "2007-04-06      ...        Landslide           Landslide           Small   \n",
       "2007-04-13      ...        Landslide           Landslide          Medium   \n",
       "2007-04-14      ...        Landslide  Riverbank collapse           Small   \n",
       "2007-04-15      ...        Landslide           Landslide           Small   \n",
       "2007-04-20      ...        Landslide           Landslide           Small   \n",
       "2007-04-24      ...        Landslide           Landslide           Small   \n",
       "2007-05-21      ...        Landslide            Mudslide           Large   \n",
       "\n",
       "              trigger storm_name injuries fatalities  \\\n",
       "date_parsed                                            \n",
       "2007-03-02       Rain        NaN      NaN        NaN   \n",
       "2007-03-22       Rain        NaN      NaN        NaN   \n",
       "2007-04-06       Rain        NaN      NaN        NaN   \n",
       "2007-04-13    Unknown        NaN      0.0        0.0   \n",
       "2007-04-14       Rain        NaN      NaN        NaN   \n",
       "2007-04-15   Downpour        NaN      NaN        0.0   \n",
       "2007-04-20       Rain        NaN      NaN        NaN   \n",
       "2007-04-24       Rain        NaN      NaN        NaN   \n",
       "2007-05-21       Rain        NaN      NaN       13.0   \n",
       "\n",
       "                            source_name  \\\n",
       "date_parsed                               \n",
       "2007-03-02                   NBC 4 news   \n",
       "2007-03-22               Canton Rep.com   \n",
       "2007-04-06   The Pittsburgh Channel.com   \n",
       "2007-04-13                         WTAE   \n",
       "2007-04-14                    Le Soleil   \n",
       "2007-04-15       Matthew Crawford (KGS)   \n",
       "2007-04-20    Applalachain news-express   \n",
       "2007-04-24          Sioux City Journnal   \n",
       "2007-05-21       Reuters - AlertNet.org   \n",
       "\n",
       "                                                   source_link  date_parsed  \n",
       "date_parsed                                                                  \n",
       "2007-03-02       http://www.nbc4.com/news/11186871/detail.html   2007-03-02  \n",
       "2007-03-22   http://www.cantonrep.com/index.php?ID=345054&C...   2007-03-22  \n",
       "2007-04-06   https://web.archive.org/web/20080423132842/htt...   2007-04-06  \n",
       "2007-04-13   https://web.archive.org/web/20080423132842/htt...   2007-04-13  \n",
       "2007-04-14   http://www.hebdos.net/lsc/edition162007/articl...   2007-04-14  \n",
       "2007-04-15                                                 NaN   2007-04-15  \n",
       "2007-04-20   http://www.news-expressky.com/articles/2007/04...   2007-04-20  \n",
       "2007-04-24   http://www.siouxcityjournal.com/articles/2007/...   2007-04-24  \n",
       "2007-05-21   http://www.reuters.com/news/video/videoStory?v...   2007-05-21  \n",
       "\n",
       "[9 rows x 24 columns]"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select row bt date\n",
    "df[\"2007-03\":\"2007-05\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 時間序列解析度操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>population</th>\n",
       "      <th>distance</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_parsed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2007-03-02</th>\n",
       "      <td>16000</td>\n",
       "      <td>3.40765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-22</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-06</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-13</th>\n",
       "      <td>3165</td>\n",
       "      <td>3.74490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-14</th>\n",
       "      <td>42786</td>\n",
       "      <td>2.98682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-15</th>\n",
       "      <td>6903</td>\n",
       "      <td>5.66542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-20</th>\n",
       "      <td>6903</td>\n",
       "      <td>0.23715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-04-24</th>\n",
       "      <td>2540</td>\n",
       "      <td>2.48033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-21</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-06-01</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             population  distance\n",
       "date_parsed                      \n",
       "2007-03-02        16000   3.40765\n",
       "2007-03-22        17288   3.33522\n",
       "2007-04-06        15930   2.91977\n",
       "2007-04-13         3165   3.74490\n",
       "2007-04-14        42786   2.98682\n",
       "2007-04-15         6903   5.66542\n",
       "2007-04-20         6903   0.23715\n",
       "2007-04-24         2540   2.48033\n",
       "2007-05-21       440118   0.62022\n",
       "2007-06-01         1864   5.53226"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select sub df\n",
    "df = df[~df.index.duplicated(keep='first')]\n",
    "sub_df = df.loc[:,['population', 'distance']].head(10)\n",
    "sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>population</th>\n",
       "      <th>distance</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_parsed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2007-03-02</th>\n",
       "      <td>16000.0</td>\n",
       "      <td>3.40765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-03</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-04</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-05</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-06</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-07</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-08</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-09</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-20</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-21</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-22</th>\n",
       "      <td>17288.0</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-23</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-25</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-26</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-27</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-28</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-29</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-30</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-31</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-03</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-04</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-05</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-06</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-07</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-08</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-09</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-20</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-21</th>\n",
       "      <td>440118.0</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-22</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-23</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-25</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-26</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-27</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-28</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-29</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-30</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-31</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-06-01</th>\n",
       "      <td>1864.0</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             population  distance\n",
       "date_parsed                      \n",
       "2007-03-02      16000.0   3.40765\n",
       "2007-03-03          NaN       NaN\n",
       "2007-03-04          NaN       NaN\n",
       "2007-03-05          NaN       NaN\n",
       "2007-03-06          NaN       NaN\n",
       "2007-03-07          NaN       NaN\n",
       "2007-03-08          NaN       NaN\n",
       "2007-03-09          NaN       NaN\n",
       "2007-03-10          NaN       NaN\n",
       "2007-03-11          NaN       NaN\n",
       "2007-03-12          NaN       NaN\n",
       "2007-03-13          NaN       NaN\n",
       "2007-03-14          NaN       NaN\n",
       "2007-03-15          NaN       NaN\n",
       "2007-03-16          NaN       NaN\n",
       "2007-03-17          NaN       NaN\n",
       "2007-03-18          NaN       NaN\n",
       "2007-03-19          NaN       NaN\n",
       "2007-03-20          NaN       NaN\n",
       "2007-03-21          NaN       NaN\n",
       "2007-03-22      17288.0   3.33522\n",
       "2007-03-23          NaN       NaN\n",
       "2007-03-24          NaN       NaN\n",
       "2007-03-25          NaN       NaN\n",
       "2007-03-26          NaN       NaN\n",
       "2007-03-27          NaN       NaN\n",
       "2007-03-28          NaN       NaN\n",
       "2007-03-29          NaN       NaN\n",
       "2007-03-30          NaN       NaN\n",
       "2007-03-31          NaN       NaN\n",
       "...                 ...       ...\n",
       "2007-05-03          NaN       NaN\n",
       "2007-05-04          NaN       NaN\n",
       "2007-05-05          NaN       NaN\n",
       "2007-05-06          NaN       NaN\n",
       "2007-05-07          NaN       NaN\n",
       "2007-05-08          NaN       NaN\n",
       "2007-05-09          NaN       NaN\n",
       "2007-05-10          NaN       NaN\n",
       "2007-05-11          NaN       NaN\n",
       "2007-05-12          NaN       NaN\n",
       "2007-05-13          NaN       NaN\n",
       "2007-05-14          NaN       NaN\n",
       "2007-05-15          NaN       NaN\n",
       "2007-05-16          NaN       NaN\n",
       "2007-05-17          NaN       NaN\n",
       "2007-05-18          NaN       NaN\n",
       "2007-05-19          NaN       NaN\n",
       "2007-05-20          NaN       NaN\n",
       "2007-05-21     440118.0   0.62022\n",
       "2007-05-22          NaN       NaN\n",
       "2007-05-23          NaN       NaN\n",
       "2007-05-24          NaN       NaN\n",
       "2007-05-25          NaN       NaN\n",
       "2007-05-26          NaN       NaN\n",
       "2007-05-27          NaN       NaN\n",
       "2007-05-28          NaN       NaN\n",
       "2007-05-29          NaN       NaN\n",
       "2007-05-30          NaN       NaN\n",
       "2007-05-31          NaN       NaN\n",
       "2007-06-01       1864.0   5.53226\n",
       "\n",
       "[92 rows x 2 columns]"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# resample interval to 1 day.\n",
    "sub_df.resample('1D').asfreq()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>population</th>\n",
       "      <th>distance</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_parsed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2007-03-02</th>\n",
       "      <td>16000</td>\n",
       "      <td>3.40765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-03</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-04</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-05</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-06</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-07</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-08</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-09</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-10</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-11</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-12</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-13</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-14</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-15</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-16</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-17</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-18</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-19</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-20</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-21</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-22</th>\n",
       "      <td>17288</td>\n",
       "      <td>3.33522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-23</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-24</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-25</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-26</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-27</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-28</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-29</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-30</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-03-31</th>\n",
       "      <td>15930</td>\n",
       "      <td>2.91977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-03</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-04</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-05</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-06</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-07</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-08</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-09</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-10</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-11</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-12</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-13</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-14</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-15</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-16</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-17</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-18</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-19</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-20</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-21</th>\n",
       "      <td>440118</td>\n",
       "      <td>0.62022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-22</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-23</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-24</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-25</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-26</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-27</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-28</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-29</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-30</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-05-31</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007-06-01</th>\n",
       "      <td>1864</td>\n",
       "      <td>5.53226</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             population  distance\n",
       "date_parsed                      \n",
       "2007-03-02        16000   3.40765\n",
       "2007-03-03        17288   3.33522\n",
       "2007-03-04        17288   3.33522\n",
       "2007-03-05        17288   3.33522\n",
       "2007-03-06        17288   3.33522\n",
       "2007-03-07        17288   3.33522\n",
       "2007-03-08        17288   3.33522\n",
       "2007-03-09        17288   3.33522\n",
       "2007-03-10        17288   3.33522\n",
       "2007-03-11        17288   3.33522\n",
       "2007-03-12        17288   3.33522\n",
       "2007-03-13        17288   3.33522\n",
       "2007-03-14        17288   3.33522\n",
       "2007-03-15        17288   3.33522\n",
       "2007-03-16        17288   3.33522\n",
       "2007-03-17        17288   3.33522\n",
       "2007-03-18        17288   3.33522\n",
       "2007-03-19        17288   3.33522\n",
       "2007-03-20        17288   3.33522\n",
       "2007-03-21        17288   3.33522\n",
       "2007-03-22        17288   3.33522\n",
       "2007-03-23        15930   2.91977\n",
       "2007-03-24        15930   2.91977\n",
       "2007-03-25        15930   2.91977\n",
       "2007-03-26        15930   2.91977\n",
       "2007-03-27        15930   2.91977\n",
       "2007-03-28        15930   2.91977\n",
       "2007-03-29        15930   2.91977\n",
       "2007-03-30        15930   2.91977\n",
       "2007-03-31        15930   2.91977\n",
       "...                 ...       ...\n",
       "2007-05-03       440118   0.62022\n",
       "2007-05-04       440118   0.62022\n",
       "2007-05-05       440118   0.62022\n",
       "2007-05-06       440118   0.62022\n",
       "2007-05-07       440118   0.62022\n",
       "2007-05-08       440118   0.62022\n",
       "2007-05-09       440118   0.62022\n",
       "2007-05-10       440118   0.62022\n",
       "2007-05-11       440118   0.62022\n",
       "2007-05-12       440118   0.62022\n",
       "2007-05-13       440118   0.62022\n",
       "2007-05-14       440118   0.62022\n",
       "2007-05-15       440118   0.62022\n",
       "2007-05-16       440118   0.62022\n",
       "2007-05-17       440118   0.62022\n",
       "2007-05-18       440118   0.62022\n",
       "2007-05-19       440118   0.62022\n",
       "2007-05-20       440118   0.62022\n",
       "2007-05-21       440118   0.62022\n",
       "2007-05-22         1864   5.53226\n",
       "2007-05-23         1864   5.53226\n",
       "2007-05-24         1864   5.53226\n",
       "2007-05-25         1864   5.53226\n",
       "2007-05-26         1864   5.53226\n",
       "2007-05-27         1864   5.53226\n",
       "2007-05-28         1864   5.53226\n",
       "2007-05-29         1864   5.53226\n",
       "2007-05-30         1864   5.53226\n",
       "2007-05-31         1864   5.53226\n",
       "2007-06-01         1864   5.53226\n",
       "\n",
       "[92 rows x 2 columns]"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# bfill\n",
    "sub_df.resample('1D').bfill()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 資料分割選擇與合併"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. .iloc vs loc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](http://104.236.88.249/wp-content/uploads/2016/10/Pandas-selections-and-indexing.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(np.random.randn(4,4), columns=['a', 'b', 'c', 'd'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.400363</td>\n",
       "      <td>1.141485</td>\n",
       "      <td>0.821102</td>\n",
       "      <td>0.237137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.122230</td>\n",
       "      <td>0.293221</td>\n",
       "      <td>-1.318223</td>\n",
       "      <td>0.765636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.530837</td>\n",
       "      <td>-0.730529</td>\n",
       "      <td>-0.614539</td>\n",
       "      <td>-0.639149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.201329</td>\n",
       "      <td>1.279875</td>\n",
       "      <td>1.875210</td>\n",
       "      <td>0.466313</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d\n",
       "0 -0.400363  1.141485  0.821102  0.237137\n",
       "1 -0.122230  0.293221 -1.318223  0.765636\n",
       "2  1.530837 -0.730529 -0.614539 -0.639149\n",
       "3 -0.201329  1.279875  1.875210  0.466313"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.574904</td>\n",
       "      <td>-0.263963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.377969</td>\n",
       "      <td>2.188388</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         c\n",
       "0  1.574904 -0.263963\n",
       "1 -1.377969  2.188388"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# iloc index based index slelction\n",
    "df.iloc[:2,[0,2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.574904</td>\n",
       "      <td>-0.263963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.377969</td>\n",
       "      <td>2.188388</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         c\n",
       "0  1.574904 -0.263963\n",
       "1 -1.377969  2.188388"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loc label based index slelction\n",
    "df.loc[:1,['a', 'c']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 兩者容易混淆之處"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.574904</td>\n",
       "      <td>-0.008773</td>\n",
       "      <td>-0.263963</td>\n",
       "      <td>-0.806723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.377969</td>\n",
       "      <td>0.754676</td>\n",
       "      <td>2.188388</td>\n",
       "      <td>-0.243525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.328553</td>\n",
       "      <td>-1.505490</td>\n",
       "      <td>-0.161446</td>\n",
       "      <td>1.653918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.565548</td>\n",
       "      <td>0.072182</td>\n",
       "      <td>-2.720340</td>\n",
       "      <td>0.950584</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d\n",
       "0  1.574904 -0.008773 -0.263963 -0.806723\n",
       "1 -1.377969  0.754676  2.188388 -0.243525\n",
       "8  0.328553 -1.505490 -0.161446  1.653918\n",
       "3  0.565548  0.072182 -2.720340  0.950584"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(np.random.randn(4,4), columns=['a', 'b', 'c', 'd'], index=[0, 1, 8, 3])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 選取 3, 4 列作法:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.328553</td>\n",
       "      <td>-1.505490</td>\n",
       "      <td>-0.161446</td>\n",
       "      <td>1.653918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.565548</td>\n",
       "      <td>0.072182</td>\n",
       "      <td>-2.720340</td>\n",
       "      <td>0.950584</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d\n",
       "8  0.328553 -1.505490 -0.161446  1.653918\n",
       "3  0.565548  0.072182 -2.720340  0.950584"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# iloc\n",
    "df.iloc[2:4,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# loc\n",
    "df.loc[2:4,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.328553</td>\n",
       "      <td>-1.505490</td>\n",
       "      <td>-0.161446</td>\n",
       "      <td>1.653918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.565548</td>\n",
       "      <td>0.072182</td>\n",
       "      <td>-2.720340</td>\n",
       "      <td>0.950584</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d\n",
       "8  0.328553 -1.505490 -0.161446  1.653918\n",
       "3  0.565548  0.072182 -2.720340  0.950584"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loc 正確 index 取法\n",
    "df.loc[8:3,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 合併 concat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 5)\n",
      "(6, 4)\n"
     ]
    }
   ],
   "source": [
    "df1 = pd.DataFrame(np.random.randn(4,5), columns=['a', 'b', 'c', 'd', 'e'])\n",
    "df2 = pd.DataFrame(np.random.randn(6,4), columns=['g', 'y', 'k', 'f'])\n",
    "print(df1.shape)\n",
    "print(df2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>k</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.919846</td>\n",
       "      <td>-0.495675</td>\n",
       "      <td>1.911603</td>\n",
       "      <td>1.309731</td>\n",
       "      <td>0.399571</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.494185</td>\n",
       "      <td>-1.246625</td>\n",
       "      <td>0.074627</td>\n",
       "      <td>-0.725263</td>\n",
       "      <td>0.814656</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.453585</td>\n",
       "      <td>0.908129</td>\n",
       "      <td>1.105333</td>\n",
       "      <td>0.108686</td>\n",
       "      <td>0.420912</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.763184</td>\n",
       "      <td>-0.427747</td>\n",
       "      <td>0.844336</td>\n",
       "      <td>-0.915844</td>\n",
       "      <td>-1.432858</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.918278</td>\n",
       "      <td>1.002602</td>\n",
       "      <td>-0.174406</td>\n",
       "      <td>-0.910231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.493787</td>\n",
       "      <td>1.015558</td>\n",
       "      <td>-0.517051</td>\n",
       "      <td>-0.790699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.080813</td>\n",
       "      <td>0.035731</td>\n",
       "      <td>-1.578405</td>\n",
       "      <td>0.119997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.402318</td>\n",
       "      <td>-2.018605</td>\n",
       "      <td>0.268080</td>\n",
       "      <td>-0.988506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.741613</td>\n",
       "      <td>-0.415160</td>\n",
       "      <td>-0.330801</td>\n",
       "      <td>0.301276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.810024</td>\n",
       "      <td>1.609868</td>\n",
       "      <td>0.397550</td>\n",
       "      <td>-0.338401</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d         e         f         g  \\\n",
       "0 -0.919846 -0.495675  1.911603  1.309731  0.399571       NaN       NaN   \n",
       "1 -0.494185 -1.246625  0.074627 -0.725263  0.814656       NaN       NaN   \n",
       "2 -1.453585  0.908129  1.105333  0.108686  0.420912       NaN       NaN   \n",
       "3  0.763184 -0.427747  0.844336 -0.915844 -1.432858       NaN       NaN   \n",
       "0       NaN       NaN       NaN       NaN       NaN  0.918278  1.002602   \n",
       "1       NaN       NaN       NaN       NaN       NaN  1.493787  1.015558   \n",
       "2       NaN       NaN       NaN       NaN       NaN  0.080813  0.035731   \n",
       "3       NaN       NaN       NaN       NaN       NaN -0.402318 -2.018605   \n",
       "4       NaN       NaN       NaN       NaN       NaN  0.741613 -0.415160   \n",
       "5       NaN       NaN       NaN       NaN       NaN  0.810024  1.609868   \n",
       "\n",
       "          k         y  \n",
       "0       NaN       NaN  \n",
       "1       NaN       NaN  \n",
       "2       NaN       NaN  \n",
       "3       NaN       NaN  \n",
       "0 -0.174406 -0.910231  \n",
       "1 -0.517051 -0.790699  \n",
       "2 -1.578405  0.119997  \n",
       "3  0.268080 -0.988506  \n",
       "4 -0.330801  0.301276  \n",
       "5  0.397550 -0.338401  "
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 列方向合併 (上下)\n",
    "merge_df = pd.concat([df1, df2], axis=0)\n",
    "merge_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 9)\n"
     ]
    }
   ],
   "source": [
    "print(merge_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>g</th>\n",
       "      <th>y</th>\n",
       "      <th>k</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.919846</td>\n",
       "      <td>-0.495675</td>\n",
       "      <td>1.911603</td>\n",
       "      <td>1.309731</td>\n",
       "      <td>0.399571</td>\n",
       "      <td>1.002602</td>\n",
       "      <td>-0.910231</td>\n",
       "      <td>-0.174406</td>\n",
       "      <td>0.918278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.494185</td>\n",
       "      <td>-1.246625</td>\n",
       "      <td>0.074627</td>\n",
       "      <td>-0.725263</td>\n",
       "      <td>0.814656</td>\n",
       "      <td>1.015558</td>\n",
       "      <td>-0.790699</td>\n",
       "      <td>-0.517051</td>\n",
       "      <td>1.493787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.453585</td>\n",
       "      <td>0.908129</td>\n",
       "      <td>1.105333</td>\n",
       "      <td>0.108686</td>\n",
       "      <td>0.420912</td>\n",
       "      <td>0.035731</td>\n",
       "      <td>0.119997</td>\n",
       "      <td>-1.578405</td>\n",
       "      <td>0.080813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.763184</td>\n",
       "      <td>-0.427747</td>\n",
       "      <td>0.844336</td>\n",
       "      <td>-0.915844</td>\n",
       "      <td>-1.432858</td>\n",
       "      <td>-2.018605</td>\n",
       "      <td>-0.988506</td>\n",
       "      <td>0.268080</td>\n",
       "      <td>-0.402318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.415160</td>\n",
       "      <td>0.301276</td>\n",
       "      <td>-0.330801</td>\n",
       "      <td>0.741613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.609868</td>\n",
       "      <td>-0.338401</td>\n",
       "      <td>0.397550</td>\n",
       "      <td>0.810024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d         e         g         y  \\\n",
       "0 -0.919846 -0.495675  1.911603  1.309731  0.399571  1.002602 -0.910231   \n",
       "1 -0.494185 -1.246625  0.074627 -0.725263  0.814656  1.015558 -0.790699   \n",
       "2 -1.453585  0.908129  1.105333  0.108686  0.420912  0.035731  0.119997   \n",
       "3  0.763184 -0.427747  0.844336 -0.915844 -1.432858 -2.018605 -0.988506   \n",
       "4       NaN       NaN       NaN       NaN       NaN -0.415160  0.301276   \n",
       "5       NaN       NaN       NaN       NaN       NaN  1.609868 -0.338401   \n",
       "\n",
       "          k         f  \n",
       "0 -0.174406  0.918278  \n",
       "1 -0.517051  1.493787  \n",
       "2 -1.578405  0.080813  \n",
       "3  0.268080 -0.402318  \n",
       "4 -0.330801  0.741613  \n",
       "5  0.397550  0.810024  "
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 行方向合併 (左右)\n",
    "merge_df = pd.concat([df1, df2], axis=1)\n",
    "merge_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6, 9)"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## reference\n",
    "\n",
    "1. https://www.kaggle.com/rtatman/data-cleaning-challenge-handling-missing-values\n",
    "2. http://www.learningthemachine.com/2017/08/20/review-pandas-loc-vs-iloc/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}