04_build_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# input file - dataset only when the pump is opening\n",
    "file_directory = './sample data/output_3_files.csv'\n",
    "\n",
    "# output file - without useless features (for modelling)\n",
    "modelling_data_directory = './sample data/modelling_dataset.csv'\n",
    "modelling_diff_data_directory = './sample data/modelling_diff_dataset.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2769, 399)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(file_directory)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>measurement_result_date</th>\n",
       "      <th>average_result_value</th>\n",
       "      <th>statistic_type</th>\n",
       "      <th>asset_code</th>\n",
       "      <th>facility_code</th>\n",
       "      <th>Average Result Value</th>\n",
       "      <th>Measurement_Result_Date_Minute_15</th>\n",
       "      <th>zone_flow_hist_0min</th>\n",
       "      <th>zone_flow_hist_15min</th>\n",
       "      <th>zone_flow_hist_30min</th>\n",
       "      <th>...</th>\n",
       "      <th>res_cl_hist_2835min</th>\n",
       "      <th>res_cl_hist_2850min</th>\n",
       "      <th>res_cl_hist_2865min</th>\n",
       "      <th>res_cl_hist_2880min</th>\n",
       "      <th>res_temperature</th>\n",
       "      <th>travel_time_hrs_nom</th>\n",
       "      <th>flow_nom</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>node_id</th>\n",
       "      <th>zone_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-12-01 00:00:00</td>\n",
       "      <td>1.4572</td>\n",
       "      <td>Snap (Current)</td>\n",
       "      <td>WQM01</td>\n",
       "      <td>WQ0118</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-12-01 00:00:00</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>...</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>1.708366</td>\n",
       "      <td>17.4</td>\n",
       "      <td>2.789096</td>\n",
       "      <td>1.931739</td>\n",
       "      <td>2016-12-01 00:00:00</td>\n",
       "      <td>31238.0</td>\n",
       "      <td>engadine_rem_flow_zone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-12-01 00:15:00</td>\n",
       "      <td>1.4560</td>\n",
       "      <td>Snap (Current)</td>\n",
       "      <td>WQM01</td>\n",
       "      <td>WQ0118</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-12-01 00:15:00</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>...</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>17.4</td>\n",
       "      <td>2.789096</td>\n",
       "      <td>1.931739</td>\n",
       "      <td>2016-12-01 00:15:00</td>\n",
       "      <td>31238.0</td>\n",
       "      <td>engadine_rem_flow_zone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-12-01 00:30:00</td>\n",
       "      <td>1.4540</td>\n",
       "      <td>Snap (Current)</td>\n",
       "      <td>WQM01</td>\n",
       "      <td>WQ0118</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-12-01 00:30:00</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>...</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>17.4</td>\n",
       "      <td>2.789096</td>\n",
       "      <td>1.931739</td>\n",
       "      <td>2016-12-01 00:30:00</td>\n",
       "      <td>31238.0</td>\n",
       "      <td>engadine_rem_flow_zone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-12-01 00:45:00</td>\n",
       "      <td>1.4432</td>\n",
       "      <td>Snap (Current)</td>\n",
       "      <td>WQM01</td>\n",
       "      <td>WQ0118</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-12-01 00:45:00</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>...</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>17.4</td>\n",
       "      <td>2.789096</td>\n",
       "      <td>1.931739</td>\n",
       "      <td>2016-12-01 00:45:00</td>\n",
       "      <td>31238.0</td>\n",
       "      <td>engadine_rem_flow_zone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-12-01 01:00:00</td>\n",
       "      <td>1.4306</td>\n",
       "      <td>Snap (Current)</td>\n",
       "      <td>WQM01</td>\n",
       "      <td>WQ0118</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-12-01 01:00:00</td>\n",
       "      <td>0.172979</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>...</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>17.4</td>\n",
       "      <td>2.789096</td>\n",
       "      <td>1.931739</td>\n",
       "      <td>2016-12-01 01:00:00</td>\n",
       "      <td>31238.0</td>\n",
       "      <td>engadine_rem_flow_zone</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 399 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  measurement_result_date  average_result_value  statistic_type asset_code  \\\n",
       "0     2016-12-01 00:00:00                1.4572  Snap (Current)      WQM01   \n",
       "1     2016-12-01 00:15:00                1.4560  Snap (Current)      WQM01   \n",
       "2     2016-12-01 00:30:00                1.4540  Snap (Current)      WQM01   \n",
       "3     2016-12-01 00:45:00                1.4432  Snap (Current)      WQM01   \n",
       "4     2016-12-01 01:00:00                1.4306  Snap (Current)      WQM01   \n",
       "\n",
       "  facility_code  Average Result Value Measurement_Result_Date_Minute_15  \\\n",
       "0        WQ0118                     1               2016-12-01 00:00:00   \n",
       "1        WQ0118                     1               2016-12-01 00:15:00   \n",
       "2        WQ0118                     1               2016-12-01 00:30:00   \n",
       "3        WQ0118                     1               2016-12-01 00:45:00   \n",
       "4        WQ0118                     1               2016-12-01 01:00:00   \n",
       "\n",
       "   zone_flow_hist_0min  zone_flow_hist_15min  zone_flow_hist_30min  \\\n",
       "0             0.537967              0.677703              0.838600   \n",
       "1             0.374730              0.537967              0.677703   \n",
       "2             0.228371              0.374730              0.537967   \n",
       "3             0.169935              0.228371              0.374730   \n",
       "4             0.172979              0.169935              0.228371   \n",
       "\n",
       "            ...            res_cl_hist_2835min  res_cl_hist_2850min  \\\n",
       "0           ...                       1.711658             1.709118   \n",
       "1           ...                       1.714862             1.711658   \n",
       "2           ...                       1.718593             1.714862   \n",
       "3           ...                       1.720893             1.718593   \n",
       "4           ...                       1.721563             1.720893   \n",
       "\n",
       "   res_cl_hist_2865min  res_cl_hist_2880min  res_temperature  \\\n",
       "0             1.707396             1.708366             17.4   \n",
       "1             1.709118             1.707396             17.4   \n",
       "2             1.711658             1.709118             17.4   \n",
       "3             1.714862             1.711658             17.4   \n",
       "4             1.718593             1.714862             17.4   \n",
       "\n",
       "   travel_time_hrs_nom  flow_nom            timestamp  node_id  \\\n",
       "0             2.789096  1.931739  2016-12-01 00:00:00  31238.0   \n",
       "1             2.789096  1.931739  2016-12-01 00:15:00  31238.0   \n",
       "2             2.789096  1.931739  2016-12-01 00:30:00  31238.0   \n",
       "3             2.789096  1.931739  2016-12-01 00:45:00  31238.0   \n",
       "4             2.789096  1.931739  2016-12-01 01:00:00  31238.0   \n",
       "\n",
       "                  zone_id  \n",
       "0  engadine_rem_flow_zone  \n",
       "1  engadine_rem_flow_zone  \n",
       "2  engadine_rem_flow_zone  \n",
       "3  engadine_rem_flow_zone  \n",
       "4  engadine_rem_flow_zone  \n",
       "\n",
       "[5 rows x 399 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 node id in feature dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    17282.000000\n",
       "mean     36573.000000\n",
       "std       5335.154358\n",
       "min      31238.000000\n",
       "25%      31238.000000\n",
       "50%      36573.000000\n",
       "75%      41908.000000\n",
       "max      41908.000000\n",
       "Name: node_id, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = \"./sample data/feature.csv\"\n",
    "df_feature = pd.read_csv(filename, parse_dates=['timestamp'])\n",
    "df_feature['node_id'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_32138 = df_feature.loc[df_feature['node_id']==31238]\n",
    "features_41908 = df_feature.loc[df_feature['node_id']==41908]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - 判断二个node的数据是否有不同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8641.000000\n",
       "mean        2.408029\n",
       "std         1.825553\n",
       "min         0.001786\n",
       "25%         0.743492\n",
       "50%         2.306323\n",
       "75%         3.631476\n",
       "max        10.305820\n",
       "Name: zone_flow_hist_0min, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_32138['zone_flow_hist_0min'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8641.000000\n",
       "mean        1.475214\n",
       "std         0.078918\n",
       "min         1.254849\n",
       "25%         1.421894\n",
       "50%         1.477509\n",
       "75%         1.528201\n",
       "max         1.734405\n",
       "Name: res_cl_hist_0min, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_32138['res_cl_hist_0min'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8.641000e+03\n",
       "mean     2.583436e+01\n",
       "std      3.048405e-12\n",
       "min      2.583436e+01\n",
       "25%      2.583436e+01\n",
       "50%      2.583436e+01\n",
       "75%      2.583436e+01\n",
       "max      2.583436e+01\n",
       "Name: travel_time_hrs_nom, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_41908['travel_time_hrs_nom'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    8.641000e+03\n",
       "mean     2.583436e+01\n",
       "std      3.048405e-12\n",
       "min      2.583436e+01\n",
       "25%      2.583436e+01\n",
       "50%      2.583436e+01\n",
       "75%      2.583436e+01\n",
       "max      2.583436e+01\n",
       "Name: travel_time_hrs_nom, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_41908['travel_time_hrs_nom'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_result_value</th>\n",
       "      <th>Average Result Value</th>\n",
       "      <th>zone_flow_hist_0min</th>\n",
       "      <th>zone_flow_hist_15min</th>\n",
       "      <th>zone_flow_hist_30min</th>\n",
       "      <th>zone_flow_hist_45min</th>\n",
       "      <th>zone_flow_hist_60min</th>\n",
       "      <th>zone_flow_hist_75min</th>\n",
       "      <th>zone_flow_hist_90min</th>\n",
       "      <th>zone_flow_hist_105min</th>\n",
       "      <th>...</th>\n",
       "      <th>res_cl_hist_2805min</th>\n",
       "      <th>res_cl_hist_2820min</th>\n",
       "      <th>res_cl_hist_2835min</th>\n",
       "      <th>res_cl_hist_2850min</th>\n",
       "      <th>res_cl_hist_2865min</th>\n",
       "      <th>res_cl_hist_2880min</th>\n",
       "      <th>res_temperature</th>\n",
       "      <th>travel_time_hrs_nom</th>\n",
       "      <th>flow_nom</th>\n",
       "      <th>node_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2.769000e+03</td>\n",
       "      <td>2.769000e+03</td>\n",
       "      <td>2769.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.276345</td>\n",
       "      <td>1.005778</td>\n",
       "      <td>1.152518</td>\n",
       "      <td>1.250574</td>\n",
       "      <td>1.352919</td>\n",
       "      <td>1.458963</td>\n",
       "      <td>1.567609</td>\n",
       "      <td>1.677540</td>\n",
       "      <td>1.789513</td>\n",
       "      <td>1.904766</td>\n",
       "      <td>...</td>\n",
       "      <td>1.477679</td>\n",
       "      <td>1.476071</td>\n",
       "      <td>1.474476</td>\n",
       "      <td>1.472915</td>\n",
       "      <td>1.471396</td>\n",
       "      <td>1.469897</td>\n",
       "      <td>21.725208</td>\n",
       "      <td>2.789096e+00</td>\n",
       "      <td>1.931739e+00</td>\n",
       "      <td>31238.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.139701</td>\n",
       "      <td>0.075809</td>\n",
       "      <td>1.743221</td>\n",
       "      <td>1.821041</td>\n",
       "      <td>1.887563</td>\n",
       "      <td>1.944308</td>\n",
       "      <td>1.992200</td>\n",
       "      <td>2.030488</td>\n",
       "      <td>2.059319</td>\n",
       "      <td>2.079790</td>\n",
       "      <td>...</td>\n",
       "      <td>0.081023</td>\n",
       "      <td>0.081313</td>\n",
       "      <td>0.081565</td>\n",
       "      <td>0.081755</td>\n",
       "      <td>0.081872</td>\n",
       "      <td>0.081920</td>\n",
       "      <td>1.888641</td>\n",
       "      <td>1.243674e-13</td>\n",
       "      <td>1.352496e-13</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.745000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.002122</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>...</td>\n",
       "      <td>1.261906</td>\n",
       "      <td>1.258727</td>\n",
       "      <td>1.256837</td>\n",
       "      <td>1.255174</td>\n",
       "      <td>1.254849</td>\n",
       "      <td>1.254849</td>\n",
       "      <td>17.400000</td>\n",
       "      <td>2.789096e+00</td>\n",
       "      <td>1.931739e+00</td>\n",
       "      <td>31238.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.197000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.163542</td>\n",
       "      <td>0.174694</td>\n",
       "      <td>0.193354</td>\n",
       "      <td>0.214630</td>\n",
       "      <td>0.239795</td>\n",
       "      <td>0.274978</td>\n",
       "      <td>0.312583</td>\n",
       "      <td>0.353525</td>\n",
       "      <td>...</td>\n",
       "      <td>1.426848</td>\n",
       "      <td>1.424709</td>\n",
       "      <td>1.422247</td>\n",
       "      <td>1.421027</td>\n",
       "      <td>1.419759</td>\n",
       "      <td>1.418278</td>\n",
       "      <td>20.650000</td>\n",
       "      <td>2.789096e+00</td>\n",
       "      <td>1.931739e+00</td>\n",
       "      <td>31238.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.290200</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.468314</td>\n",
       "      <td>0.516030</td>\n",
       "      <td>0.603306</td>\n",
       "      <td>0.690759</td>\n",
       "      <td>0.781015</td>\n",
       "      <td>0.892549</td>\n",
       "      <td>1.011893</td>\n",
       "      <td>1.117621</td>\n",
       "      <td>...</td>\n",
       "      <td>1.478592</td>\n",
       "      <td>1.476838</td>\n",
       "      <td>1.475486</td>\n",
       "      <td>1.474543</td>\n",
       "      <td>1.473251</td>\n",
       "      <td>1.471740</td>\n",
       "      <td>22.350000</td>\n",
       "      <td>2.789096e+00</td>\n",
       "      <td>1.931739e+00</td>\n",
       "      <td>31238.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.369600</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.315512</td>\n",
       "      <td>1.467496</td>\n",
       "      <td>1.619548</td>\n",
       "      <td>1.790862</td>\n",
       "      <td>2.010682</td>\n",
       "      <td>2.266233</td>\n",
       "      <td>2.510676</td>\n",
       "      <td>2.750851</td>\n",
       "      <td>...</td>\n",
       "      <td>1.526162</td>\n",
       "      <td>1.524947</td>\n",
       "      <td>1.523480</td>\n",
       "      <td>1.522765</td>\n",
       "      <td>1.521777</td>\n",
       "      <td>1.520649</td>\n",
       "      <td>22.950000</td>\n",
       "      <td>2.789096e+00</td>\n",
       "      <td>1.931739e+00</td>\n",
       "      <td>31238.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.617200</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>...</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>24.550000</td>\n",
       "      <td>2.789096e+00</td>\n",
       "      <td>1.931739e+00</td>\n",
       "      <td>31238.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 392 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       average_result_value  Average Result Value  zone_flow_hist_0min  \\\n",
       "count           2769.000000           2769.000000          2769.000000   \n",
       "mean               1.276345              1.005778             1.152518   \n",
       "std                0.139701              0.075809             1.743221   \n",
       "min                0.745000              1.000000             0.001786   \n",
       "25%                1.197000              1.000000             0.163542   \n",
       "50%                1.290200              1.000000             0.468314   \n",
       "75%                1.369600              1.000000             1.315512   \n",
       "max                1.617200              2.000000            10.305820   \n",
       "\n",
       "       zone_flow_hist_15min  zone_flow_hist_30min  zone_flow_hist_45min  \\\n",
       "count           2769.000000           2769.000000           2769.000000   \n",
       "mean               1.250574              1.352919              1.458963   \n",
       "std                1.821041              1.887563              1.944308   \n",
       "min                0.001786              0.001786              0.001786   \n",
       "25%                0.174694              0.193354              0.214630   \n",
       "50%                0.516030              0.603306              0.690759   \n",
       "75%                1.467496              1.619548              1.790862   \n",
       "max               10.305820             10.305820             10.305820   \n",
       "\n",
       "       zone_flow_hist_60min  zone_flow_hist_75min  zone_flow_hist_90min  \\\n",
       "count           2769.000000           2769.000000           2769.000000   \n",
       "mean               1.567609              1.677540              1.789513   \n",
       "std                1.992200              2.030488              2.059319   \n",
       "min                0.001786              0.001786              0.002122   \n",
       "25%                0.239795              0.274978              0.312583   \n",
       "50%                0.781015              0.892549              1.011893   \n",
       "75%                2.010682              2.266233              2.510676   \n",
       "max               10.305820             10.305820             10.305820   \n",
       "\n",
       "       zone_flow_hist_105min   ...     res_cl_hist_2805min  \\\n",
       "count            2769.000000   ...             2769.000000   \n",
       "mean                1.904766   ...                1.477679   \n",
       "std                 2.079790   ...                0.081023   \n",
       "min                 0.002476   ...                1.261906   \n",
       "25%                 0.353525   ...                1.426848   \n",
       "50%                 1.117621   ...                1.478592   \n",
       "75%                 2.750851   ...                1.526162   \n",
       "max                10.305820   ...                1.734405   \n",
       "\n",
       "       res_cl_hist_2820min  res_cl_hist_2835min  res_cl_hist_2850min  \\\n",
       "count          2769.000000          2769.000000          2769.000000   \n",
       "mean              1.476071             1.474476             1.472915   \n",
       "std               0.081313             0.081565             0.081755   \n",
       "min               1.258727             1.256837             1.255174   \n",
       "25%               1.424709             1.422247             1.421027   \n",
       "50%               1.476838             1.475486             1.474543   \n",
       "75%               1.524947             1.523480             1.522765   \n",
       "max               1.734405             1.734405             1.734405   \n",
       "\n",
       "       res_cl_hist_2865min  res_cl_hist_2880min  res_temperature  \\\n",
       "count          2769.000000          2769.000000      2769.000000   \n",
       "mean              1.471396             1.469897        21.725208   \n",
       "std               0.081872             0.081920         1.888641   \n",
       "min               1.254849             1.254849        17.400000   \n",
       "25%               1.419759             1.418278        20.650000   \n",
       "50%               1.473251             1.471740        22.350000   \n",
       "75%               1.521777             1.520649        22.950000   \n",
       "max               1.734405             1.734405        24.550000   \n",
       "\n",
       "       travel_time_hrs_nom      flow_nom  node_id  \n",
       "count         2.769000e+03  2.769000e+03   2769.0  \n",
       "mean          2.789096e+00  1.931739e+00  31238.0  \n",
       "std           1.243674e-13  1.352496e-13      0.0  \n",
       "min           2.789096e+00  1.931739e+00  31238.0  \n",
       "25%           2.789096e+00  1.931739e+00  31238.0  \n",
       "50%           2.789096e+00  1.931739e+00  31238.0  \n",
       "75%           2.789096e+00  1.931739e+00  31238.0  \n",
       "max           2.789096e+00  1.931739e+00  31238.0  \n",
       "\n",
       "[8 rows x 392 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2769, 399)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* average_result_value: label\n",
    "* Average Result Value: pump speed - sum 3 nodes\n",
    "* zone_flow_hist_0min: flow speed (record in every 15mins - total: 2 days)\n",
    "* res_cl_hist_0min: upstream result history of chlorine (record in every 15mins - total: 2 days)\n",
    "* res_temperature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2769, 389)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_droped = ['zone_id', 'node_id', 'timestamp', 'flow_nom', 'travel_time_hrs_nom', 'Measurement_Result_Date_Minute_15', 'facility_code', 'asset_code', 'statistic_type', 'measurement_result_date']\n",
    "df_tem = df.drop(feature_droped ,axis=1)\n",
    "df_tem.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_result_value</th>\n",
       "      <th>Average Result Value</th>\n",
       "      <th>zone_flow_hist_0min</th>\n",
       "      <th>zone_flow_hist_15min</th>\n",
       "      <th>zone_flow_hist_30min</th>\n",
       "      <th>zone_flow_hist_45min</th>\n",
       "      <th>zone_flow_hist_60min</th>\n",
       "      <th>zone_flow_hist_75min</th>\n",
       "      <th>zone_flow_hist_90min</th>\n",
       "      <th>zone_flow_hist_105min</th>\n",
       "      <th>...</th>\n",
       "      <th>res_cl_hist_2760min</th>\n",
       "      <th>res_cl_hist_2775min</th>\n",
       "      <th>res_cl_hist_2790min</th>\n",
       "      <th>res_cl_hist_2805min</th>\n",
       "      <th>res_cl_hist_2820min</th>\n",
       "      <th>res_cl_hist_2835min</th>\n",
       "      <th>res_cl_hist_2850min</th>\n",
       "      <th>res_cl_hist_2865min</th>\n",
       "      <th>res_cl_hist_2880min</th>\n",
       "      <th>res_temperature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.4572</td>\n",
       "      <td>1</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>1.529712</td>\n",
       "      <td>...</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>1.708366</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.4560</td>\n",
       "      <td>1</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>...</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.4540</td>\n",
       "      <td>1</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>...</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.4432</td>\n",
       "      <td>1</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>...</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.4306</td>\n",
       "      <td>1</td>\n",
       "      <td>0.172979</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>...</td>\n",
       "      <td>1.731583</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 389 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   average_result_value  Average Result Value  zone_flow_hist_0min  \\\n",
       "0                1.4572                     1             0.537967   \n",
       "1                1.4560                     1             0.374730   \n",
       "2                1.4540                     1             0.228371   \n",
       "3                1.4432                     1             0.169935   \n",
       "4                1.4306                     1             0.172979   \n",
       "\n",
       "   zone_flow_hist_15min  zone_flow_hist_30min  zone_flow_hist_45min  \\\n",
       "0              0.677703              0.838600              0.992196   \n",
       "1              0.537967              0.677703              0.838600   \n",
       "2              0.374730              0.537967              0.677703   \n",
       "3              0.228371              0.374730              0.537967   \n",
       "4              0.169935              0.228371              0.374730   \n",
       "\n",
       "   zone_flow_hist_60min  zone_flow_hist_75min  zone_flow_hist_90min  \\\n",
       "0              1.086506              1.151700              1.272814   \n",
       "1              0.992196              1.086506              1.151700   \n",
       "2              0.838600              0.992196              1.086506   \n",
       "3              0.677703              0.838600              0.992196   \n",
       "4              0.537967              0.677703              0.838600   \n",
       "\n",
       "   zone_flow_hist_105min       ...         res_cl_hist_2760min  \\\n",
       "0               1.529712       ...                    1.723967   \n",
       "1               1.272814       ...                    1.727027   \n",
       "2               1.151700       ...                    1.729251   \n",
       "3               1.086506       ...                    1.730591   \n",
       "4               0.992196       ...                    1.731583   \n",
       "\n",
       "   res_cl_hist_2775min  res_cl_hist_2790min  res_cl_hist_2805min  \\\n",
       "0             1.721563             1.720893             1.718593   \n",
       "1             1.723967             1.721563             1.720893   \n",
       "2             1.727027             1.723967             1.721563   \n",
       "3             1.729251             1.727027             1.723967   \n",
       "4             1.730591             1.729251             1.727027   \n",
       "\n",
       "   res_cl_hist_2820min  res_cl_hist_2835min  res_cl_hist_2850min  \\\n",
       "0             1.714862             1.711658             1.709118   \n",
       "1             1.718593             1.714862             1.711658   \n",
       "2             1.720893             1.718593             1.714862   \n",
       "3             1.721563             1.720893             1.718593   \n",
       "4             1.723967             1.721563             1.720893   \n",
       "\n",
       "   res_cl_hist_2865min  res_cl_hist_2880min  res_temperature  \n",
       "0             1.707396             1.708366             17.4  \n",
       "1             1.709118             1.707396             17.4  \n",
       "2             1.711658             1.709118             17.4  \n",
       "3             1.714862             1.711658             17.4  \n",
       "4             1.718593             1.714862             17.4  \n",
       "\n",
       "[5 rows x 389 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tem.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## output training data\n",
    "* without timestamp\n",
    "* 18/09/2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tem.to_csv(modelling_data_directory, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_result_value</th>\n",
       "      <th>Average Result Value</th>\n",
       "      <th>zone_flow_hist_0min</th>\n",
       "      <th>zone_flow_hist_15min</th>\n",
       "      <th>zone_flow_hist_30min</th>\n",
       "      <th>zone_flow_hist_45min</th>\n",
       "      <th>zone_flow_hist_60min</th>\n",
       "      <th>zone_flow_hist_75min</th>\n",
       "      <th>zone_flow_hist_90min</th>\n",
       "      <th>zone_flow_hist_105min</th>\n",
       "      <th>...</th>\n",
       "      <th>res_cl_hist_2760min</th>\n",
       "      <th>res_cl_hist_2775min</th>\n",
       "      <th>res_cl_hist_2790min</th>\n",
       "      <th>res_cl_hist_2805min</th>\n",
       "      <th>res_cl_hist_2820min</th>\n",
       "      <th>res_cl_hist_2835min</th>\n",
       "      <th>res_cl_hist_2850min</th>\n",
       "      <th>res_cl_hist_2865min</th>\n",
       "      <th>res_cl_hist_2880min</th>\n",
       "      <th>res_temperature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "      <td>2769.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.276345</td>\n",
       "      <td>1.005778</td>\n",
       "      <td>1.152518</td>\n",
       "      <td>1.250574</td>\n",
       "      <td>1.352919</td>\n",
       "      <td>1.458963</td>\n",
       "      <td>1.567609</td>\n",
       "      <td>1.677540</td>\n",
       "      <td>1.789513</td>\n",
       "      <td>1.904766</td>\n",
       "      <td>...</td>\n",
       "      <td>1.482490</td>\n",
       "      <td>1.480896</td>\n",
       "      <td>1.479288</td>\n",
       "      <td>1.477679</td>\n",
       "      <td>1.476071</td>\n",
       "      <td>1.474476</td>\n",
       "      <td>1.472915</td>\n",
       "      <td>1.471396</td>\n",
       "      <td>1.469897</td>\n",
       "      <td>21.725208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.139701</td>\n",
       "      <td>0.075809</td>\n",
       "      <td>1.743221</td>\n",
       "      <td>1.821041</td>\n",
       "      <td>1.887563</td>\n",
       "      <td>1.944308</td>\n",
       "      <td>1.992200</td>\n",
       "      <td>2.030488</td>\n",
       "      <td>2.059319</td>\n",
       "      <td>2.079790</td>\n",
       "      <td>...</td>\n",
       "      <td>0.080038</td>\n",
       "      <td>0.080380</td>\n",
       "      <td>0.080711</td>\n",
       "      <td>0.081023</td>\n",
       "      <td>0.081313</td>\n",
       "      <td>0.081565</td>\n",
       "      <td>0.081755</td>\n",
       "      <td>0.081872</td>\n",
       "      <td>0.081920</td>\n",
       "      <td>1.888641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.745000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.001786</td>\n",
       "      <td>0.002122</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>...</td>\n",
       "      <td>1.275442</td>\n",
       "      <td>1.269354</td>\n",
       "      <td>1.265012</td>\n",
       "      <td>1.261906</td>\n",
       "      <td>1.258727</td>\n",
       "      <td>1.256837</td>\n",
       "      <td>1.255174</td>\n",
       "      <td>1.254849</td>\n",
       "      <td>1.254849</td>\n",
       "      <td>17.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.197000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.163542</td>\n",
       "      <td>0.174694</td>\n",
       "      <td>0.193354</td>\n",
       "      <td>0.214630</td>\n",
       "      <td>0.239795</td>\n",
       "      <td>0.274978</td>\n",
       "      <td>0.312583</td>\n",
       "      <td>0.353525</td>\n",
       "      <td>...</td>\n",
       "      <td>1.432245</td>\n",
       "      <td>1.430814</td>\n",
       "      <td>1.429068</td>\n",
       "      <td>1.426848</td>\n",
       "      <td>1.424709</td>\n",
       "      <td>1.422247</td>\n",
       "      <td>1.421027</td>\n",
       "      <td>1.419759</td>\n",
       "      <td>1.418278</td>\n",
       "      <td>20.650000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.290200</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.468314</td>\n",
       "      <td>0.516030</td>\n",
       "      <td>0.603306</td>\n",
       "      <td>0.690759</td>\n",
       "      <td>0.781015</td>\n",
       "      <td>0.892549</td>\n",
       "      <td>1.011893</td>\n",
       "      <td>1.117621</td>\n",
       "      <td>...</td>\n",
       "      <td>1.484488</td>\n",
       "      <td>1.482226</td>\n",
       "      <td>1.480484</td>\n",
       "      <td>1.478592</td>\n",
       "      <td>1.476838</td>\n",
       "      <td>1.475486</td>\n",
       "      <td>1.474543</td>\n",
       "      <td>1.473251</td>\n",
       "      <td>1.471740</td>\n",
       "      <td>22.350000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.369600</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.315512</td>\n",
       "      <td>1.467496</td>\n",
       "      <td>1.619548</td>\n",
       "      <td>1.790862</td>\n",
       "      <td>2.010682</td>\n",
       "      <td>2.266233</td>\n",
       "      <td>2.510676</td>\n",
       "      <td>2.750851</td>\n",
       "      <td>...</td>\n",
       "      <td>1.531268</td>\n",
       "      <td>1.529775</td>\n",
       "      <td>1.527973</td>\n",
       "      <td>1.526162</td>\n",
       "      <td>1.524947</td>\n",
       "      <td>1.523480</td>\n",
       "      <td>1.522765</td>\n",
       "      <td>1.521777</td>\n",
       "      <td>1.520649</td>\n",
       "      <td>22.950000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.617200</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>10.305820</td>\n",
       "      <td>...</td>\n",
       "      <td>1.735576</td>\n",
       "      <td>1.735576</td>\n",
       "      <td>1.735576</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>1.734405</td>\n",
       "      <td>24.550000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 389 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       average_result_value  Average Result Value  zone_flow_hist_0min  \\\n",
       "count           2769.000000           2769.000000          2769.000000   \n",
       "mean               1.276345              1.005778             1.152518   \n",
       "std                0.139701              0.075809             1.743221   \n",
       "min                0.745000              1.000000             0.001786   \n",
       "25%                1.197000              1.000000             0.163542   \n",
       "50%                1.290200              1.000000             0.468314   \n",
       "75%                1.369600              1.000000             1.315512   \n",
       "max                1.617200              2.000000            10.305820   \n",
       "\n",
       "       zone_flow_hist_15min  zone_flow_hist_30min  zone_flow_hist_45min  \\\n",
       "count           2769.000000           2769.000000           2769.000000   \n",
       "mean               1.250574              1.352919              1.458963   \n",
       "std                1.821041              1.887563              1.944308   \n",
       "min                0.001786              0.001786              0.001786   \n",
       "25%                0.174694              0.193354              0.214630   \n",
       "50%                0.516030              0.603306              0.690759   \n",
       "75%                1.467496              1.619548              1.790862   \n",
       "max               10.305820             10.305820             10.305820   \n",
       "\n",
       "       zone_flow_hist_60min  zone_flow_hist_75min  zone_flow_hist_90min  \\\n",
       "count           2769.000000           2769.000000           2769.000000   \n",
       "mean               1.567609              1.677540              1.789513   \n",
       "std                1.992200              2.030488              2.059319   \n",
       "min                0.001786              0.001786              0.002122   \n",
       "25%                0.239795              0.274978              0.312583   \n",
       "50%                0.781015              0.892549              1.011893   \n",
       "75%                2.010682              2.266233              2.510676   \n",
       "max               10.305820             10.305820             10.305820   \n",
       "\n",
       "       zone_flow_hist_105min       ...         res_cl_hist_2760min  \\\n",
       "count            2769.000000       ...                 2769.000000   \n",
       "mean                1.904766       ...                    1.482490   \n",
       "std                 2.079790       ...                    0.080038   \n",
       "min                 0.002476       ...                    1.275442   \n",
       "25%                 0.353525       ...                    1.432245   \n",
       "50%                 1.117621       ...                    1.484488   \n",
       "75%                 2.750851       ...                    1.531268   \n",
       "max                10.305820       ...                    1.735576   \n",
       "\n",
       "       res_cl_hist_2775min  res_cl_hist_2790min  res_cl_hist_2805min  \\\n",
       "count          2769.000000          2769.000000          2769.000000   \n",
       "mean              1.480896             1.479288             1.477679   \n",
       "std               0.080380             0.080711             0.081023   \n",
       "min               1.269354             1.265012             1.261906   \n",
       "25%               1.430814             1.429068             1.426848   \n",
       "50%               1.482226             1.480484             1.478592   \n",
       "75%               1.529775             1.527973             1.526162   \n",
       "max               1.735576             1.735576             1.734405   \n",
       "\n",
       "       res_cl_hist_2820min  res_cl_hist_2835min  res_cl_hist_2850min  \\\n",
       "count          2769.000000          2769.000000          2769.000000   \n",
       "mean              1.476071             1.474476             1.472915   \n",
       "std               0.081313             0.081565             0.081755   \n",
       "min               1.258727             1.256837             1.255174   \n",
       "25%               1.424709             1.422247             1.421027   \n",
       "50%               1.476838             1.475486             1.474543   \n",
       "75%               1.524947             1.523480             1.522765   \n",
       "max               1.734405             1.734405             1.734405   \n",
       "\n",
       "       res_cl_hist_2865min  res_cl_hist_2880min  res_temperature  \n",
       "count          2769.000000          2769.000000      2769.000000  \n",
       "mean              1.471396             1.469897        21.725208  \n",
       "std               0.081872             0.081920         1.888641  \n",
       "min               1.254849             1.254849        17.400000  \n",
       "25%               1.419759             1.418278        20.650000  \n",
       "50%               1.473251             1.471740        22.350000  \n",
       "75%               1.521777             1.520649        22.950000  \n",
       "max               1.734405             1.734405        24.550000  \n",
       "\n",
       "[8 rows x 389 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tem.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2769 entries, 0 to 2768\n",
      "Columns: 389 entries, average_result_value to res_temperature\n",
      "dtypes: float64(388), int64(1)\n",
      "memory usage: 8.2 MB\n"
     ]
    }
   ],
   "source": [
    "df_tem.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_result_value</th>\n",
       "      <th>Average Result Value</th>\n",
       "      <th>zone_flow_hist_0min</th>\n",
       "      <th>zone_flow_hist_15min</th>\n",
       "      <th>zone_flow_hist_30min</th>\n",
       "      <th>zone_flow_hist_45min</th>\n",
       "      <th>zone_flow_hist_60min</th>\n",
       "      <th>zone_flow_hist_75min</th>\n",
       "      <th>zone_flow_hist_90min</th>\n",
       "      <th>zone_flow_hist_105min</th>\n",
       "      <th>...</th>\n",
       "      <th>res_cl_hist_2760min</th>\n",
       "      <th>res_cl_hist_2775min</th>\n",
       "      <th>res_cl_hist_2790min</th>\n",
       "      <th>res_cl_hist_2805min</th>\n",
       "      <th>res_cl_hist_2820min</th>\n",
       "      <th>res_cl_hist_2835min</th>\n",
       "      <th>res_cl_hist_2850min</th>\n",
       "      <th>res_cl_hist_2865min</th>\n",
       "      <th>res_cl_hist_2880min</th>\n",
       "      <th>res_temperature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.4572</td>\n",
       "      <td>1</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>1.529712</td>\n",
       "      <td>...</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>1.708366</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.4560</td>\n",
       "      <td>1</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>...</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.4540</td>\n",
       "      <td>1</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>...</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.4432</td>\n",
       "      <td>1</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>...</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.4306</td>\n",
       "      <td>1</td>\n",
       "      <td>0.172979</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>...</td>\n",
       "      <td>1.731583</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.4292</td>\n",
       "      <td>1</td>\n",
       "      <td>0.189868</td>\n",
       "      <td>0.172979</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>...</td>\n",
       "      <td>1.731413</td>\n",
       "      <td>1.731583</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>17.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 389 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   average_result_value  Average Result Value  zone_flow_hist_0min  \\\n",
       "0                1.4572                     1             0.537967   \n",
       "1                1.4560                     1             0.374730   \n",
       "2                1.4540                     1             0.228371   \n",
       "3                1.4432                     1             0.169935   \n",
       "4                1.4306                     1             0.172979   \n",
       "5                1.4292                     1             0.189868   \n",
       "\n",
       "   zone_flow_hist_15min  zone_flow_hist_30min  zone_flow_hist_45min  \\\n",
       "0              0.677703              0.838600              0.992196   \n",
       "1              0.537967              0.677703              0.838600   \n",
       "2              0.374730              0.537967              0.677703   \n",
       "3              0.228371              0.374730              0.537967   \n",
       "4              0.169935              0.228371              0.374730   \n",
       "5              0.172979              0.169935              0.228371   \n",
       "\n",
       "   zone_flow_hist_60min  zone_flow_hist_75min  zone_flow_hist_90min  \\\n",
       "0              1.086506              1.151700              1.272814   \n",
       "1              0.992196              1.086506              1.151700   \n",
       "2              0.838600              0.992196              1.086506   \n",
       "3              0.677703              0.838600              0.992196   \n",
       "4              0.537967              0.677703              0.838600   \n",
       "5              0.374730              0.537967              0.677703   \n",
       "\n",
       "   zone_flow_hist_105min       ...         res_cl_hist_2760min  \\\n",
       "0               1.529712       ...                    1.723967   \n",
       "1               1.272814       ...                    1.727027   \n",
       "2               1.151700       ...                    1.729251   \n",
       "3               1.086506       ...                    1.730591   \n",
       "4               0.992196       ...                    1.731583   \n",
       "5               0.838600       ...                    1.731413   \n",
       "\n",
       "   res_cl_hist_2775min  res_cl_hist_2790min  res_cl_hist_2805min  \\\n",
       "0             1.721563             1.720893             1.718593   \n",
       "1             1.723967             1.721563             1.720893   \n",
       "2             1.727027             1.723967             1.721563   \n",
       "3             1.729251             1.727027             1.723967   \n",
       "4             1.730591             1.729251             1.727027   \n",
       "5             1.731583             1.730591             1.729251   \n",
       "\n",
       "   res_cl_hist_2820min  res_cl_hist_2835min  res_cl_hist_2850min  \\\n",
       "0             1.714862             1.711658             1.709118   \n",
       "1             1.718593             1.714862             1.711658   \n",
       "2             1.720893             1.718593             1.714862   \n",
       "3             1.721563             1.720893             1.718593   \n",
       "4             1.723967             1.721563             1.720893   \n",
       "5             1.727027             1.723967             1.721563   \n",
       "\n",
       "   res_cl_hist_2865min  res_cl_hist_2880min  res_temperature  \n",
       "0             1.707396             1.708366             17.4  \n",
       "1             1.709118             1.707396             17.4  \n",
       "2             1.711658             1.709118             17.4  \n",
       "3             1.714862             1.711658             17.4  \n",
       "4             1.718593             1.714862             17.4  \n",
       "5             1.720893             1.718593             17.4  \n",
       "\n",
       "[6 rows x 389 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tem.head(6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Output diff training data\n",
    "* change label to diff value\n",
    "* 19/09/2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2766    0.0000\n",
       "2767    0.1630\n",
       "2768   -0.0038\n",
       "Name: average_result_value, dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_label_diff = df_tem['average_result_value'].diff()\n",
    "df_label_diff[2766:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2768.000000\n",
       "mean       -0.000079\n",
       "std         0.079448\n",
       "min        -0.462800\n",
       "25%        -0.010200\n",
       "50%        -0.003200\n",
       "75%         0.003800\n",
       "max         0.419000\n",
       "Name: average_result_value, dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_label_diff.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2769, 389)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tem_1 = df_tem.drop(['average_result_value'], axis=1)\n",
    "df_diff = pd.concat([df_tem_1, df_label_diff], axis=1)\n",
    "df_diff.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Average Result Value</th>\n",
       "      <th>zone_flow_hist_0min</th>\n",
       "      <th>zone_flow_hist_15min</th>\n",
       "      <th>zone_flow_hist_30min</th>\n",
       "      <th>zone_flow_hist_45min</th>\n",
       "      <th>zone_flow_hist_60min</th>\n",
       "      <th>zone_flow_hist_75min</th>\n",
       "      <th>zone_flow_hist_90min</th>\n",
       "      <th>zone_flow_hist_105min</th>\n",
       "      <th>zone_flow_hist_120min</th>\n",
       "      <th>...</th>\n",
       "      <th>res_cl_hist_2775min</th>\n",
       "      <th>res_cl_hist_2790min</th>\n",
       "      <th>res_cl_hist_2805min</th>\n",
       "      <th>res_cl_hist_2820min</th>\n",
       "      <th>res_cl_hist_2835min</th>\n",
       "      <th>res_cl_hist_2850min</th>\n",
       "      <th>res_cl_hist_2865min</th>\n",
       "      <th>res_cl_hist_2880min</th>\n",
       "      <th>res_temperature</th>\n",
       "      <th>average_result_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>1.529712</td>\n",
       "      <td>1.903710</td>\n",
       "      <td>...</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>1.708366</td>\n",
       "      <td>17.4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>1.529712</td>\n",
       "      <td>...</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>...</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>...</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.172979</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>...</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0126</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 389 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Average Result Value  zone_flow_hist_0min  zone_flow_hist_15min  \\\n",
       "0                     1             0.537967              0.677703   \n",
       "1                     1             0.374730              0.537967   \n",
       "2                     1             0.228371              0.374730   \n",
       "3                     1             0.169935              0.228371   \n",
       "4                     1             0.172979              0.169935   \n",
       "\n",
       "   zone_flow_hist_30min  zone_flow_hist_45min  zone_flow_hist_60min  \\\n",
       "0              0.838600              0.992196              1.086506   \n",
       "1              0.677703              0.838600              0.992196   \n",
       "2              0.537967              0.677703              0.838600   \n",
       "3              0.374730              0.537967              0.677703   \n",
       "4              0.228371              0.374730              0.537967   \n",
       "\n",
       "   zone_flow_hist_75min  zone_flow_hist_90min  zone_flow_hist_105min  \\\n",
       "0              1.151700              1.272814               1.529712   \n",
       "1              1.086506              1.151700               1.272814   \n",
       "2              0.992196              1.086506               1.151700   \n",
       "3              0.838600              0.992196               1.086506   \n",
       "4              0.677703              0.838600               0.992196   \n",
       "\n",
       "   zone_flow_hist_120min          ...           res_cl_hist_2775min  \\\n",
       "0               1.903710          ...                      1.721563   \n",
       "1               1.529712          ...                      1.723967   \n",
       "2               1.272814          ...                      1.727027   \n",
       "3               1.151700          ...                      1.729251   \n",
       "4               1.086506          ...                      1.730591   \n",
       "\n",
       "   res_cl_hist_2790min  res_cl_hist_2805min  res_cl_hist_2820min  \\\n",
       "0             1.720893             1.718593             1.714862   \n",
       "1             1.721563             1.720893             1.718593   \n",
       "2             1.723967             1.721563             1.720893   \n",
       "3             1.727027             1.723967             1.721563   \n",
       "4             1.729251             1.727027             1.723967   \n",
       "\n",
       "   res_cl_hist_2835min  res_cl_hist_2850min  res_cl_hist_2865min  \\\n",
       "0             1.711658             1.709118             1.707396   \n",
       "1             1.714862             1.711658             1.709118   \n",
       "2             1.718593             1.714862             1.711658   \n",
       "3             1.720893             1.718593             1.714862   \n",
       "4             1.721563             1.720893             1.718593   \n",
       "\n",
       "   res_cl_hist_2880min  res_temperature  average_result_value  \n",
       "0             1.708366             17.4                   NaN  \n",
       "1             1.707396             17.4               -0.0012  \n",
       "2             1.709118             17.4               -0.0020  \n",
       "3             1.711658             17.4               -0.0108  \n",
       "4             1.714862             17.4               -0.0126  \n",
       "\n",
       "[5 rows x 389 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diff.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diff.drop([0], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Average Result Value</th>\n",
       "      <th>zone_flow_hist_0min</th>\n",
       "      <th>zone_flow_hist_15min</th>\n",
       "      <th>zone_flow_hist_30min</th>\n",
       "      <th>zone_flow_hist_45min</th>\n",
       "      <th>zone_flow_hist_60min</th>\n",
       "      <th>zone_flow_hist_75min</th>\n",
       "      <th>zone_flow_hist_90min</th>\n",
       "      <th>zone_flow_hist_105min</th>\n",
       "      <th>zone_flow_hist_120min</th>\n",
       "      <th>...</th>\n",
       "      <th>res_cl_hist_2775min</th>\n",
       "      <th>res_cl_hist_2790min</th>\n",
       "      <th>res_cl_hist_2805min</th>\n",
       "      <th>res_cl_hist_2820min</th>\n",
       "      <th>res_cl_hist_2835min</th>\n",
       "      <th>res_cl_hist_2850min</th>\n",
       "      <th>res_cl_hist_2865min</th>\n",
       "      <th>res_cl_hist_2880min</th>\n",
       "      <th>res_temperature</th>\n",
       "      <th>average_result_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>1.529712</td>\n",
       "      <td>...</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>1.707396</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>1.272814</td>\n",
       "      <td>...</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>1.709118</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>1.151700</td>\n",
       "      <td>...</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>1.711658</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.172979</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>1.086506</td>\n",
       "      <td>...</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>1.714862</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0.189868</td>\n",
       "      <td>0.172979</td>\n",
       "      <td>0.169935</td>\n",
       "      <td>0.228371</td>\n",
       "      <td>0.374730</td>\n",
       "      <td>0.537967</td>\n",
       "      <td>0.677703</td>\n",
       "      <td>0.838600</td>\n",
       "      <td>0.992196</td>\n",
       "      <td>...</td>\n",
       "      <td>1.731583</td>\n",
       "      <td>1.730591</td>\n",
       "      <td>1.729251</td>\n",
       "      <td>1.727027</td>\n",
       "      <td>1.723967</td>\n",
       "      <td>1.721563</td>\n",
       "      <td>1.720893</td>\n",
       "      <td>1.718593</td>\n",
       "      <td>17.4</td>\n",
       "      <td>-0.0014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 389 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Average Result Value  zone_flow_hist_0min  zone_flow_hist_15min  \\\n",
       "1                     1             0.374730              0.537967   \n",
       "2                     1             0.228371              0.374730   \n",
       "3                     1             0.169935              0.228371   \n",
       "4                     1             0.172979              0.169935   \n",
       "5                     1             0.189868              0.172979   \n",
       "\n",
       "   zone_flow_hist_30min  zone_flow_hist_45min  zone_flow_hist_60min  \\\n",
       "1              0.677703              0.838600              0.992196   \n",
       "2              0.537967              0.677703              0.838600   \n",
       "3              0.374730              0.537967              0.677703   \n",
       "4              0.228371              0.374730              0.537967   \n",
       "5              0.169935              0.228371              0.374730   \n",
       "\n",
       "   zone_flow_hist_75min  zone_flow_hist_90min  zone_flow_hist_105min  \\\n",
       "1              1.086506              1.151700               1.272814   \n",
       "2              0.992196              1.086506               1.151700   \n",
       "3              0.838600              0.992196               1.086506   \n",
       "4              0.677703              0.838600               0.992196   \n",
       "5              0.537967              0.677703               0.838600   \n",
       "\n",
       "   zone_flow_hist_120min          ...           res_cl_hist_2775min  \\\n",
       "1               1.529712          ...                      1.723967   \n",
       "2               1.272814          ...                      1.727027   \n",
       "3               1.151700          ...                      1.729251   \n",
       "4               1.086506          ...                      1.730591   \n",
       "5               0.992196          ...                      1.731583   \n",
       "\n",
       "   res_cl_hist_2790min  res_cl_hist_2805min  res_cl_hist_2820min  \\\n",
       "1             1.721563             1.720893             1.718593   \n",
       "2             1.723967             1.721563             1.720893   \n",
       "3             1.727027             1.723967             1.721563   \n",
       "4             1.729251             1.727027             1.723967   \n",
       "5             1.730591             1.729251             1.727027   \n",
       "\n",
       "   res_cl_hist_2835min  res_cl_hist_2850min  res_cl_hist_2865min  \\\n",
       "1             1.714862             1.711658             1.709118   \n",
       "2             1.718593             1.714862             1.711658   \n",
       "3             1.720893             1.718593             1.714862   \n",
       "4             1.721563             1.720893             1.718593   \n",
       "5             1.723967             1.721563             1.720893   \n",
       "\n",
       "   res_cl_hist_2880min  res_temperature  average_result_value  \n",
       "1             1.707396             17.4               -0.0012  \n",
       "2             1.709118             17.4               -0.0020  \n",
       "3             1.711658             17.4               -0.0108  \n",
       "4             1.714862             17.4               -0.0126  \n",
       "5             1.718593             17.4               -0.0014  \n",
       "\n",
       "[5 rows x 389 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_diff.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_diff.to_csv(modelling_diff_data_directory, index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}