1_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing of wind turbine data\n",
    "\n",
    "Cells in this notebook can be run independently provided the first two (\"Imports\" and \"Preprocessing function...\") are run first.\n",
    "\n",
    "1. [Imports.](#Cell1)\n",
    "1. [Preprocessing function, good for training data as well as testing data.](#Cell2)\n",
    "1. [Preprocess the training data set, output a pickle file.](#Cell3)\n",
    "1. [Preprocess the training data set in Feather format for notebooks running R kernel.](#Cell4)\n",
    "1. [Preprocess the testing data set, output a pickle file.](#Cell5)\n",
    "1. [Preprocess the testing data set in Feather format for notebooks running R kernel.](#Cell6)\n",
    "1. [Preprocess the solution file (the testing data set with TARGETVAR included). Output in pickle and Feather.](#Cell7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Cell1'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Imports.\n",
    "'''\n",
    "import feather\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from time import strftime\n",
    "import datetime\n",
    "from IPython.display import display_html\n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_rows\", None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Cell2'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "Preprocessing function, good for training data as well as testing data.\n",
    "'''\n",
    "def preProcess(fname, Nroll=3, inclusive=False, keepID=False):\n",
    "    '''\n",
    "    Read in the wind turbine data, store in a Pandas dataframe, and rearrange.\n",
    "\n",
    "    1.  Records in the input csv file are assumed to have the following form:\n",
    "            ID, ZONEID, TIMESTAMP, TARGETVAR, U10, V10, U100, V100\n",
    "        where:\n",
    "            ID =         Unique ID for each observation\n",
    "            ZONEID =     Zone/Turbine ID, a number between 1 and 10\n",
    "            TIMESTAMP =  Time of observation, in the format \"YYYYMMDD h:mm\" or \"YYYYMMDD hh:mm\"\n",
    "            TARGETVAR =  Wind turbine output (only present in the training data set)\n",
    "            U10 =        Zonal Wind Vector at 10 m\n",
    "            V10 =        Meridional Wind Vector at 10 m\n",
    "            U100 =       Zonal Wind vector at 100 m\n",
    "            V100 =       Meridional Wind vector at 100 m\n",
    "    \n",
    "    2. If Nroll<=1, column names in the output data frame have the form:\n",
    "            TARGETVAR U10 V10 U100 V100 YEAR DAYOFYEAR HOUR\n",
    "       and TARGETVAR, U10, V10, U100, V100 each have a subindex 1 ... 10 indicating zone id.\n",
    "       \n",
    "       If Nroll>1, the structure is the same, but the U10, V10, U100, V100 columns are replaced\n",
    "       by rolling means over Nroll measurements, and the names are U10_rmX, V10_rmX, U100_rmX, \n",
    "       and V100_rmX, where X equals Nroll.\n",
    "       \n",
    "       If Nroll>1 and inclusive==True, all rolling means from 1 to Nroll are kept in the \n",
    "       dataframe. If inclusive==False, only the rolling mean corresponding to Nroll is kept.\n",
    "    '''\n",
    "\n",
    "    # Read in the windmill data, parsing the third column as datetimes. Make a hierarchical\n",
    "    # index out of the timestamp and zone id columns.  Then drop the measurement ID column.\n",
    "    df0 = pd.read_csv(fname, header=0, parse_dates=[2], index_col=[2,1])\n",
    "    if not keepID: \n",
    "        df0 = df0.drop('ID', 1)\n",
    "        print('ID column dropped')\n",
    "    else:\n",
    "        print('ID column kept')\n",
    "        \n",
    "    # Unstack the inner level of the index. The index is thus reduced to the timestamp, and \n",
    "    # the zone ids become subcolumns within the existing columns. In other words, where in\n",
    "    # df0 there are 10 rows per time stamp, one for each zone id, in df1 there is only \n",
    "    # 1 row, which contains the measurements made at the given time stamp in all the zone ids.\n",
    "    df1 = df0.unstack()\n",
    "\n",
    "    # Put the timestamp index back as a dataframe column and verify that all delta-times are one hour.\n",
    "    df2              = df1.reset_index()\n",
    "    dtimes           = df2[\"TIMESTAMP\"].subtract(df2[\"TIMESTAMP\"].shift(+1))\n",
    "    Ntimepoints      = len(df2)\n",
    "    Ndeltatimes1h    = dtimes[dtimes==np.timedelta64(1, 'h')].count()\n",
    "    assert (Ntimepoints == Ndeltatimes1h+1), 'Delta times not all equal to 1 h!'\n",
    "    print('Number of time points:                 {0}'.format(Ntimepoints))\n",
    "    print('Number of delta-times equal to 1 hour: {0}'.format(Ndeltatimes1h))\n",
    "    \n",
    "    # Remove duplicate wind measurements (5 is duplicate with 4, and 8 with 7)\n",
    "    nturbines         = 10\n",
    "    wind_measurements = [1, 2, 3, 4, 6, 7, 9, 10]\n",
    "    for i in range(1,nturbines+1):\n",
    "        if i not in wind_measurements:\n",
    "            df2 = df2.drop([(\"U10\",i), (\"V10\",i), (\"U100\",i), (\"V100\",i)], axis=1)\n",
    "    \n",
    "    # Add rolling means of the wind measurements\n",
    "    if Nroll>1:\n",
    "        NrollSet = range(2, Nroll+1) if inclusive else [Nroll]\n",
    "        for n in NrollSet:\n",
    "            print('Computing rolling means over {0} measurements'.format(n))\n",
    "            dname = \"_rm\"+str(n)\n",
    "            for i in wind_measurements:\n",
    "                df2[(\"U10\"+dname,i)]  = df2[(\"U10\",i)].rolling(n, min_periods=1).mean()\n",
    "                df2[(\"U100\"+dname,i)] = df2[(\"U100\",i)].rolling(n, min_periods=1).mean()\n",
    "                df2[(\"V10\"+dname,i)]  = df2[(\"V10\",i)].rolling(n, min_periods=1).mean()\n",
    "                df2[(\"V100\"+dname,i)] = df2[(\"V100\",i)].rolling(n, min_periods=1).mean()\n",
    "        if inclusive:\n",
    "            df2.rename(columns={\"U10\": \"U10_rm1\", \"U100\": \"U100_rm1\", \"V10\": \"V10_rm1\", \"V100\": \"V100_rm1\"}, inplace=True)\n",
    "        else:\n",
    "            df2 = df2.drop([\"U10\", \"U100\", \"V10\", \"V100\"], axis=1, level=0)\n",
    "    else:\n",
    "        print('No rolling means computed')\n",
    "    \n",
    "    # Extract year, day of year, and hour from timestamp, then drop timestamp column\n",
    "    df2[\"YEAR\"]      = df2[\"TIMESTAMP\"].map(lambda x: x.year)\n",
    "    df2[\"DAYOFYEAR\"] = df2[\"TIMESTAMP\"].map(lambda x: x.timetuple().tm_yday)\n",
    "    df2[\"HOUR\"]      = df2[\"TIMESTAMP\"].map(lambda x: x.hour)\n",
    "    df2              = df2.drop('TIMESTAMP', axis=1, level=0)\n",
    "\n",
    "    return df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Cell3'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID column dropped\n",
      "Number of time points:                 13871\n",
      "Number of delta-times equal to 1 hour: 13870\n",
      "Computing rolling means over 3 measurements\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"10\" halign=\"left\">TARGETVAR</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>U10_rm3</th>\n",
       "      <th>U100_rm3</th>\n",
       "      <th>V10_rm3</th>\n",
       "      <th>V100_rm3</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>DAYOFYEAR</th>\n",
       "      <th>HOUR</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZONEID</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>2</th>\n",
       "      <th>2</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>3</th>\n",
       "      <th>3</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>4</th>\n",
       "      <th>4</th>\n",
       "      <th>4</th>\n",
       "      <th>6</th>\n",
       "      <th>6</th>\n",
       "      <th>6</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>7</th>\n",
       "      <th>7</th>\n",
       "      <th>7</th>\n",
       "      <th>9</th>\n",
       "      <th>9</th>\n",
       "      <th>9</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>10</th>\n",
       "      <th>10</th>\n",
       "      <th>10</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.596273</td>\n",
       "      <td>0.425465</td>\n",
       "      <td>0.378229</td>\n",
       "      <td>0.273678</td>\n",
       "      <td>0.268101</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.594361</td>\n",
       "      <td>2.124600</td>\n",
       "      <td>2.864280</td>\n",
       "      <td>-2.681966</td>\n",
       "      <td>-3.666076</td>\n",
       "      <td>-0.171642</td>\n",
       "      <td>-0.112594</td>\n",
       "      <td>-5.466031</td>\n",
       "      <td>-7.101347</td>\n",
       "      <td>2.193217</td>\n",
       "      <td>3.028435</td>\n",
       "      <td>-4.649168</td>\n",
       "      <td>-6.221648</td>\n",
       "      <td>0.534894</td>\n",
       "      <td>0.765355</td>\n",
       "      <td>-3.660243</td>\n",
       "      <td>-4.486657</td>\n",
       "      <td>1.205572</td>\n",
       "      <td>1.467827</td>\n",
       "      <td>-2.417913</td>\n",
       "      <td>-2.956134</td>\n",
       "      <td>-0.307548</td>\n",
       "      <td>-0.625438</td>\n",
       "      <td>0.371263</td>\n",
       "      <td>0.298533</td>\n",
       "      <td>1.903254</td>\n",
       "      <td>2.516723</td>\n",
       "      <td>-1.495002</td>\n",
       "      <td>-2.078063</td>\n",
       "      <td>1.655790</td>\n",
       "      <td>2.024257</td>\n",
       "      <td>-4.649593</td>\n",
       "      <td>-5.992232</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.054879</td>\n",
       "      <td>0.411180</td>\n",
       "      <td>0.363851</td>\n",
       "      <td>0.063012</td>\n",
       "      <td>0.086796</td>\n",
       "      <td>0.034542</td>\n",
       "      <td>0.014708</td>\n",
       "      <td>0.014954</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.569679</td>\n",
       "      <td>2.323147</td>\n",
       "      <td>3.104569</td>\n",
       "      <td>-2.239463</td>\n",
       "      <td>-3.065419</td>\n",
       "      <td>-0.130250</td>\n",
       "      <td>-0.063347</td>\n",
       "      <td>-5.054618</td>\n",
       "      <td>-6.499062</td>\n",
       "      <td>2.581679</td>\n",
       "      <td>3.504769</td>\n",
       "      <td>-4.296741</td>\n",
       "      <td>-5.737868</td>\n",
       "      <td>0.432854</td>\n",
       "      <td>0.617615</td>\n",
       "      <td>-3.168337</td>\n",
       "      <td>-3.850205</td>\n",
       "      <td>1.093350</td>\n",
       "      <td>1.308483</td>\n",
       "      <td>-1.997769</td>\n",
       "      <td>-2.446593</td>\n",
       "      <td>-0.083768</td>\n",
       "      <td>-0.300240</td>\n",
       "      <td>0.561424</td>\n",
       "      <td>0.548015</td>\n",
       "      <td>2.038275</td>\n",
       "      <td>2.676916</td>\n",
       "      <td>-1.069321</td>\n",
       "      <td>-1.542488</td>\n",
       "      <td>1.917199</td>\n",
       "      <td>2.323699</td>\n",
       "      <td>-4.417269</td>\n",
       "      <td>-5.655159</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.110234</td>\n",
       "      <td>0.167243</td>\n",
       "      <td>0.297129</td>\n",
       "      <td>0.036704</td>\n",
       "      <td>0.006811</td>\n",
       "      <td>0.020621</td>\n",
       "      <td>0.035098</td>\n",
       "      <td>0.071785</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.330539</td>\n",
       "      <td>2.439502</td>\n",
       "      <td>3.239195</td>\n",
       "      <td>-1.767148</td>\n",
       "      <td>-2.448310</td>\n",
       "      <td>-0.168838</td>\n",
       "      <td>-0.117311</td>\n",
       "      <td>-4.574210</td>\n",
       "      <td>-5.829165</td>\n",
       "      <td>2.800738</td>\n",
       "      <td>3.788456</td>\n",
       "      <td>-3.912656</td>\n",
       "      <td>-5.213050</td>\n",
       "      <td>0.266623</td>\n",
       "      <td>0.404779</td>\n",
       "      <td>-2.788582</td>\n",
       "      <td>-3.372843</td>\n",
       "      <td>0.933317</td>\n",
       "      <td>1.104798</td>\n",
       "      <td>-1.668908</td>\n",
       "      <td>-2.051307</td>\n",
       "      <td>0.169526</td>\n",
       "      <td>0.024769</td>\n",
       "      <td>0.865015</td>\n",
       "      <td>0.907596</td>\n",
       "      <td>2.129765</td>\n",
       "      <td>2.795976</td>\n",
       "      <td>-0.667349</td>\n",
       "      <td>-1.033489</td>\n",
       "      <td>2.021010</td>\n",
       "      <td>2.430872</td>\n",
       "      <td>-4.025660</td>\n",
       "      <td>-5.120850</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.165116</td>\n",
       "      <td>0.037326</td>\n",
       "      <td>0.235438</td>\n",
       "      <td>0.034342</td>\n",
       "      <td>0.018646</td>\n",
       "      <td>0.001867</td>\n",
       "      <td>0.045055</td>\n",
       "      <td>0.066796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.211308</td>\n",
       "      <td>2.550470</td>\n",
       "      <td>3.356180</td>\n",
       "      <td>-0.921040</td>\n",
       "      <td>-1.344800</td>\n",
       "      <td>-0.338537</td>\n",
       "      <td>-0.337128</td>\n",
       "      <td>-3.725405</td>\n",
       "      <td>-4.661657</td>\n",
       "      <td>3.156662</td>\n",
       "      <td>4.216958</td>\n",
       "      <td>-3.105872</td>\n",
       "      <td>-4.110987</td>\n",
       "      <td>-0.051525</td>\n",
       "      <td>0.001372</td>\n",
       "      <td>-2.168197</td>\n",
       "      <td>-2.593842</td>\n",
       "      <td>0.612365</td>\n",
       "      <td>0.702900</td>\n",
       "      <td>-1.111840</td>\n",
       "      <td>-1.384754</td>\n",
       "      <td>0.574441</td>\n",
       "      <td>0.545537</td>\n",
       "      <td>1.494201</td>\n",
       "      <td>1.683343</td>\n",
       "      <td>2.234535</td>\n",
       "      <td>2.937678</td>\n",
       "      <td>0.096703</td>\n",
       "      <td>-0.066291</td>\n",
       "      <td>2.161491</td>\n",
       "      <td>2.567102</td>\n",
       "      <td>-3.257490</td>\n",
       "      <td>-4.094716</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.156940</td>\n",
       "      <td>0.051206</td>\n",
       "      <td>0.120904</td>\n",
       "      <td>0.033554</td>\n",
       "      <td>0.034812</td>\n",
       "      <td>0.015174</td>\n",
       "      <td>0.050160</td>\n",
       "      <td>0.034797</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.172140</td>\n",
       "      <td>2.458537</td>\n",
       "      <td>3.227120</td>\n",
       "      <td>-0.192194</td>\n",
       "      <td>-0.412313</td>\n",
       "      <td>-0.729286</td>\n",
       "      <td>-0.828273</td>\n",
       "      <td>-3.050932</td>\n",
       "      <td>-3.777620</td>\n",
       "      <td>3.146628</td>\n",
       "      <td>4.201108</td>\n",
       "      <td>-2.471149</td>\n",
       "      <td>-3.256237</td>\n",
       "      <td>-0.413204</td>\n",
       "      <td>-0.434678</td>\n",
       "      <td>-1.829896</td>\n",
       "      <td>-2.186534</td>\n",
       "      <td>0.239176</td>\n",
       "      <td>0.264020</td>\n",
       "      <td>-0.753096</td>\n",
       "      <td>-0.969672</td>\n",
       "      <td>0.774243</td>\n",
       "      <td>0.777431</td>\n",
       "      <td>2.212967</td>\n",
       "      <td>2.584736</td>\n",
       "      <td>2.143636</td>\n",
       "      <td>2.845034</td>\n",
       "      <td>0.739802</td>\n",
       "      <td>0.759714</td>\n",
       "      <td>2.040194</td>\n",
       "      <td>2.392003</td>\n",
       "      <td>-2.388384</td>\n",
       "      <td>-2.980721</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data frame dimensions: (13871, 45)\n",
      "\n",
      "Training data frame saved to data/Train_pp_2018_10_21_12_22_36.csv\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Preprocess the training data set, output a pickle or csv file.\n",
    "'''\n",
    "output_format = \"csv\"\n",
    "\n",
    "# Load and preprocess data\n",
    "train_csv = \"data/Train_O4UPEyW.csv\"\n",
    "train_df  = preProcess(train_csv, Nroll=3, inclusive=False, keepID=False)\n",
    "display_html(train_df.head())\n",
    "print(\"Data frame dimensions: {0}\".format(train_df.shape))\n",
    "\n",
    "# Output dataframe\n",
    "if output_format == \"pkl\":\n",
    "    train_fname = \"data/Train_pp_\" + strftime(\"%Y_%m_%d_%H_%M_%S\") + \".pkl\"\n",
    "    try:\n",
    "        train_df.to_pickle(train_fname)\n",
    "        print('\\nTraining data frame saved to {0}'.format(train_fname))\n",
    "    except:\n",
    "        print('\\nError saving training data frame to {0}'.format(train_fname))\n",
    "elif output_format == \"csv\":\n",
    "    train_fname = \"data/Train_pp_\" + strftime(\"%Y_%m_%d_%H_%M_%S\") + \".csv\"\n",
    "    try:\n",
    "        train_df.to_csv(train_fname, header=True, index=False)\n",
    "        print('\\nTraining data frame saved to {0}'.format(train_fname))\n",
    "    except:\n",
    "        print('\\nError saving training data frame to {0}'.format(train_fname))\n",
    "else:\n",
    "    print('\\nUnrecognized output format')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Cell4'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID column dropped\n",
      "Number of time points:                 13871\n",
      "Number of delta-times equal to 1 hour: 13870\n",
      "Computing rolling means over 2 measurements\n",
      "Computing rolling means over 3 measurements\n",
      "Computing rolling means over 4 measurements\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TARGETVAR_1</th>\n",
       "      <th>TARGETVAR_2</th>\n",
       "      <th>TARGETVAR_3</th>\n",
       "      <th>TARGETVAR_4</th>\n",
       "      <th>TARGETVAR_5</th>\n",
       "      <th>TARGETVAR_6</th>\n",
       "      <th>TARGETVAR_7</th>\n",
       "      <th>TARGETVAR_8</th>\n",
       "      <th>TARGETVAR_9</th>\n",
       "      <th>TARGETVAR_10</th>\n",
       "      <th>U10_rm1_1</th>\n",
       "      <th>U10_rm1_2</th>\n",
       "      <th>U10_rm1_3</th>\n",
       "      <th>U10_rm1_4</th>\n",
       "      <th>U10_rm1_6</th>\n",
       "      <th>U10_rm1_7</th>\n",
       "      <th>U10_rm1_9</th>\n",
       "      <th>U10_rm1_10</th>\n",
       "      <th>V10_rm1_1</th>\n",
       "      <th>V10_rm1_2</th>\n",
       "      <th>V10_rm1_3</th>\n",
       "      <th>V10_rm1_4</th>\n",
       "      <th>V10_rm1_6</th>\n",
       "      <th>V10_rm1_7</th>\n",
       "      <th>V10_rm1_9</th>\n",
       "      <th>V10_rm1_10</th>\n",
       "      <th>U100_rm1_1</th>\n",
       "      <th>U100_rm1_2</th>\n",
       "      <th>U100_rm1_3</th>\n",
       "      <th>U100_rm1_4</th>\n",
       "      <th>U100_rm1_6</th>\n",
       "      <th>U100_rm1_7</th>\n",
       "      <th>U100_rm1_9</th>\n",
       "      <th>U100_rm1_10</th>\n",
       "      <th>V100_rm1_1</th>\n",
       "      <th>V100_rm1_2</th>\n",
       "      <th>V100_rm1_3</th>\n",
       "      <th>V100_rm1_4</th>\n",
       "      <th>V100_rm1_6</th>\n",
       "      <th>V100_rm1_7</th>\n",
       "      <th>V100_rm1_9</th>\n",
       "      <th>V100_rm1_10</th>\n",
       "      <th>U10_rm2_1</th>\n",
       "      <th>U100_rm2_1</th>\n",
       "      <th>V10_rm2_1</th>\n",
       "      <th>V100_rm2_1</th>\n",
       "      <th>U10_rm2_2</th>\n",
       "      <th>U100_rm2_2</th>\n",
       "      <th>V10_rm2_2</th>\n",
       "      <th>V100_rm2_2</th>\n",
       "      <th>U10_rm2_3</th>\n",
       "      <th>U100_rm2_3</th>\n",
       "      <th>V10_rm2_3</th>\n",
       "      <th>V100_rm2_3</th>\n",
       "      <th>U10_rm2_4</th>\n",
       "      <th>U100_rm2_4</th>\n",
       "      <th>V10_rm2_4</th>\n",
       "      <th>V100_rm2_4</th>\n",
       "      <th>U10_rm2_6</th>\n",
       "      <th>U100_rm2_6</th>\n",
       "      <th>V10_rm2_6</th>\n",
       "      <th>V100_rm2_6</th>\n",
       "      <th>U10_rm2_7</th>\n",
       "      <th>U100_rm2_7</th>\n",
       "      <th>V10_rm2_7</th>\n",
       "      <th>V100_rm2_7</th>\n",
       "      <th>U10_rm2_9</th>\n",
       "      <th>U100_rm2_9</th>\n",
       "      <th>V10_rm2_9</th>\n",
       "      <th>V100_rm2_9</th>\n",
       "      <th>U10_rm2_10</th>\n",
       "      <th>U100_rm2_10</th>\n",
       "      <th>V10_rm2_10</th>\n",
       "      <th>V100_rm2_10</th>\n",
       "      <th>U10_rm3_1</th>\n",
       "      <th>U100_rm3_1</th>\n",
       "      <th>V10_rm3_1</th>\n",
       "      <th>V100_rm3_1</th>\n",
       "      <th>U10_rm3_2</th>\n",
       "      <th>U100_rm3_2</th>\n",
       "      <th>V10_rm3_2</th>\n",
       "      <th>V100_rm3_2</th>\n",
       "      <th>U10_rm3_3</th>\n",
       "      <th>U100_rm3_3</th>\n",
       "      <th>V10_rm3_3</th>\n",
       "      <th>V100_rm3_3</th>\n",
       "      <th>U10_rm3_4</th>\n",
       "      <th>U100_rm3_4</th>\n",
       "      <th>V10_rm3_4</th>\n",
       "      <th>V100_rm3_4</th>\n",
       "      <th>U10_rm3_6</th>\n",
       "      <th>U100_rm3_6</th>\n",
       "      <th>V10_rm3_6</th>\n",
       "      <th>V100_rm3_6</th>\n",
       "      <th>U10_rm3_7</th>\n",
       "      <th>U100_rm3_7</th>\n",
       "      <th>V10_rm3_7</th>\n",
       "      <th>V100_rm3_7</th>\n",
       "      <th>U10_rm3_9</th>\n",
       "      <th>U100_rm3_9</th>\n",
       "      <th>V10_rm3_9</th>\n",
       "      <th>V100_rm3_9</th>\n",
       "      <th>U10_rm3_10</th>\n",
       "      <th>U100_rm3_10</th>\n",
       "      <th>V10_rm3_10</th>\n",
       "      <th>V100_rm3_10</th>\n",
       "      <th>U10_rm4_1</th>\n",
       "      <th>U100_rm4_1</th>\n",
       "      <th>V10_rm4_1</th>\n",
       "      <th>V100_rm4_1</th>\n",
       "      <th>U10_rm4_2</th>\n",
       "      <th>U100_rm4_2</th>\n",
       "      <th>V10_rm4_2</th>\n",
       "      <th>V100_rm4_2</th>\n",
       "      <th>U10_rm4_3</th>\n",
       "      <th>U100_rm4_3</th>\n",
       "      <th>V10_rm4_3</th>\n",
       "      <th>V100_rm4_3</th>\n",
       "      <th>U10_rm4_4</th>\n",
       "      <th>U100_rm4_4</th>\n",
       "      <th>V10_rm4_4</th>\n",
       "      <th>V100_rm4_4</th>\n",
       "      <th>U10_rm4_6</th>\n",
       "      <th>U100_rm4_6</th>\n",
       "      <th>V10_rm4_6</th>\n",
       "      <th>V100_rm4_6</th>\n",
       "      <th>U10_rm4_7</th>\n",
       "      <th>U100_rm4_7</th>\n",
       "      <th>V10_rm4_7</th>\n",
       "      <th>V100_rm4_7</th>\n",
       "      <th>U10_rm4_9</th>\n",
       "      <th>U100_rm4_9</th>\n",
       "      <th>V10_rm4_9</th>\n",
       "      <th>V100_rm4_9</th>\n",
       "      <th>U10_rm4_10</th>\n",
       "      <th>U100_rm4_10</th>\n",
       "      <th>V10_rm4_10</th>\n",
       "      <th>V100_rm4_10</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>DAYOFYEAR</th>\n",
       "      <th>HOUR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.596273</td>\n",
       "      <td>0.425465</td>\n",
       "      <td>0.378229</td>\n",
       "      <td>0.273678</td>\n",
       "      <td>0.268101</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.594361</td>\n",
       "      <td>2.124600</td>\n",
       "      <td>-0.171642</td>\n",
       "      <td>2.193217</td>\n",
       "      <td>0.534894</td>\n",
       "      <td>1.205572</td>\n",
       "      <td>-0.307548</td>\n",
       "      <td>1.903254</td>\n",
       "      <td>1.655790</td>\n",
       "      <td>-2.681966</td>\n",
       "      <td>-5.466031</td>\n",
       "      <td>-4.649168</td>\n",
       "      <td>-3.660243</td>\n",
       "      <td>-2.417913</td>\n",
       "      <td>0.371263</td>\n",
       "      <td>-1.495002</td>\n",
       "      <td>-4.649593</td>\n",
       "      <td>2.864280</td>\n",
       "      <td>-0.112594</td>\n",
       "      <td>3.028435</td>\n",
       "      <td>0.765355</td>\n",
       "      <td>1.467827</td>\n",
       "      <td>-0.625438</td>\n",
       "      <td>2.516723</td>\n",
       "      <td>2.024257</td>\n",
       "      <td>-3.666076</td>\n",
       "      <td>-7.101347</td>\n",
       "      <td>-6.221648</td>\n",
       "      <td>-4.486657</td>\n",
       "      <td>-2.956134</td>\n",
       "      <td>0.298533</td>\n",
       "      <td>-2.078063</td>\n",
       "      <td>-5.992232</td>\n",
       "      <td>2.124600</td>\n",
       "      <td>2.864280</td>\n",
       "      <td>-2.681966</td>\n",
       "      <td>-3.666076</td>\n",
       "      <td>-0.171642</td>\n",
       "      <td>-0.112594</td>\n",
       "      <td>-5.466031</td>\n",
       "      <td>-7.101347</td>\n",
       "      <td>2.193217</td>\n",
       "      <td>3.028435</td>\n",
       "      <td>-4.649168</td>\n",
       "      <td>-6.221648</td>\n",
       "      <td>0.534894</td>\n",
       "      <td>0.765355</td>\n",
       "      <td>-3.660243</td>\n",
       "      <td>-4.486657</td>\n",
       "      <td>1.205572</td>\n",
       "      <td>1.467827</td>\n",
       "      <td>-2.417913</td>\n",
       "      <td>-2.956134</td>\n",
       "      <td>-0.307548</td>\n",
       "      <td>-0.625438</td>\n",
       "      <td>0.371263</td>\n",
       "      <td>0.298533</td>\n",
       "      <td>1.903254</td>\n",
       "      <td>2.516723</td>\n",
       "      <td>-1.495002</td>\n",
       "      <td>-2.078063</td>\n",
       "      <td>1.655790</td>\n",
       "      <td>2.024257</td>\n",
       "      <td>-4.649593</td>\n",
       "      <td>-5.992232</td>\n",
       "      <td>2.124600</td>\n",
       "      <td>2.864280</td>\n",
       "      <td>-2.681966</td>\n",
       "      <td>-3.666076</td>\n",
       "      <td>-0.171642</td>\n",
       "      <td>-0.112594</td>\n",
       "      <td>-5.466031</td>\n",
       "      <td>-7.101347</td>\n",
       "      <td>2.193217</td>\n",
       "      <td>3.028435</td>\n",
       "      <td>-4.649168</td>\n",
       "      <td>-6.221648</td>\n",
       "      <td>0.534894</td>\n",
       "      <td>0.765355</td>\n",
       "      <td>-3.660243</td>\n",
       "      <td>-4.486657</td>\n",
       "      <td>1.205572</td>\n",
       "      <td>1.467827</td>\n",
       "      <td>-2.417913</td>\n",
       "      <td>-2.956134</td>\n",
       "      <td>-0.307548</td>\n",
       "      <td>-0.625438</td>\n",
       "      <td>0.371263</td>\n",
       "      <td>0.298533</td>\n",
       "      <td>1.903254</td>\n",
       "      <td>2.516723</td>\n",
       "      <td>-1.495002</td>\n",
       "      <td>-2.078063</td>\n",
       "      <td>1.655790</td>\n",
       "      <td>2.024257</td>\n",
       "      <td>-4.649593</td>\n",
       "      <td>-5.992232</td>\n",
       "      <td>2.124600</td>\n",
       "      <td>2.864280</td>\n",
       "      <td>-2.681966</td>\n",
       "      <td>-3.666076</td>\n",
       "      <td>-0.171642</td>\n",
       "      <td>-0.112594</td>\n",
       "      <td>-5.466031</td>\n",
       "      <td>-7.101347</td>\n",
       "      <td>2.193217</td>\n",
       "      <td>3.028435</td>\n",
       "      <td>-4.649168</td>\n",
       "      <td>-6.221648</td>\n",
       "      <td>0.534894</td>\n",
       "      <td>0.765355</td>\n",
       "      <td>-3.660243</td>\n",
       "      <td>-4.486657</td>\n",
       "      <td>1.205572</td>\n",
       "      <td>1.467827</td>\n",
       "      <td>-2.417913</td>\n",
       "      <td>-2.956134</td>\n",
       "      <td>-0.307548</td>\n",
       "      <td>-0.625438</td>\n",
       "      <td>0.371263</td>\n",
       "      <td>0.298533</td>\n",
       "      <td>1.903254</td>\n",
       "      <td>2.516723</td>\n",
       "      <td>-1.495002</td>\n",
       "      <td>-2.078063</td>\n",
       "      <td>1.655790</td>\n",
       "      <td>2.024257</td>\n",
       "      <td>-4.649593</td>\n",
       "      <td>-5.992232</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.054879</td>\n",
       "      <td>0.411180</td>\n",
       "      <td>0.363851</td>\n",
       "      <td>0.063012</td>\n",
       "      <td>0.086796</td>\n",
       "      <td>0.034542</td>\n",
       "      <td>0.014708</td>\n",
       "      <td>0.014954</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.569679</td>\n",
       "      <td>2.521695</td>\n",
       "      <td>-0.088859</td>\n",
       "      <td>2.970141</td>\n",
       "      <td>0.330813</td>\n",
       "      <td>0.981127</td>\n",
       "      <td>0.140013</td>\n",
       "      <td>2.173296</td>\n",
       "      <td>2.178609</td>\n",
       "      <td>-1.796960</td>\n",
       "      <td>-4.643205</td>\n",
       "      <td>-3.944314</td>\n",
       "      <td>-2.676430</td>\n",
       "      <td>-1.577625</td>\n",
       "      <td>0.751585</td>\n",
       "      <td>-0.643641</td>\n",
       "      <td>-4.184944</td>\n",
       "      <td>3.344859</td>\n",
       "      <td>-0.014100</td>\n",
       "      <td>3.981102</td>\n",
       "      <td>0.469875</td>\n",
       "      <td>1.149139</td>\n",
       "      <td>0.024957</td>\n",
       "      <td>2.837109</td>\n",
       "      <td>2.623141</td>\n",
       "      <td>-2.464761</td>\n",
       "      <td>-5.896778</td>\n",
       "      <td>-5.254089</td>\n",
       "      <td>-3.213752</td>\n",
       "      <td>-1.937051</td>\n",
       "      <td>0.797498</td>\n",
       "      <td>-1.006914</td>\n",
       "      <td>-5.318086</td>\n",
       "      <td>2.323147</td>\n",
       "      <td>3.104569</td>\n",
       "      <td>-2.239463</td>\n",
       "      <td>-3.065419</td>\n",
       "      <td>-0.130250</td>\n",
       "      <td>-0.063347</td>\n",
       "      <td>-5.054618</td>\n",
       "      <td>-6.499062</td>\n",
       "      <td>2.581679</td>\n",
       "      <td>3.504769</td>\n",
       "      <td>-4.296741</td>\n",
       "      <td>-5.737868</td>\n",
       "      <td>0.432854</td>\n",
       "      <td>0.617615</td>\n",
       "      <td>-3.168337</td>\n",
       "      <td>-3.850205</td>\n",
       "      <td>1.093350</td>\n",
       "      <td>1.308483</td>\n",
       "      <td>-1.997769</td>\n",
       "      <td>-2.446593</td>\n",
       "      <td>-0.083768</td>\n",
       "      <td>-0.300240</td>\n",
       "      <td>0.561424</td>\n",
       "      <td>0.548015</td>\n",
       "      <td>2.038275</td>\n",
       "      <td>2.676916</td>\n",
       "      <td>-1.069321</td>\n",
       "      <td>-1.542488</td>\n",
       "      <td>1.917199</td>\n",
       "      <td>2.323699</td>\n",
       "      <td>-4.417269</td>\n",
       "      <td>-5.655159</td>\n",
       "      <td>2.323147</td>\n",
       "      <td>3.104569</td>\n",
       "      <td>-2.239463</td>\n",
       "      <td>-3.065419</td>\n",
       "      <td>-0.130250</td>\n",
       "      <td>-0.063347</td>\n",
       "      <td>-5.054618</td>\n",
       "      <td>-6.499062</td>\n",
       "      <td>2.581679</td>\n",
       "      <td>3.504769</td>\n",
       "      <td>-4.296741</td>\n",
       "      <td>-5.737868</td>\n",
       "      <td>0.432854</td>\n",
       "      <td>0.617615</td>\n",
       "      <td>-3.168337</td>\n",
       "      <td>-3.850205</td>\n",
       "      <td>1.093350</td>\n",
       "      <td>1.308483</td>\n",
       "      <td>-1.997769</td>\n",
       "      <td>-2.446593</td>\n",
       "      <td>-0.083768</td>\n",
       "      <td>-0.300240</td>\n",
       "      <td>0.561424</td>\n",
       "      <td>0.548015</td>\n",
       "      <td>2.038275</td>\n",
       "      <td>2.676916</td>\n",
       "      <td>-1.069321</td>\n",
       "      <td>-1.542488</td>\n",
       "      <td>1.917199</td>\n",
       "      <td>2.323699</td>\n",
       "      <td>-4.417269</td>\n",
       "      <td>-5.655159</td>\n",
       "      <td>2.323147</td>\n",
       "      <td>3.104569</td>\n",
       "      <td>-2.239463</td>\n",
       "      <td>-3.065419</td>\n",
       "      <td>-0.130250</td>\n",
       "      <td>-0.063347</td>\n",
       "      <td>-5.054618</td>\n",
       "      <td>-6.499062</td>\n",
       "      <td>2.581679</td>\n",
       "      <td>3.504769</td>\n",
       "      <td>-4.296741</td>\n",
       "      <td>-5.737868</td>\n",
       "      <td>0.432854</td>\n",
       "      <td>0.617615</td>\n",
       "      <td>-3.168337</td>\n",
       "      <td>-3.850205</td>\n",
       "      <td>1.093350</td>\n",
       "      <td>1.308483</td>\n",
       "      <td>-1.997769</td>\n",
       "      <td>-2.446593</td>\n",
       "      <td>-0.083768</td>\n",
       "      <td>-0.300240</td>\n",
       "      <td>0.561424</td>\n",
       "      <td>0.548015</td>\n",
       "      <td>2.038275</td>\n",
       "      <td>2.676916</td>\n",
       "      <td>-1.069321</td>\n",
       "      <td>-1.542488</td>\n",
       "      <td>1.917199</td>\n",
       "      <td>2.323699</td>\n",
       "      <td>-4.417269</td>\n",
       "      <td>-5.655159</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.110234</td>\n",
       "      <td>0.167243</td>\n",
       "      <td>0.297129</td>\n",
       "      <td>0.036704</td>\n",
       "      <td>0.006811</td>\n",
       "      <td>0.020621</td>\n",
       "      <td>0.035098</td>\n",
       "      <td>0.071785</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.330539</td>\n",
       "      <td>2.672210</td>\n",
       "      <td>-0.246014</td>\n",
       "      <td>3.238855</td>\n",
       "      <td>-0.065839</td>\n",
       "      <td>0.613250</td>\n",
       "      <td>0.676113</td>\n",
       "      <td>2.312744</td>\n",
       "      <td>2.228633</td>\n",
       "      <td>-0.822516</td>\n",
       "      <td>-3.613395</td>\n",
       "      <td>-3.144486</td>\n",
       "      <td>-2.029072</td>\n",
       "      <td>-1.011187</td>\n",
       "      <td>1.472197</td>\n",
       "      <td>0.136595</td>\n",
       "      <td>-3.242442</td>\n",
       "      <td>3.508448</td>\n",
       "      <td>-0.225238</td>\n",
       "      <td>4.355830</td>\n",
       "      <td>-0.020893</td>\n",
       "      <td>0.697428</td>\n",
       "      <td>0.674786</td>\n",
       "      <td>3.034096</td>\n",
       "      <td>2.645217</td>\n",
       "      <td>-1.214093</td>\n",
       "      <td>-4.489369</td>\n",
       "      <td>-4.163414</td>\n",
       "      <td>-2.418119</td>\n",
       "      <td>-1.260735</td>\n",
       "      <td>1.626758</td>\n",
       "      <td>-0.015490</td>\n",
       "      <td>-4.052232</td>\n",
       "      <td>2.596952</td>\n",
       "      <td>3.426653</td>\n",
       "      <td>-1.309738</td>\n",
       "      <td>-1.839427</td>\n",
       "      <td>-0.167436</td>\n",
       "      <td>-0.119669</td>\n",
       "      <td>-4.128300</td>\n",
       "      <td>-5.193074</td>\n",
       "      <td>3.104498</td>\n",
       "      <td>4.168466</td>\n",
       "      <td>-3.544400</td>\n",
       "      <td>-4.708751</td>\n",
       "      <td>0.132487</td>\n",
       "      <td>0.224491</td>\n",
       "      <td>-2.352751</td>\n",
       "      <td>-2.815936</td>\n",
       "      <td>0.797189</td>\n",
       "      <td>0.923284</td>\n",
       "      <td>-1.294406</td>\n",
       "      <td>-1.598893</td>\n",
       "      <td>0.408063</td>\n",
       "      <td>0.349872</td>\n",
       "      <td>1.111891</td>\n",
       "      <td>1.212128</td>\n",
       "      <td>2.243020</td>\n",
       "      <td>2.935602</td>\n",
       "      <td>-0.253523</td>\n",
       "      <td>-0.511202</td>\n",
       "      <td>2.203621</td>\n",
       "      <td>2.634179</td>\n",
       "      <td>-3.713693</td>\n",
       "      <td>-4.685159</td>\n",
       "      <td>2.439502</td>\n",
       "      <td>3.239195</td>\n",
       "      <td>-1.767148</td>\n",
       "      <td>-2.448310</td>\n",
       "      <td>-0.168838</td>\n",
       "      <td>-0.117311</td>\n",
       "      <td>-4.574210</td>\n",
       "      <td>-5.829165</td>\n",
       "      <td>2.800738</td>\n",
       "      <td>3.788456</td>\n",
       "      <td>-3.912656</td>\n",
       "      <td>-5.213050</td>\n",
       "      <td>0.266623</td>\n",
       "      <td>0.404779</td>\n",
       "      <td>-2.788582</td>\n",
       "      <td>-3.372843</td>\n",
       "      <td>0.933317</td>\n",
       "      <td>1.104798</td>\n",
       "      <td>-1.668908</td>\n",
       "      <td>-2.051307</td>\n",
       "      <td>0.169526</td>\n",
       "      <td>0.024769</td>\n",
       "      <td>0.865015</td>\n",
       "      <td>0.907596</td>\n",
       "      <td>2.129765</td>\n",
       "      <td>2.795976</td>\n",
       "      <td>-0.667349</td>\n",
       "      <td>-1.033489</td>\n",
       "      <td>2.021010</td>\n",
       "      <td>2.430872</td>\n",
       "      <td>-4.025660</td>\n",
       "      <td>-5.120850</td>\n",
       "      <td>2.439502</td>\n",
       "      <td>3.239195</td>\n",
       "      <td>-1.767148</td>\n",
       "      <td>-2.448310</td>\n",
       "      <td>-0.168838</td>\n",
       "      <td>-0.117311</td>\n",
       "      <td>-4.574210</td>\n",
       "      <td>-5.829165</td>\n",
       "      <td>2.800738</td>\n",
       "      <td>3.788456</td>\n",
       "      <td>-3.912656</td>\n",
       "      <td>-5.213050</td>\n",
       "      <td>0.266623</td>\n",
       "      <td>0.404779</td>\n",
       "      <td>-2.788582</td>\n",
       "      <td>-3.372843</td>\n",
       "      <td>0.933317</td>\n",
       "      <td>1.104798</td>\n",
       "      <td>-1.668908</td>\n",
       "      <td>-2.051307</td>\n",
       "      <td>0.169526</td>\n",
       "      <td>0.024769</td>\n",
       "      <td>0.865015</td>\n",
       "      <td>0.907596</td>\n",
       "      <td>2.129765</td>\n",
       "      <td>2.795976</td>\n",
       "      <td>-0.667349</td>\n",
       "      <td>-1.033489</td>\n",
       "      <td>2.021010</td>\n",
       "      <td>2.430872</td>\n",
       "      <td>-4.025660</td>\n",
       "      <td>-5.120850</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.165116</td>\n",
       "      <td>0.037326</td>\n",
       "      <td>0.235438</td>\n",
       "      <td>0.034342</td>\n",
       "      <td>0.018646</td>\n",
       "      <td>0.001867</td>\n",
       "      <td>0.045055</td>\n",
       "      <td>0.066796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.211308</td>\n",
       "      <td>2.457504</td>\n",
       "      <td>-0.680738</td>\n",
       "      <td>3.260990</td>\n",
       "      <td>-0.419549</td>\n",
       "      <td>0.242717</td>\n",
       "      <td>0.907198</td>\n",
       "      <td>2.217565</td>\n",
       "      <td>2.077232</td>\n",
       "      <td>-0.143642</td>\n",
       "      <td>-2.919615</td>\n",
       "      <td>-2.228816</td>\n",
       "      <td>-1.799090</td>\n",
       "      <td>-0.746707</td>\n",
       "      <td>2.258822</td>\n",
       "      <td>0.797156</td>\n",
       "      <td>-2.345085</td>\n",
       "      <td>3.215233</td>\n",
       "      <td>-0.772046</td>\n",
       "      <td>4.313942</td>\n",
       "      <td>-0.444867</td>\n",
       "      <td>0.262134</td>\n",
       "      <td>0.936869</td>\n",
       "      <td>2.941829</td>\n",
       "      <td>2.432947</td>\n",
       "      <td>-0.355546</td>\n",
       "      <td>-3.598824</td>\n",
       "      <td>-2.915458</td>\n",
       "      <td>-2.149654</td>\n",
       "      <td>-0.956475</td>\n",
       "      <td>2.625774</td>\n",
       "      <td>0.823531</td>\n",
       "      <td>-2.913831</td>\n",
       "      <td>2.564857</td>\n",
       "      <td>3.361840</td>\n",
       "      <td>-0.483079</td>\n",
       "      <td>-0.784820</td>\n",
       "      <td>-0.463376</td>\n",
       "      <td>-0.498642</td>\n",
       "      <td>-3.266505</td>\n",
       "      <td>-4.044097</td>\n",
       "      <td>3.249923</td>\n",
       "      <td>4.334886</td>\n",
       "      <td>-2.686651</td>\n",
       "      <td>-3.539436</td>\n",
       "      <td>-0.242694</td>\n",
       "      <td>-0.232880</td>\n",
       "      <td>-1.914081</td>\n",
       "      <td>-2.283886</td>\n",
       "      <td>0.427984</td>\n",
       "      <td>0.479781</td>\n",
       "      <td>-0.878947</td>\n",
       "      <td>-1.108605</td>\n",
       "      <td>0.791655</td>\n",
       "      <td>0.805828</td>\n",
       "      <td>1.865509</td>\n",
       "      <td>2.126266</td>\n",
       "      <td>2.265155</td>\n",
       "      <td>2.987962</td>\n",
       "      <td>0.466876</td>\n",
       "      <td>0.404020</td>\n",
       "      <td>2.152932</td>\n",
       "      <td>2.539082</td>\n",
       "      <td>-2.793763</td>\n",
       "      <td>-3.483031</td>\n",
       "      <td>2.550470</td>\n",
       "      <td>3.356180</td>\n",
       "      <td>-0.921040</td>\n",
       "      <td>-1.344800</td>\n",
       "      <td>-0.338537</td>\n",
       "      <td>-0.337128</td>\n",
       "      <td>-3.725405</td>\n",
       "      <td>-4.661657</td>\n",
       "      <td>3.156662</td>\n",
       "      <td>4.216958</td>\n",
       "      <td>-3.105872</td>\n",
       "      <td>-4.110987</td>\n",
       "      <td>-0.051525</td>\n",
       "      <td>0.001372</td>\n",
       "      <td>-2.168197</td>\n",
       "      <td>-2.593842</td>\n",
       "      <td>0.612365</td>\n",
       "      <td>0.702900</td>\n",
       "      <td>-1.111840</td>\n",
       "      <td>-1.384754</td>\n",
       "      <td>0.574441</td>\n",
       "      <td>0.545537</td>\n",
       "      <td>1.494201</td>\n",
       "      <td>1.683343</td>\n",
       "      <td>2.234535</td>\n",
       "      <td>2.937678</td>\n",
       "      <td>0.096703</td>\n",
       "      <td>-0.066291</td>\n",
       "      <td>2.161491</td>\n",
       "      <td>2.567102</td>\n",
       "      <td>-3.257490</td>\n",
       "      <td>-4.094716</td>\n",
       "      <td>2.444002</td>\n",
       "      <td>3.233205</td>\n",
       "      <td>-1.361271</td>\n",
       "      <td>-1.925119</td>\n",
       "      <td>-0.296813</td>\n",
       "      <td>-0.280994</td>\n",
       "      <td>-4.160561</td>\n",
       "      <td>-5.271579</td>\n",
       "      <td>2.915801</td>\n",
       "      <td>3.919827</td>\n",
       "      <td>-3.491696</td>\n",
       "      <td>-4.638652</td>\n",
       "      <td>0.095080</td>\n",
       "      <td>0.192367</td>\n",
       "      <td>-2.541209</td>\n",
       "      <td>-3.067046</td>\n",
       "      <td>0.760667</td>\n",
       "      <td>0.894132</td>\n",
       "      <td>-1.438358</td>\n",
       "      <td>-1.777599</td>\n",
       "      <td>0.353944</td>\n",
       "      <td>0.252794</td>\n",
       "      <td>1.213467</td>\n",
       "      <td>1.337141</td>\n",
       "      <td>2.151715</td>\n",
       "      <td>2.832439</td>\n",
       "      <td>-0.301223</td>\n",
       "      <td>-0.569234</td>\n",
       "      <td>2.035066</td>\n",
       "      <td>2.431390</td>\n",
       "      <td>-3.605516</td>\n",
       "      <td>-4.569095</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.156940</td>\n",
       "      <td>0.051206</td>\n",
       "      <td>0.120904</td>\n",
       "      <td>0.033554</td>\n",
       "      <td>0.034812</td>\n",
       "      <td>0.015174</td>\n",
       "      <td>0.050160</td>\n",
       "      <td>0.034797</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.172140</td>\n",
       "      <td>2.245898</td>\n",
       "      <td>-1.261106</td>\n",
       "      <td>2.940038</td>\n",
       "      <td>-0.754224</td>\n",
       "      <td>-0.138440</td>\n",
       "      <td>0.739418</td>\n",
       "      <td>1.900598</td>\n",
       "      <td>1.814716</td>\n",
       "      <td>0.389576</td>\n",
       "      <td>-2.619786</td>\n",
       "      <td>-2.040145</td>\n",
       "      <td>-1.661526</td>\n",
       "      <td>-0.501393</td>\n",
       "      <td>2.907883</td>\n",
       "      <td>1.285656</td>\n",
       "      <td>-1.577625</td>\n",
       "      <td>2.957678</td>\n",
       "      <td>-1.487537</td>\n",
       "      <td>3.933554</td>\n",
       "      <td>-0.838274</td>\n",
       "      <td>-0.167501</td>\n",
       "      <td>0.720637</td>\n",
       "      <td>2.559177</td>\n",
       "      <td>2.097844</td>\n",
       "      <td>0.332701</td>\n",
       "      <td>-3.244667</td>\n",
       "      <td>-2.689838</td>\n",
       "      <td>-1.991829</td>\n",
       "      <td>-0.691806</td>\n",
       "      <td>3.501676</td>\n",
       "      <td>1.471102</td>\n",
       "      <td>-1.976100</td>\n",
       "      <td>2.351701</td>\n",
       "      <td>3.086455</td>\n",
       "      <td>0.122967</td>\n",
       "      <td>-0.011423</td>\n",
       "      <td>-0.970922</td>\n",
       "      <td>-1.129791</td>\n",
       "      <td>-2.769700</td>\n",
       "      <td>-3.421745</td>\n",
       "      <td>3.100514</td>\n",
       "      <td>4.123748</td>\n",
       "      <td>-2.134481</td>\n",
       "      <td>-2.802648</td>\n",
       "      <td>-0.586887</td>\n",
       "      <td>-0.641570</td>\n",
       "      <td>-1.730308</td>\n",
       "      <td>-2.070741</td>\n",
       "      <td>0.052139</td>\n",
       "      <td>0.047316</td>\n",
       "      <td>-0.624050</td>\n",
       "      <td>-0.824140</td>\n",
       "      <td>0.823308</td>\n",
       "      <td>0.828753</td>\n",
       "      <td>2.583353</td>\n",
       "      <td>3.063725</td>\n",
       "      <td>2.059082</td>\n",
       "      <td>2.750503</td>\n",
       "      <td>1.041406</td>\n",
       "      <td>1.147317</td>\n",
       "      <td>1.945974</td>\n",
       "      <td>2.265395</td>\n",
       "      <td>-1.961355</td>\n",
       "      <td>-2.444966</td>\n",
       "      <td>2.458537</td>\n",
       "      <td>3.227120</td>\n",
       "      <td>-0.192194</td>\n",
       "      <td>-0.412313</td>\n",
       "      <td>-0.729286</td>\n",
       "      <td>-0.828273</td>\n",
       "      <td>-3.050932</td>\n",
       "      <td>-3.777620</td>\n",
       "      <td>3.146628</td>\n",
       "      <td>4.201108</td>\n",
       "      <td>-2.471149</td>\n",
       "      <td>-3.256237</td>\n",
       "      <td>-0.413204</td>\n",
       "      <td>-0.434678</td>\n",
       "      <td>-1.829896</td>\n",
       "      <td>-2.186534</td>\n",
       "      <td>0.239176</td>\n",
       "      <td>0.264020</td>\n",
       "      <td>-0.753096</td>\n",
       "      <td>-0.969672</td>\n",
       "      <td>0.774243</td>\n",
       "      <td>0.777431</td>\n",
       "      <td>2.212967</td>\n",
       "      <td>2.584736</td>\n",
       "      <td>2.143636</td>\n",
       "      <td>2.845034</td>\n",
       "      <td>0.739802</td>\n",
       "      <td>0.759714</td>\n",
       "      <td>2.040194</td>\n",
       "      <td>2.392003</td>\n",
       "      <td>-2.388384</td>\n",
       "      <td>-2.980721</td>\n",
       "      <td>2.474327</td>\n",
       "      <td>3.256554</td>\n",
       "      <td>-0.593386</td>\n",
       "      <td>-0.925425</td>\n",
       "      <td>-0.569179</td>\n",
       "      <td>-0.624730</td>\n",
       "      <td>-3.449000</td>\n",
       "      <td>-4.307409</td>\n",
       "      <td>3.102506</td>\n",
       "      <td>4.146107</td>\n",
       "      <td>-2.839440</td>\n",
       "      <td>-3.755700</td>\n",
       "      <td>-0.227200</td>\n",
       "      <td>-0.208540</td>\n",
       "      <td>-2.041529</td>\n",
       "      <td>-2.443338</td>\n",
       "      <td>0.424664</td>\n",
       "      <td>0.485300</td>\n",
       "      <td>-0.959228</td>\n",
       "      <td>-1.211517</td>\n",
       "      <td>0.615685</td>\n",
       "      <td>0.589312</td>\n",
       "      <td>1.847622</td>\n",
       "      <td>2.137927</td>\n",
       "      <td>2.151051</td>\n",
       "      <td>2.843053</td>\n",
       "      <td>0.393942</td>\n",
       "      <td>0.318057</td>\n",
       "      <td>2.074797</td>\n",
       "      <td>2.449787</td>\n",
       "      <td>-2.837524</td>\n",
       "      <td>-3.565062</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data frame dimensions: (13871, 141)\n",
      "\n",
      "Training data frame saved to data/Train_pp_2017_11_04_21_59_47.feather\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Preprocess the training data set in Feather format for notebooks running R kernel.\n",
    "'''\n",
    "# Load and preprocess data\n",
    "train_csv = \"data/Train_O4UPEyW.csv\"\n",
    "train_df  = preProcess(train_csv, Nroll=4, inclusive=True, keepID=False)\n",
    "\n",
    "# Rename dataframe columns\n",
    "predictors  = [item[0]+'_'+str(item[1]) for item in train_df.columns.values if item[0]!='TARGETVAR']\n",
    "predictors  = [predictor if predictor[-1]!='_' else predictor[:-1] for predictor in predictors]\n",
    "targets     = [item[0]+'_'+str(item[1]) for item in train_df.columns.values if item[0]=='TARGETVAR']\n",
    "train_df.columns = targets+predictors\n",
    "display_html(train_df.head())\n",
    "print(\"Data frame dimensions: {0}\".format(train_df.shape))\n",
    "\n",
    "# Output data frame in feather format\n",
    "train_fname = \"data/Train_pp_\" + strftime(\"%Y_%m_%d_%H_%M_%S\") + \".feather\"\n",
    "try:\n",
    "    feather.write_dataframe(train_df, train_fname)\n",
    "    print('\\nTraining data frame saved to {0}'.format(train_fname))\n",
    "except:\n",
    "    print('\\nError saving training data frame to {0}'.format(train_fname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Cell5'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID column kept\n",
      "Number of time points:                 2929\n",
      "Number of delta-times equal to 1 hour: 2928\n",
      "Computing rolling means over 3 measurements\n",
      "\n",
      "Dataframe test_df:\n",
      "              ID                                                              \\\n",
      "ZONEID         1         2         3         4         5         6         7   \n",
      "0       30010001  30010002  30010003  30010004  30010005  30010006  30010007   \n",
      "1       30010101  30010102  30010103  30010104  30010105  30010106  30010107   \n",
      "2       30010201  30010202  30010203  30010204  30010205  30010206  30010207   \n",
      "3       30010301  30010302  30010303  30010304  30010305  30010306  30010307   \n",
      "4       30010401  30010402  30010403  30010404  30010405  30010406  30010407   \n",
      "\n",
      "                                     ...   U100_rm3   V10_rm3  V100_rm3  \\\n",
      "ZONEID         8         9        10 ...          9         9         9   \n",
      "0       30010008  30010009  30010010 ...   1.473541 -3.757118 -5.748985   \n",
      "1       30010108  30010109  30010110 ...   2.425901 -3.923264 -5.798490   \n",
      "2       30010208  30010209  30010210 ...   2.899162 -4.043484 -5.885207   \n",
      "3       30010308  30010309  30010310 ...   3.475966 -4.194771 -5.943312   \n",
      "4       30010408  30010409  30010410 ...   3.356855 -4.239347 -5.972145   \n",
      "\n",
      "         U10_rm3  U100_rm3   V10_rm3  V100_rm3  YEAR DAYOFYEAR HOUR  \n",
      "ZONEID        10        10        10        10                       \n",
      "0      -0.165117  0.320843 -2.658255 -4.277716  2013       213    0  \n",
      "1      -0.257103 -0.014866 -3.223966 -4.800565  2013       213    1  \n",
      "2       0.085075  0.330257 -3.693685 -5.259272  2013       213    2  \n",
      "3       0.764744  1.048942 -4.490173 -6.070175  2013       213    3  \n",
      "4       1.679930  2.239443 -4.925310 -6.571012  2013       213    4  \n",
      "\n",
      "[5 rows x 77 columns]\n",
      "\n",
      "Testing data frame saved to data/Test_pp_2017_10_22_17_24_03.pkl\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Preprocess the testing data set, output a pickle file.\n",
    "'''\n",
    "test_csv  = \"data/Test_uP7dymh.csv\"\n",
    "\n",
    "test_df = preProcess(test_csv, Nroll=3, inclusive=False, keepID=True)\n",
    "display_html(test_df.head())\n",
    "print(\"Data frame dimensions: {0}\".format(test_df.shape))\n",
    "\n",
    "test_fname = \"data/Test_pp_\" + strftime(\"%Y_%m_%d_%H_%M_%S\") + \".pkl\"\n",
    "try:\n",
    "    test_df.to_pickle(test_fname)\n",
    "    print('\\nTesting data frame saved to {0}'.format(test_fname))\n",
    "except:\n",
    "    print('\\nError saving testing data frame to {0}'.format(test_fname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Cell6'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID column kept\n",
      "Number of time points:                 2929\n",
      "Number of delta-times equal to 1 hour: 2928\n",
      "Computing rolling means over 2 measurements\n",
      "Computing rolling means over 3 measurements\n",
      "Computing rolling means over 4 measurements\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_1</th>\n",
       "      <th>ID_2</th>\n",
       "      <th>ID_3</th>\n",
       "      <th>ID_4</th>\n",
       "      <th>ID_5</th>\n",
       "      <th>ID_6</th>\n",
       "      <th>ID_7</th>\n",
       "      <th>ID_8</th>\n",
       "      <th>ID_9</th>\n",
       "      <th>ID_10</th>\n",
       "      <th>U10_rm1_1</th>\n",
       "      <th>U10_rm1_2</th>\n",
       "      <th>U10_rm1_3</th>\n",
       "      <th>U10_rm1_4</th>\n",
       "      <th>U10_rm1_6</th>\n",
       "      <th>U10_rm1_7</th>\n",
       "      <th>U10_rm1_9</th>\n",
       "      <th>U10_rm1_10</th>\n",
       "      <th>V10_rm1_1</th>\n",
       "      <th>V10_rm1_2</th>\n",
       "      <th>V10_rm1_3</th>\n",
       "      <th>V10_rm1_4</th>\n",
       "      <th>V10_rm1_6</th>\n",
       "      <th>V10_rm1_7</th>\n",
       "      <th>V10_rm1_9</th>\n",
       "      <th>V10_rm1_10</th>\n",
       "      <th>U100_rm1_1</th>\n",
       "      <th>U100_rm1_2</th>\n",
       "      <th>U100_rm1_3</th>\n",
       "      <th>U100_rm1_4</th>\n",
       "      <th>U100_rm1_6</th>\n",
       "      <th>U100_rm1_7</th>\n",
       "      <th>U100_rm1_9</th>\n",
       "      <th>U100_rm1_10</th>\n",
       "      <th>V100_rm1_1</th>\n",
       "      <th>V100_rm1_2</th>\n",
       "      <th>V100_rm1_3</th>\n",
       "      <th>V100_rm1_4</th>\n",
       "      <th>V100_rm1_6</th>\n",
       "      <th>V100_rm1_7</th>\n",
       "      <th>V100_rm1_9</th>\n",
       "      <th>V100_rm1_10</th>\n",
       "      <th>U10_rm2_1</th>\n",
       "      <th>U100_rm2_1</th>\n",
       "      <th>V10_rm2_1</th>\n",
       "      <th>V100_rm2_1</th>\n",
       "      <th>U10_rm2_2</th>\n",
       "      <th>U100_rm2_2</th>\n",
       "      <th>V10_rm2_2</th>\n",
       "      <th>V100_rm2_2</th>\n",
       "      <th>U10_rm2_3</th>\n",
       "      <th>U100_rm2_3</th>\n",
       "      <th>V10_rm2_3</th>\n",
       "      <th>V100_rm2_3</th>\n",
       "      <th>U10_rm2_4</th>\n",
       "      <th>U100_rm2_4</th>\n",
       "      <th>V10_rm2_4</th>\n",
       "      <th>V100_rm2_4</th>\n",
       "      <th>U10_rm2_6</th>\n",
       "      <th>U100_rm2_6</th>\n",
       "      <th>V10_rm2_6</th>\n",
       "      <th>V100_rm2_6</th>\n",
       "      <th>U10_rm2_7</th>\n",
       "      <th>U100_rm2_7</th>\n",
       "      <th>V10_rm2_7</th>\n",
       "      <th>V100_rm2_7</th>\n",
       "      <th>U10_rm2_9</th>\n",
       "      <th>U100_rm2_9</th>\n",
       "      <th>V10_rm2_9</th>\n",
       "      <th>V100_rm2_9</th>\n",
       "      <th>U10_rm2_10</th>\n",
       "      <th>U100_rm2_10</th>\n",
       "      <th>V10_rm2_10</th>\n",
       "      <th>V100_rm2_10</th>\n",
       "      <th>U10_rm3_1</th>\n",
       "      <th>U100_rm3_1</th>\n",
       "      <th>V10_rm3_1</th>\n",
       "      <th>V100_rm3_1</th>\n",
       "      <th>U10_rm3_2</th>\n",
       "      <th>U100_rm3_2</th>\n",
       "      <th>V10_rm3_2</th>\n",
       "      <th>V100_rm3_2</th>\n",
       "      <th>U10_rm3_3</th>\n",
       "      <th>U100_rm3_3</th>\n",
       "      <th>V10_rm3_3</th>\n",
       "      <th>V100_rm3_3</th>\n",
       "      <th>U10_rm3_4</th>\n",
       "      <th>U100_rm3_4</th>\n",
       "      <th>V10_rm3_4</th>\n",
       "      <th>V100_rm3_4</th>\n",
       "      <th>U10_rm3_6</th>\n",
       "      <th>U100_rm3_6</th>\n",
       "      <th>V10_rm3_6</th>\n",
       "      <th>V100_rm3_6</th>\n",
       "      <th>U10_rm3_7</th>\n",
       "      <th>U100_rm3_7</th>\n",
       "      <th>V10_rm3_7</th>\n",
       "      <th>V100_rm3_7</th>\n",
       "      <th>U10_rm3_9</th>\n",
       "      <th>U100_rm3_9</th>\n",
       "      <th>V10_rm3_9</th>\n",
       "      <th>V100_rm3_9</th>\n",
       "      <th>U10_rm3_10</th>\n",
       "      <th>U100_rm3_10</th>\n",
       "      <th>V10_rm3_10</th>\n",
       "      <th>V100_rm3_10</th>\n",
       "      <th>U10_rm4_1</th>\n",
       "      <th>U100_rm4_1</th>\n",
       "      <th>V10_rm4_1</th>\n",
       "      <th>V100_rm4_1</th>\n",
       "      <th>U10_rm4_2</th>\n",
       "      <th>U100_rm4_2</th>\n",
       "      <th>V10_rm4_2</th>\n",
       "      <th>V100_rm4_2</th>\n",
       "      <th>U10_rm4_3</th>\n",
       "      <th>U100_rm4_3</th>\n",
       "      <th>V10_rm4_3</th>\n",
       "      <th>V100_rm4_3</th>\n",
       "      <th>U10_rm4_4</th>\n",
       "      <th>U100_rm4_4</th>\n",
       "      <th>V10_rm4_4</th>\n",
       "      <th>V100_rm4_4</th>\n",
       "      <th>U10_rm4_6</th>\n",
       "      <th>U100_rm4_6</th>\n",
       "      <th>V10_rm4_6</th>\n",
       "      <th>V100_rm4_6</th>\n",
       "      <th>U10_rm4_7</th>\n",
       "      <th>U100_rm4_7</th>\n",
       "      <th>V10_rm4_7</th>\n",
       "      <th>V100_rm4_7</th>\n",
       "      <th>U10_rm4_9</th>\n",
       "      <th>U100_rm4_9</th>\n",
       "      <th>V10_rm4_9</th>\n",
       "      <th>V100_rm4_9</th>\n",
       "      <th>U10_rm4_10</th>\n",
       "      <th>U100_rm4_10</th>\n",
       "      <th>V10_rm4_10</th>\n",
       "      <th>V100_rm4_10</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>DAYOFYEAR</th>\n",
       "      <th>HOUR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30010001</td>\n",
       "      <td>30010002</td>\n",
       "      <td>30010003</td>\n",
       "      <td>30010004</td>\n",
       "      <td>30010005</td>\n",
       "      <td>30010006</td>\n",
       "      <td>30010007</td>\n",
       "      <td>30010008</td>\n",
       "      <td>30010009</td>\n",
       "      <td>30010010</td>\n",
       "      <td>0.566472</td>\n",
       "      <td>1.552273</td>\n",
       "      <td>-0.056684</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>1.167131</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>0.781539</td>\n",
       "      <td>-0.165117</td>\n",
       "      <td>-3.866858</td>\n",
       "      <td>-3.037934</td>\n",
       "      <td>-3.197154</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-2.284945</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-3.757118</td>\n",
       "      <td>-2.658255</td>\n",
       "      <td>0.935726</td>\n",
       "      <td>2.819186</td>\n",
       "      <td>0.151737</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>1.829496</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>1.473541</td>\n",
       "      <td>0.320843</td>\n",
       "      <td>-5.888026</td>\n",
       "      <td>-5.077052</td>\n",
       "      <td>-4.559475</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>-3.287274</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>-5.748985</td>\n",
       "      <td>-4.277716</td>\n",
       "      <td>0.566472</td>\n",
       "      <td>0.935726</td>\n",
       "      <td>-3.866858</td>\n",
       "      <td>-5.888026</td>\n",
       "      <td>1.552273</td>\n",
       "      <td>2.819186</td>\n",
       "      <td>-3.037934</td>\n",
       "      <td>-5.077052</td>\n",
       "      <td>-0.056684</td>\n",
       "      <td>0.151737</td>\n",
       "      <td>-3.197154</td>\n",
       "      <td>-4.559475</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>1.167131</td>\n",
       "      <td>1.829496</td>\n",
       "      <td>-2.284945</td>\n",
       "      <td>-3.287274</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>0.781539</td>\n",
       "      <td>1.473541</td>\n",
       "      <td>-3.757118</td>\n",
       "      <td>-5.748985</td>\n",
       "      <td>-0.165117</td>\n",
       "      <td>0.320843</td>\n",
       "      <td>-2.658255</td>\n",
       "      <td>-4.277716</td>\n",
       "      <td>0.566472</td>\n",
       "      <td>0.935726</td>\n",
       "      <td>-3.866858</td>\n",
       "      <td>-5.888026</td>\n",
       "      <td>1.552273</td>\n",
       "      <td>2.819186</td>\n",
       "      <td>-3.037934</td>\n",
       "      <td>-5.077052</td>\n",
       "      <td>-0.056684</td>\n",
       "      <td>0.151737</td>\n",
       "      <td>-3.197154</td>\n",
       "      <td>-4.559475</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>1.167131</td>\n",
       "      <td>1.829496</td>\n",
       "      <td>-2.284945</td>\n",
       "      <td>-3.287274</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>0.781539</td>\n",
       "      <td>1.473541</td>\n",
       "      <td>-3.757118</td>\n",
       "      <td>-5.748985</td>\n",
       "      <td>-0.165117</td>\n",
       "      <td>0.320843</td>\n",
       "      <td>-2.658255</td>\n",
       "      <td>-4.277716</td>\n",
       "      <td>0.566472</td>\n",
       "      <td>0.935726</td>\n",
       "      <td>-3.866858</td>\n",
       "      <td>-5.888026</td>\n",
       "      <td>1.552273</td>\n",
       "      <td>2.819186</td>\n",
       "      <td>-3.037934</td>\n",
       "      <td>-5.077052</td>\n",
       "      <td>-0.056684</td>\n",
       "      <td>0.151737</td>\n",
       "      <td>-3.197154</td>\n",
       "      <td>-4.559475</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>1.167131</td>\n",
       "      <td>1.829496</td>\n",
       "      <td>-2.284945</td>\n",
       "      <td>-3.287274</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>0.781539</td>\n",
       "      <td>1.473541</td>\n",
       "      <td>-3.757118</td>\n",
       "      <td>-5.748985</td>\n",
       "      <td>-0.165117</td>\n",
       "      <td>0.320843</td>\n",
       "      <td>-2.658255</td>\n",
       "      <td>-4.277716</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30010101</td>\n",
       "      <td>30010102</td>\n",
       "      <td>30010103</td>\n",
       "      <td>30010104</td>\n",
       "      <td>30010105</td>\n",
       "      <td>30010106</td>\n",
       "      <td>30010107</td>\n",
       "      <td>30010108</td>\n",
       "      <td>30010109</td>\n",
       "      <td>30010110</td>\n",
       "      <td>2.513531</td>\n",
       "      <td>1.856405</td>\n",
       "      <td>1.865527</td>\n",
       "      <td>2.099645</td>\n",
       "      <td>2.254329</td>\n",
       "      <td>-0.144996</td>\n",
       "      <td>2.461843</td>\n",
       "      <td>-0.349089</td>\n",
       "      <td>-4.475081</td>\n",
       "      <td>-3.909570</td>\n",
       "      <td>-3.798061</td>\n",
       "      <td>-1.608958</td>\n",
       "      <td>-1.636207</td>\n",
       "      <td>-5.039754</td>\n",
       "      <td>-4.089410</td>\n",
       "      <td>-3.789677</td>\n",
       "      <td>3.426426</td>\n",
       "      <td>2.678273</td>\n",
       "      <td>2.653962</td>\n",
       "      <td>2.709924</td>\n",
       "      <td>2.918636</td>\n",
       "      <td>-0.033150</td>\n",
       "      <td>3.378262</td>\n",
       "      <td>-0.350576</td>\n",
       "      <td>-6.415826</td>\n",
       "      <td>-5.350381</td>\n",
       "      <td>-5.098012</td>\n",
       "      <td>-2.039563</td>\n",
       "      <td>-2.081795</td>\n",
       "      <td>-7.279274</td>\n",
       "      <td>-5.847995</td>\n",
       "      <td>-5.323414</td>\n",
       "      <td>1.540001</td>\n",
       "      <td>2.181076</td>\n",
       "      <td>-4.170969</td>\n",
       "      <td>-6.151926</td>\n",
       "      <td>1.704339</td>\n",
       "      <td>2.748730</td>\n",
       "      <td>-3.473752</td>\n",
       "      <td>-5.213717</td>\n",
       "      <td>0.904422</td>\n",
       "      <td>1.402849</td>\n",
       "      <td>-3.497607</td>\n",
       "      <td>-4.828743</td>\n",
       "      <td>1.395599</td>\n",
       "      <td>1.986110</td>\n",
       "      <td>-1.745600</td>\n",
       "      <td>-2.396974</td>\n",
       "      <td>1.710730</td>\n",
       "      <td>2.374066</td>\n",
       "      <td>-1.960576</td>\n",
       "      <td>-2.684534</td>\n",
       "      <td>-0.305109</td>\n",
       "      <td>-0.258960</td>\n",
       "      <td>-4.716631</td>\n",
       "      <td>-7.021601</td>\n",
       "      <td>1.621691</td>\n",
       "      <td>2.425901</td>\n",
       "      <td>-3.923264</td>\n",
       "      <td>-5.798490</td>\n",
       "      <td>-0.257103</td>\n",
       "      <td>-0.014866</td>\n",
       "      <td>-3.223966</td>\n",
       "      <td>-4.800565</td>\n",
       "      <td>1.540001</td>\n",
       "      <td>2.181076</td>\n",
       "      <td>-4.170969</td>\n",
       "      <td>-6.151926</td>\n",
       "      <td>1.704339</td>\n",
       "      <td>2.748730</td>\n",
       "      <td>-3.473752</td>\n",
       "      <td>-5.213717</td>\n",
       "      <td>0.904422</td>\n",
       "      <td>1.402849</td>\n",
       "      <td>-3.497607</td>\n",
       "      <td>-4.828743</td>\n",
       "      <td>1.395599</td>\n",
       "      <td>1.986110</td>\n",
       "      <td>-1.745600</td>\n",
       "      <td>-2.396974</td>\n",
       "      <td>1.710730</td>\n",
       "      <td>2.374066</td>\n",
       "      <td>-1.960576</td>\n",
       "      <td>-2.684534</td>\n",
       "      <td>-0.305109</td>\n",
       "      <td>-0.258960</td>\n",
       "      <td>-4.716631</td>\n",
       "      <td>-7.021601</td>\n",
       "      <td>1.621691</td>\n",
       "      <td>2.425901</td>\n",
       "      <td>-3.923264</td>\n",
       "      <td>-5.798490</td>\n",
       "      <td>-0.257103</td>\n",
       "      <td>-0.014866</td>\n",
       "      <td>-3.223966</td>\n",
       "      <td>-4.800565</td>\n",
       "      <td>1.540001</td>\n",
       "      <td>2.181076</td>\n",
       "      <td>-4.170969</td>\n",
       "      <td>-6.151926</td>\n",
       "      <td>1.704339</td>\n",
       "      <td>2.748730</td>\n",
       "      <td>-3.473752</td>\n",
       "      <td>-5.213717</td>\n",
       "      <td>0.904422</td>\n",
       "      <td>1.402849</td>\n",
       "      <td>-3.497607</td>\n",
       "      <td>-4.828743</td>\n",
       "      <td>1.395599</td>\n",
       "      <td>1.986110</td>\n",
       "      <td>-1.745600</td>\n",
       "      <td>-2.396974</td>\n",
       "      <td>1.710730</td>\n",
       "      <td>2.374066</td>\n",
       "      <td>-1.960576</td>\n",
       "      <td>-2.684534</td>\n",
       "      <td>-0.305109</td>\n",
       "      <td>-0.258960</td>\n",
       "      <td>-4.716631</td>\n",
       "      <td>-7.021601</td>\n",
       "      <td>1.621691</td>\n",
       "      <td>2.425901</td>\n",
       "      <td>-3.923264</td>\n",
       "      <td>-5.798490</td>\n",
       "      <td>-0.257103</td>\n",
       "      <td>-0.014866</td>\n",
       "      <td>-3.223966</td>\n",
       "      <td>-4.800565</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30010201</td>\n",
       "      <td>30010202</td>\n",
       "      <td>30010203</td>\n",
       "      <td>30010204</td>\n",
       "      <td>30010205</td>\n",
       "      <td>30010206</td>\n",
       "      <td>30010207</td>\n",
       "      <td>30010208</td>\n",
       "      <td>30010209</td>\n",
       "      <td>30010210</td>\n",
       "      <td>2.730166</td>\n",
       "      <td>2.283594</td>\n",
       "      <td>1.522712</td>\n",
       "      <td>2.638951</td>\n",
       "      <td>2.653013</td>\n",
       "      <td>0.056817</td>\n",
       "      <td>2.637051</td>\n",
       "      <td>0.769431</td>\n",
       "      <td>-4.752177</td>\n",
       "      <td>-4.232779</td>\n",
       "      <td>-3.935561</td>\n",
       "      <td>-1.645848</td>\n",
       "      <td>-1.640399</td>\n",
       "      <td>-5.192765</td>\n",
       "      <td>-4.283922</td>\n",
       "      <td>-4.633122</td>\n",
       "      <td>3.915867</td>\n",
       "      <td>3.169549</td>\n",
       "      <td>2.140667</td>\n",
       "      <td>3.317253</td>\n",
       "      <td>3.320464</td>\n",
       "      <td>0.207672</td>\n",
       "      <td>3.845685</td>\n",
       "      <td>1.020502</td>\n",
       "      <td>-6.847804</td>\n",
       "      <td>-5.587994</td>\n",
       "      <td>-5.242004</td>\n",
       "      <td>-2.112832</td>\n",
       "      <td>-2.149975</td>\n",
       "      <td>-7.343893</td>\n",
       "      <td>-6.058642</td>\n",
       "      <td>-6.176686</td>\n",
       "      <td>2.621848</td>\n",
       "      <td>3.671147</td>\n",
       "      <td>-4.613629</td>\n",
       "      <td>-6.631815</td>\n",
       "      <td>2.070000</td>\n",
       "      <td>2.923911</td>\n",
       "      <td>-4.071175</td>\n",
       "      <td>-5.469187</td>\n",
       "      <td>1.694119</td>\n",
       "      <td>2.397315</td>\n",
       "      <td>-3.866811</td>\n",
       "      <td>-5.170008</td>\n",
       "      <td>2.369298</td>\n",
       "      <td>3.013589</td>\n",
       "      <td>-1.627403</td>\n",
       "      <td>-2.076198</td>\n",
       "      <td>2.453671</td>\n",
       "      <td>3.119550</td>\n",
       "      <td>-1.638303</td>\n",
       "      <td>-2.115885</td>\n",
       "      <td>-0.044090</td>\n",
       "      <td>0.087261</td>\n",
       "      <td>-5.116259</td>\n",
       "      <td>-7.311583</td>\n",
       "      <td>2.549447</td>\n",
       "      <td>3.611973</td>\n",
       "      <td>-4.186666</td>\n",
       "      <td>-5.953319</td>\n",
       "      <td>0.210171</td>\n",
       "      <td>0.334963</td>\n",
       "      <td>-4.211399</td>\n",
       "      <td>-5.750050</td>\n",
       "      <td>1.936723</td>\n",
       "      <td>2.759340</td>\n",
       "      <td>-4.364705</td>\n",
       "      <td>-6.383885</td>\n",
       "      <td>1.897424</td>\n",
       "      <td>2.889003</td>\n",
       "      <td>-3.726761</td>\n",
       "      <td>-5.338476</td>\n",
       "      <td>1.110518</td>\n",
       "      <td>1.648789</td>\n",
       "      <td>-3.643592</td>\n",
       "      <td>-4.966497</td>\n",
       "      <td>1.810050</td>\n",
       "      <td>2.429824</td>\n",
       "      <td>-1.712349</td>\n",
       "      <td>-2.302260</td>\n",
       "      <td>2.024825</td>\n",
       "      <td>2.689532</td>\n",
       "      <td>-1.853850</td>\n",
       "      <td>-2.506348</td>\n",
       "      <td>-0.184467</td>\n",
       "      <td>-0.103416</td>\n",
       "      <td>-4.875342</td>\n",
       "      <td>-7.129031</td>\n",
       "      <td>1.960144</td>\n",
       "      <td>2.899162</td>\n",
       "      <td>-4.043484</td>\n",
       "      <td>-5.885207</td>\n",
       "      <td>0.085075</td>\n",
       "      <td>0.330257</td>\n",
       "      <td>-3.693685</td>\n",
       "      <td>-5.259272</td>\n",
       "      <td>1.936723</td>\n",
       "      <td>2.759340</td>\n",
       "      <td>-4.364705</td>\n",
       "      <td>-6.383885</td>\n",
       "      <td>1.897424</td>\n",
       "      <td>2.889003</td>\n",
       "      <td>-3.726761</td>\n",
       "      <td>-5.338476</td>\n",
       "      <td>1.110518</td>\n",
       "      <td>1.648789</td>\n",
       "      <td>-3.643592</td>\n",
       "      <td>-4.966497</td>\n",
       "      <td>1.810050</td>\n",
       "      <td>2.429824</td>\n",
       "      <td>-1.712349</td>\n",
       "      <td>-2.302260</td>\n",
       "      <td>2.024825</td>\n",
       "      <td>2.689532</td>\n",
       "      <td>-1.853850</td>\n",
       "      <td>-2.506348</td>\n",
       "      <td>-0.184467</td>\n",
       "      <td>-0.103416</td>\n",
       "      <td>-4.875342</td>\n",
       "      <td>-7.129031</td>\n",
       "      <td>1.960144</td>\n",
       "      <td>2.899162</td>\n",
       "      <td>-4.043484</td>\n",
       "      <td>-5.885207</td>\n",
       "      <td>0.085075</td>\n",
       "      <td>0.330257</td>\n",
       "      <td>-3.693685</td>\n",
       "      <td>-5.259272</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30010301</td>\n",
       "      <td>30010302</td>\n",
       "      <td>30010303</td>\n",
       "      <td>30010304</td>\n",
       "      <td>30010305</td>\n",
       "      <td>30010306</td>\n",
       "      <td>30010307</td>\n",
       "      <td>30010308</td>\n",
       "      <td>30010309</td>\n",
       "      <td>30010310</td>\n",
       "      <td>2.461083</td>\n",
       "      <td>3.095404</td>\n",
       "      <td>1.449740</td>\n",
       "      <td>2.958202</td>\n",
       "      <td>2.892832</td>\n",
       "      <td>0.359725</td>\n",
       "      <td>2.187819</td>\n",
       "      <td>1.873888</td>\n",
       "      <td>-4.583656</td>\n",
       "      <td>-4.070965</td>\n",
       "      <td>-4.013953</td>\n",
       "      <td>-1.850422</td>\n",
       "      <td>-1.892762</td>\n",
       "      <td>-5.167193</td>\n",
       "      <td>-4.210980</td>\n",
       "      <td>-5.047719</td>\n",
       "      <td>3.541562</td>\n",
       "      <td>4.191092</td>\n",
       "      <td>2.054430</td>\n",
       "      <td>3.704862</td>\n",
       "      <td>3.611744</td>\n",
       "      <td>0.637481</td>\n",
       "      <td>3.203953</td>\n",
       "      <td>2.476900</td>\n",
       "      <td>-6.545063</td>\n",
       "      <td>-5.364627</td>\n",
       "      <td>-5.389050</td>\n",
       "      <td>-2.391150</td>\n",
       "      <td>-2.497491</td>\n",
       "      <td>-7.275712</td>\n",
       "      <td>-5.923299</td>\n",
       "      <td>-6.710426</td>\n",
       "      <td>2.595624</td>\n",
       "      <td>3.728715</td>\n",
       "      <td>-4.667917</td>\n",
       "      <td>-6.696434</td>\n",
       "      <td>2.689499</td>\n",
       "      <td>3.680321</td>\n",
       "      <td>-4.151872</td>\n",
       "      <td>-5.476311</td>\n",
       "      <td>1.486226</td>\n",
       "      <td>2.097549</td>\n",
       "      <td>-3.974757</td>\n",
       "      <td>-5.315527</td>\n",
       "      <td>2.798577</td>\n",
       "      <td>3.511058</td>\n",
       "      <td>-1.748135</td>\n",
       "      <td>-2.251991</td>\n",
       "      <td>2.772923</td>\n",
       "      <td>3.466104</td>\n",
       "      <td>-1.766580</td>\n",
       "      <td>-2.323733</td>\n",
       "      <td>0.208271</td>\n",
       "      <td>0.422577</td>\n",
       "      <td>-5.179979</td>\n",
       "      <td>-7.309802</td>\n",
       "      <td>2.412435</td>\n",
       "      <td>3.524819</td>\n",
       "      <td>-4.247451</td>\n",
       "      <td>-5.990971</td>\n",
       "      <td>1.321660</td>\n",
       "      <td>1.748701</td>\n",
       "      <td>-4.840421</td>\n",
       "      <td>-6.443556</td>\n",
       "      <td>2.568260</td>\n",
       "      <td>3.627952</td>\n",
       "      <td>-4.603638</td>\n",
       "      <td>-6.602898</td>\n",
       "      <td>2.411801</td>\n",
       "      <td>3.346305</td>\n",
       "      <td>-4.071105</td>\n",
       "      <td>-5.434334</td>\n",
       "      <td>1.612660</td>\n",
       "      <td>2.283020</td>\n",
       "      <td>-3.915858</td>\n",
       "      <td>-5.243022</td>\n",
       "      <td>2.565599</td>\n",
       "      <td>3.244013</td>\n",
       "      <td>-1.701743</td>\n",
       "      <td>-2.181182</td>\n",
       "      <td>2.600058</td>\n",
       "      <td>3.283615</td>\n",
       "      <td>-1.723122</td>\n",
       "      <td>-2.243087</td>\n",
       "      <td>0.090515</td>\n",
       "      <td>0.270668</td>\n",
       "      <td>-5.133237</td>\n",
       "      <td>-7.299626</td>\n",
       "      <td>2.428904</td>\n",
       "      <td>3.475966</td>\n",
       "      <td>-4.194771</td>\n",
       "      <td>-5.943312</td>\n",
       "      <td>0.764744</td>\n",
       "      <td>1.048942</td>\n",
       "      <td>-4.490173</td>\n",
       "      <td>-6.070175</td>\n",
       "      <td>2.067813</td>\n",
       "      <td>2.954895</td>\n",
       "      <td>-4.419443</td>\n",
       "      <td>-6.424180</td>\n",
       "      <td>2.196919</td>\n",
       "      <td>3.214525</td>\n",
       "      <td>-3.812812</td>\n",
       "      <td>-5.345014</td>\n",
       "      <td>1.195324</td>\n",
       "      <td>1.750199</td>\n",
       "      <td>-3.736182</td>\n",
       "      <td>-5.072135</td>\n",
       "      <td>2.097088</td>\n",
       "      <td>2.748584</td>\n",
       "      <td>-1.746867</td>\n",
       "      <td>-2.324482</td>\n",
       "      <td>2.241826</td>\n",
       "      <td>2.920085</td>\n",
       "      <td>-1.863578</td>\n",
       "      <td>-2.504134</td>\n",
       "      <td>-0.048419</td>\n",
       "      <td>0.081808</td>\n",
       "      <td>-4.948305</td>\n",
       "      <td>-7.165701</td>\n",
       "      <td>2.017063</td>\n",
       "      <td>2.975360</td>\n",
       "      <td>-4.085358</td>\n",
       "      <td>-5.894730</td>\n",
       "      <td>0.532278</td>\n",
       "      <td>0.866918</td>\n",
       "      <td>-4.032193</td>\n",
       "      <td>-5.622060</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30010401</td>\n",
       "      <td>30010402</td>\n",
       "      <td>30010403</td>\n",
       "      <td>30010404</td>\n",
       "      <td>30010405</td>\n",
       "      <td>30010406</td>\n",
       "      <td>30010407</td>\n",
       "      <td>30010408</td>\n",
       "      <td>30010409</td>\n",
       "      <td>30010410</td>\n",
       "      <td>2.146012</td>\n",
       "      <td>3.417696</td>\n",
       "      <td>1.553117</td>\n",
       "      <td>3.497509</td>\n",
       "      <td>3.488768</td>\n",
       "      <td>0.528852</td>\n",
       "      <td>2.050237</td>\n",
       "      <td>2.396472</td>\n",
       "      <td>-4.407169</td>\n",
       "      <td>-3.920889</td>\n",
       "      <td>-4.087314</td>\n",
       "      <td>-1.891085</td>\n",
       "      <td>-1.964027</td>\n",
       "      <td>-5.059037</td>\n",
       "      <td>-4.223137</td>\n",
       "      <td>-5.095089</td>\n",
       "      <td>3.158082</td>\n",
       "      <td>4.636039</td>\n",
       "      <td>2.179199</td>\n",
       "      <td>4.430997</td>\n",
       "      <td>4.420446</td>\n",
       "      <td>0.868211</td>\n",
       "      <td>3.020928</td>\n",
       "      <td>3.220925</td>\n",
       "      <td>-6.276412</td>\n",
       "      <td>-5.191124</td>\n",
       "      <td>-5.500988</td>\n",
       "      <td>-2.484262</td>\n",
       "      <td>-2.632834</td>\n",
       "      <td>-7.140369</td>\n",
       "      <td>-5.934493</td>\n",
       "      <td>-6.825925</td>\n",
       "      <td>2.303547</td>\n",
       "      <td>3.349822</td>\n",
       "      <td>-4.495413</td>\n",
       "      <td>-6.410738</td>\n",
       "      <td>3.256550</td>\n",
       "      <td>4.413566</td>\n",
       "      <td>-3.995927</td>\n",
       "      <td>-5.277875</td>\n",
       "      <td>1.501428</td>\n",
       "      <td>2.116815</td>\n",
       "      <td>-4.050633</td>\n",
       "      <td>-5.445019</td>\n",
       "      <td>3.227856</td>\n",
       "      <td>4.067929</td>\n",
       "      <td>-1.870753</td>\n",
       "      <td>-2.437706</td>\n",
       "      <td>3.190800</td>\n",
       "      <td>4.016095</td>\n",
       "      <td>-1.928394</td>\n",
       "      <td>-2.565163</td>\n",
       "      <td>0.444289</td>\n",
       "      <td>0.752846</td>\n",
       "      <td>-5.113115</td>\n",
       "      <td>-7.208041</td>\n",
       "      <td>2.119028</td>\n",
       "      <td>3.112440</td>\n",
       "      <td>-4.217059</td>\n",
       "      <td>-5.928896</td>\n",
       "      <td>2.135180</td>\n",
       "      <td>2.848913</td>\n",
       "      <td>-5.071404</td>\n",
       "      <td>-6.768176</td>\n",
       "      <td>2.445753</td>\n",
       "      <td>3.538504</td>\n",
       "      <td>-4.581001</td>\n",
       "      <td>-6.556426</td>\n",
       "      <td>2.932232</td>\n",
       "      <td>3.998894</td>\n",
       "      <td>-4.074878</td>\n",
       "      <td>-5.381248</td>\n",
       "      <td>1.508523</td>\n",
       "      <td>2.124766</td>\n",
       "      <td>-4.012276</td>\n",
       "      <td>-5.377348</td>\n",
       "      <td>3.031554</td>\n",
       "      <td>3.817704</td>\n",
       "      <td>-1.795785</td>\n",
       "      <td>-2.329415</td>\n",
       "      <td>3.011538</td>\n",
       "      <td>3.784218</td>\n",
       "      <td>-1.832396</td>\n",
       "      <td>-2.426767</td>\n",
       "      <td>0.315131</td>\n",
       "      <td>0.571121</td>\n",
       "      <td>-5.139665</td>\n",
       "      <td>-7.253325</td>\n",
       "      <td>2.291702</td>\n",
       "      <td>3.356855</td>\n",
       "      <td>-4.239347</td>\n",
       "      <td>-5.972145</td>\n",
       "      <td>1.679930</td>\n",
       "      <td>2.239443</td>\n",
       "      <td>-4.925310</td>\n",
       "      <td>-6.571012</td>\n",
       "      <td>2.462698</td>\n",
       "      <td>3.510484</td>\n",
       "      <td>-4.554521</td>\n",
       "      <td>-6.521276</td>\n",
       "      <td>2.663275</td>\n",
       "      <td>3.668738</td>\n",
       "      <td>-4.033551</td>\n",
       "      <td>-5.373531</td>\n",
       "      <td>1.597774</td>\n",
       "      <td>2.257065</td>\n",
       "      <td>-3.958722</td>\n",
       "      <td>-5.307514</td>\n",
       "      <td>2.798577</td>\n",
       "      <td>3.540759</td>\n",
       "      <td>-1.749078</td>\n",
       "      <td>-2.256952</td>\n",
       "      <td>2.822236</td>\n",
       "      <td>3.567823</td>\n",
       "      <td>-1.783348</td>\n",
       "      <td>-2.340524</td>\n",
       "      <td>0.200100</td>\n",
       "      <td>0.420054</td>\n",
       "      <td>-5.114687</td>\n",
       "      <td>-7.259812</td>\n",
       "      <td>2.334237</td>\n",
       "      <td>3.362207</td>\n",
       "      <td>-4.201863</td>\n",
       "      <td>-5.941107</td>\n",
       "      <td>1.172676</td>\n",
       "      <td>1.591938</td>\n",
       "      <td>-4.641402</td>\n",
       "      <td>-6.259113</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data frame dimensions: (2929, 141)\n",
      "\n",
      "Testing data frame saved to data/Test_pp_2017_11_04_22_01_57.feather\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Preprocess the testing data set in Feather format for notebooks running R kernel.\n",
    "'''\n",
    "test_csv  = \"data/Test_uP7dymh.csv\"\n",
    "\n",
    "test_df = preProcess(test_csv, Nroll=4, inclusive=True, keepID=True)\n",
    "\n",
    "# Rename dataframe columns\n",
    "predictors  = [item[0]+'_'+str(item[1]) for item in test_df.columns.values]\n",
    "predictors  = [predictor if predictor[-1]!='_' else predictor[:-1] for predictor in predictors]\n",
    "test_df.columns = predictors\n",
    "display_html(test_df.head())\n",
    "print(\"Data frame dimensions: {0}\".format(test_df.shape))\n",
    "    \n",
    "# Output data frame in feather format\n",
    "test_fname = \"data/Test_pp_\" + strftime(\"%Y_%m_%d_%H_%M_%S\") + \".feather\"\n",
    "try:\n",
    "    feather.write_dataframe(test_df, test_fname)\n",
    "    print('\\nTesting data frame saved to {0}'.format(test_fname))\n",
    "except:\n",
    "    print('\\nError saving testing data frame to {0}'.format(test_fname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Cell7'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID column dropped\n",
      "Number of time points:                 2929\n",
      "Number of delta-times equal to 1 hour: 2928\n",
      "No rolling means computed\n",
      "\n",
      "Dataframe solution_df for pickle:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"10\" halign=\"left\">TARGETVAR</th>\n",
       "      <th colspan=\"10\" halign=\"left\">U10</th>\n",
       "      <th colspan=\"10\" halign=\"left\">V10</th>\n",
       "      <th colspan=\"10\" halign=\"left\">U100</th>\n",
       "      <th colspan=\"10\" halign=\"left\">V100</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>DAYOFYEAR</th>\n",
       "      <th>HOUR</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ZONEID</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.270840</td>\n",
       "      <td>0.070839</td>\n",
       "      <td>0.032609</td>\n",
       "      <td>0.029816</td>\n",
       "      <td>0.052044</td>\n",
       "      <td>0.140550</td>\n",
       "      <td>0.224602</td>\n",
       "      <td>0.054787</td>\n",
       "      <td>0.121128</td>\n",
       "      <td>0.566472</td>\n",
       "      <td>1.552273</td>\n",
       "      <td>-0.056684</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>1.167131</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>0.781539</td>\n",
       "      <td>-0.165117</td>\n",
       "      <td>-3.866858</td>\n",
       "      <td>-3.037934</td>\n",
       "      <td>-3.197154</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-2.284945</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-3.757118</td>\n",
       "      <td>-2.658255</td>\n",
       "      <td>0.935726</td>\n",
       "      <td>2.819186</td>\n",
       "      <td>0.151737</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>1.829496</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>1.473541</td>\n",
       "      <td>0.320843</td>\n",
       "      <td>-5.888026</td>\n",
       "      <td>-5.077052</td>\n",
       "      <td>-4.559475</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>-3.287274</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>-5.748985</td>\n",
       "      <td>-4.277716</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.019168</td>\n",
       "      <td>0.195737</td>\n",
       "      <td>0.081343</td>\n",
       "      <td>0.028040</td>\n",
       "      <td>0.041659</td>\n",
       "      <td>0.051252</td>\n",
       "      <td>0.152884</td>\n",
       "      <td>0.150140</td>\n",
       "      <td>0.184262</td>\n",
       "      <td>0.235245</td>\n",
       "      <td>2.513531</td>\n",
       "      <td>1.856405</td>\n",
       "      <td>1.865527</td>\n",
       "      <td>2.099645</td>\n",
       "      <td>2.099645</td>\n",
       "      <td>2.254329</td>\n",
       "      <td>-0.144996</td>\n",
       "      <td>-0.144996</td>\n",
       "      <td>2.461843</td>\n",
       "      <td>-0.349089</td>\n",
       "      <td>-4.475081</td>\n",
       "      <td>-3.909570</td>\n",
       "      <td>-3.798061</td>\n",
       "      <td>-1.608958</td>\n",
       "      <td>-1.608958</td>\n",
       "      <td>-1.636207</td>\n",
       "      <td>-5.039754</td>\n",
       "      <td>-5.039754</td>\n",
       "      <td>-4.089410</td>\n",
       "      <td>-3.789677</td>\n",
       "      <td>3.426426</td>\n",
       "      <td>2.678273</td>\n",
       "      <td>2.653962</td>\n",
       "      <td>2.709924</td>\n",
       "      <td>2.709924</td>\n",
       "      <td>2.918636</td>\n",
       "      <td>-0.033150</td>\n",
       "      <td>-0.033150</td>\n",
       "      <td>3.378262</td>\n",
       "      <td>-0.350576</td>\n",
       "      <td>-6.415826</td>\n",
       "      <td>-5.350381</td>\n",
       "      <td>-5.098012</td>\n",
       "      <td>-2.039563</td>\n",
       "      <td>-2.039563</td>\n",
       "      <td>-2.081795</td>\n",
       "      <td>-7.279274</td>\n",
       "      <td>-7.279274</td>\n",
       "      <td>-5.847995</td>\n",
       "      <td>-5.323414</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.286627</td>\n",
       "      <td>0.207469</td>\n",
       "      <td>0.107185</td>\n",
       "      <td>0.064430</td>\n",
       "      <td>0.052353</td>\n",
       "      <td>0.086763</td>\n",
       "      <td>0.199029</td>\n",
       "      <td>0.151965</td>\n",
       "      <td>0.389327</td>\n",
       "      <td>0.271003</td>\n",
       "      <td>2.730166</td>\n",
       "      <td>2.283594</td>\n",
       "      <td>1.522712</td>\n",
       "      <td>2.638951</td>\n",
       "      <td>2.638951</td>\n",
       "      <td>2.653013</td>\n",
       "      <td>0.056817</td>\n",
       "      <td>0.056817</td>\n",
       "      <td>2.637051</td>\n",
       "      <td>0.769431</td>\n",
       "      <td>-4.752177</td>\n",
       "      <td>-4.232779</td>\n",
       "      <td>-3.935561</td>\n",
       "      <td>-1.645848</td>\n",
       "      <td>-1.645848</td>\n",
       "      <td>-1.640399</td>\n",
       "      <td>-5.192765</td>\n",
       "      <td>-5.192765</td>\n",
       "      <td>-4.283922</td>\n",
       "      <td>-4.633122</td>\n",
       "      <td>3.915867</td>\n",
       "      <td>3.169549</td>\n",
       "      <td>2.140667</td>\n",
       "      <td>3.317253</td>\n",
       "      <td>3.317253</td>\n",
       "      <td>3.320464</td>\n",
       "      <td>0.207672</td>\n",
       "      <td>0.207672</td>\n",
       "      <td>3.845685</td>\n",
       "      <td>1.020502</td>\n",
       "      <td>-6.847804</td>\n",
       "      <td>-5.587994</td>\n",
       "      <td>-5.242004</td>\n",
       "      <td>-2.112832</td>\n",
       "      <td>-2.112832</td>\n",
       "      <td>-2.149975</td>\n",
       "      <td>-7.343893</td>\n",
       "      <td>-7.343893</td>\n",
       "      <td>-6.058642</td>\n",
       "      <td>-6.176686</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.287943</td>\n",
       "      <td>0.395594</td>\n",
       "      <td>0.157022</td>\n",
       "      <td>0.055293</td>\n",
       "      <td>0.052712</td>\n",
       "      <td>0.103719</td>\n",
       "      <td>0.281050</td>\n",
       "      <td>0.217666</td>\n",
       "      <td>0.503941</td>\n",
       "      <td>0.272300</td>\n",
       "      <td>2.461083</td>\n",
       "      <td>3.095404</td>\n",
       "      <td>1.449740</td>\n",
       "      <td>2.958202</td>\n",
       "      <td>2.958202</td>\n",
       "      <td>2.892832</td>\n",
       "      <td>0.359725</td>\n",
       "      <td>0.359725</td>\n",
       "      <td>2.187819</td>\n",
       "      <td>1.873888</td>\n",
       "      <td>-4.583656</td>\n",
       "      <td>-4.070965</td>\n",
       "      <td>-4.013953</td>\n",
       "      <td>-1.850422</td>\n",
       "      <td>-1.850422</td>\n",
       "      <td>-1.892762</td>\n",
       "      <td>-5.167193</td>\n",
       "      <td>-5.167193</td>\n",
       "      <td>-4.210980</td>\n",
       "      <td>-5.047719</td>\n",
       "      <td>3.541562</td>\n",
       "      <td>4.191092</td>\n",
       "      <td>2.054430</td>\n",
       "      <td>3.704862</td>\n",
       "      <td>3.704862</td>\n",
       "      <td>3.611744</td>\n",
       "      <td>0.637481</td>\n",
       "      <td>0.637481</td>\n",
       "      <td>3.203953</td>\n",
       "      <td>2.476900</td>\n",
       "      <td>-6.545063</td>\n",
       "      <td>-5.364627</td>\n",
       "      <td>-5.389050</td>\n",
       "      <td>-2.391150</td>\n",
       "      <td>-2.391150</td>\n",
       "      <td>-2.497491</td>\n",
       "      <td>-7.275712</td>\n",
       "      <td>-7.275712</td>\n",
       "      <td>-5.923299</td>\n",
       "      <td>-6.710426</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.352317</td>\n",
       "      <td>0.509844</td>\n",
       "      <td>0.188443</td>\n",
       "      <td>0.143825</td>\n",
       "      <td>0.089402</td>\n",
       "      <td>0.111278</td>\n",
       "      <td>0.307617</td>\n",
       "      <td>0.312690</td>\n",
       "      <td>0.389198</td>\n",
       "      <td>0.331207</td>\n",
       "      <td>2.146012</td>\n",
       "      <td>3.417696</td>\n",
       "      <td>1.553117</td>\n",
       "      <td>3.497509</td>\n",
       "      <td>3.497509</td>\n",
       "      <td>3.488768</td>\n",
       "      <td>0.528852</td>\n",
       "      <td>0.528852</td>\n",
       "      <td>2.050237</td>\n",
       "      <td>2.396472</td>\n",
       "      <td>-4.407169</td>\n",
       "      <td>-3.920889</td>\n",
       "      <td>-4.087314</td>\n",
       "      <td>-1.891085</td>\n",
       "      <td>-1.891085</td>\n",
       "      <td>-1.964027</td>\n",
       "      <td>-5.059037</td>\n",
       "      <td>-5.059037</td>\n",
       "      <td>-4.223137</td>\n",
       "      <td>-5.095089</td>\n",
       "      <td>3.158082</td>\n",
       "      <td>4.636039</td>\n",
       "      <td>2.179199</td>\n",
       "      <td>4.430997</td>\n",
       "      <td>4.430997</td>\n",
       "      <td>4.420446</td>\n",
       "      <td>0.868211</td>\n",
       "      <td>0.868211</td>\n",
       "      <td>3.020928</td>\n",
       "      <td>3.220925</td>\n",
       "      <td>-6.276412</td>\n",
       "      <td>-5.191124</td>\n",
       "      <td>-5.500988</td>\n",
       "      <td>-2.484262</td>\n",
       "      <td>-2.484262</td>\n",
       "      <td>-2.632834</td>\n",
       "      <td>-7.140369</td>\n",
       "      <td>-7.140369</td>\n",
       "      <td>-5.934493</td>\n",
       "      <td>-6.825925</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Solution data frame saved to data/Solution_pp_2018_10_21_00_41_10.pkl\n",
      "\n",
      "Dataframe solution_df for feather:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TARGETVAR_1</th>\n",
       "      <th>TARGETVAR_2</th>\n",
       "      <th>TARGETVAR_3</th>\n",
       "      <th>TARGETVAR_4</th>\n",
       "      <th>TARGETVAR_5</th>\n",
       "      <th>TARGETVAR_6</th>\n",
       "      <th>TARGETVAR_7</th>\n",
       "      <th>TARGETVAR_8</th>\n",
       "      <th>TARGETVAR_9</th>\n",
       "      <th>TARGETVAR_10</th>\n",
       "      <th>U10_1</th>\n",
       "      <th>U10_2</th>\n",
       "      <th>U10_3</th>\n",
       "      <th>U10_4</th>\n",
       "      <th>U10_5</th>\n",
       "      <th>U10_6</th>\n",
       "      <th>U10_7</th>\n",
       "      <th>U10_8</th>\n",
       "      <th>U10_9</th>\n",
       "      <th>U10_10</th>\n",
       "      <th>V10_1</th>\n",
       "      <th>V10_2</th>\n",
       "      <th>V10_3</th>\n",
       "      <th>V10_4</th>\n",
       "      <th>V10_5</th>\n",
       "      <th>V10_6</th>\n",
       "      <th>V10_7</th>\n",
       "      <th>V10_8</th>\n",
       "      <th>V10_9</th>\n",
       "      <th>V10_10</th>\n",
       "      <th>U100_1</th>\n",
       "      <th>U100_2</th>\n",
       "      <th>U100_3</th>\n",
       "      <th>U100_4</th>\n",
       "      <th>U100_5</th>\n",
       "      <th>U100_6</th>\n",
       "      <th>U100_7</th>\n",
       "      <th>U100_8</th>\n",
       "      <th>U100_9</th>\n",
       "      <th>U100_10</th>\n",
       "      <th>V100_1</th>\n",
       "      <th>V100_2</th>\n",
       "      <th>V100_3</th>\n",
       "      <th>V100_4</th>\n",
       "      <th>V100_5</th>\n",
       "      <th>V100_6</th>\n",
       "      <th>V100_7</th>\n",
       "      <th>V100_8</th>\n",
       "      <th>V100_9</th>\n",
       "      <th>V100_10</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>DAYOFYEAR</th>\n",
       "      <th>HOUR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.270840</td>\n",
       "      <td>0.070839</td>\n",
       "      <td>0.032609</td>\n",
       "      <td>0.029816</td>\n",
       "      <td>0.052044</td>\n",
       "      <td>0.140550</td>\n",
       "      <td>0.224602</td>\n",
       "      <td>0.054787</td>\n",
       "      <td>0.121128</td>\n",
       "      <td>0.566472</td>\n",
       "      <td>1.552273</td>\n",
       "      <td>-0.056684</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>0.691553</td>\n",
       "      <td>1.167131</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>-0.465222</td>\n",
       "      <td>0.781539</td>\n",
       "      <td>-0.165117</td>\n",
       "      <td>-3.866858</td>\n",
       "      <td>-3.037934</td>\n",
       "      <td>-3.197154</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-1.882241</td>\n",
       "      <td>-2.284945</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-4.393509</td>\n",
       "      <td>-3.757118</td>\n",
       "      <td>-2.658255</td>\n",
       "      <td>0.935726</td>\n",
       "      <td>2.819186</td>\n",
       "      <td>0.151737</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>1.262296</td>\n",
       "      <td>1.829496</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>-0.484770</td>\n",
       "      <td>1.473541</td>\n",
       "      <td>0.320843</td>\n",
       "      <td>-5.888026</td>\n",
       "      <td>-5.077052</td>\n",
       "      <td>-4.559475</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>-2.754384</td>\n",
       "      <td>-3.287274</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>-6.763927</td>\n",
       "      <td>-5.748985</td>\n",
       "      <td>-4.277716</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.019168</td>\n",
       "      <td>0.195737</td>\n",
       "      <td>0.081343</td>\n",
       "      <td>0.028040</td>\n",
       "      <td>0.041659</td>\n",
       "      <td>0.051252</td>\n",
       "      <td>0.152884</td>\n",
       "      <td>0.150140</td>\n",
       "      <td>0.184262</td>\n",
       "      <td>0.235245</td>\n",
       "      <td>2.513531</td>\n",
       "      <td>1.856405</td>\n",
       "      <td>1.865527</td>\n",
       "      <td>2.099645</td>\n",
       "      <td>2.099645</td>\n",
       "      <td>2.254329</td>\n",
       "      <td>-0.144996</td>\n",
       "      <td>-0.144996</td>\n",
       "      <td>2.461843</td>\n",
       "      <td>-0.349089</td>\n",
       "      <td>-4.475081</td>\n",
       "      <td>-3.909570</td>\n",
       "      <td>-3.798061</td>\n",
       "      <td>-1.608958</td>\n",
       "      <td>-1.608958</td>\n",
       "      <td>-1.636207</td>\n",
       "      <td>-5.039754</td>\n",
       "      <td>-5.039754</td>\n",
       "      <td>-4.089410</td>\n",
       "      <td>-3.789677</td>\n",
       "      <td>3.426426</td>\n",
       "      <td>2.678273</td>\n",
       "      <td>2.653962</td>\n",
       "      <td>2.709924</td>\n",
       "      <td>2.709924</td>\n",
       "      <td>2.918636</td>\n",
       "      <td>-0.033150</td>\n",
       "      <td>-0.033150</td>\n",
       "      <td>3.378262</td>\n",
       "      <td>-0.350576</td>\n",
       "      <td>-6.415826</td>\n",
       "      <td>-5.350381</td>\n",
       "      <td>-5.098012</td>\n",
       "      <td>-2.039563</td>\n",
       "      <td>-2.039563</td>\n",
       "      <td>-2.081795</td>\n",
       "      <td>-7.279274</td>\n",
       "      <td>-7.279274</td>\n",
       "      <td>-5.847995</td>\n",
       "      <td>-5.323414</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.286627</td>\n",
       "      <td>0.207469</td>\n",
       "      <td>0.107185</td>\n",
       "      <td>0.064430</td>\n",
       "      <td>0.052353</td>\n",
       "      <td>0.086763</td>\n",
       "      <td>0.199029</td>\n",
       "      <td>0.151965</td>\n",
       "      <td>0.389327</td>\n",
       "      <td>0.271003</td>\n",
       "      <td>2.730166</td>\n",
       "      <td>2.283594</td>\n",
       "      <td>1.522712</td>\n",
       "      <td>2.638951</td>\n",
       "      <td>2.638951</td>\n",
       "      <td>2.653013</td>\n",
       "      <td>0.056817</td>\n",
       "      <td>0.056817</td>\n",
       "      <td>2.637051</td>\n",
       "      <td>0.769431</td>\n",
       "      <td>-4.752177</td>\n",
       "      <td>-4.232779</td>\n",
       "      <td>-3.935561</td>\n",
       "      <td>-1.645848</td>\n",
       "      <td>-1.645848</td>\n",
       "      <td>-1.640399</td>\n",
       "      <td>-5.192765</td>\n",
       "      <td>-5.192765</td>\n",
       "      <td>-4.283922</td>\n",
       "      <td>-4.633122</td>\n",
       "      <td>3.915867</td>\n",
       "      <td>3.169549</td>\n",
       "      <td>2.140667</td>\n",
       "      <td>3.317253</td>\n",
       "      <td>3.317253</td>\n",
       "      <td>3.320464</td>\n",
       "      <td>0.207672</td>\n",
       "      <td>0.207672</td>\n",
       "      <td>3.845685</td>\n",
       "      <td>1.020502</td>\n",
       "      <td>-6.847804</td>\n",
       "      <td>-5.587994</td>\n",
       "      <td>-5.242004</td>\n",
       "      <td>-2.112832</td>\n",
       "      <td>-2.112832</td>\n",
       "      <td>-2.149975</td>\n",
       "      <td>-7.343893</td>\n",
       "      <td>-7.343893</td>\n",
       "      <td>-6.058642</td>\n",
       "      <td>-6.176686</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.287943</td>\n",
       "      <td>0.395594</td>\n",
       "      <td>0.157022</td>\n",
       "      <td>0.055293</td>\n",
       "      <td>0.052712</td>\n",
       "      <td>0.103719</td>\n",
       "      <td>0.281050</td>\n",
       "      <td>0.217666</td>\n",
       "      <td>0.503941</td>\n",
       "      <td>0.272300</td>\n",
       "      <td>2.461083</td>\n",
       "      <td>3.095404</td>\n",
       "      <td>1.449740</td>\n",
       "      <td>2.958202</td>\n",
       "      <td>2.958202</td>\n",
       "      <td>2.892832</td>\n",
       "      <td>0.359725</td>\n",
       "      <td>0.359725</td>\n",
       "      <td>2.187819</td>\n",
       "      <td>1.873888</td>\n",
       "      <td>-4.583656</td>\n",
       "      <td>-4.070965</td>\n",
       "      <td>-4.013953</td>\n",
       "      <td>-1.850422</td>\n",
       "      <td>-1.850422</td>\n",
       "      <td>-1.892762</td>\n",
       "      <td>-5.167193</td>\n",
       "      <td>-5.167193</td>\n",
       "      <td>-4.210980</td>\n",
       "      <td>-5.047719</td>\n",
       "      <td>3.541562</td>\n",
       "      <td>4.191092</td>\n",
       "      <td>2.054430</td>\n",
       "      <td>3.704862</td>\n",
       "      <td>3.704862</td>\n",
       "      <td>3.611744</td>\n",
       "      <td>0.637481</td>\n",
       "      <td>0.637481</td>\n",
       "      <td>3.203953</td>\n",
       "      <td>2.476900</td>\n",
       "      <td>-6.545063</td>\n",
       "      <td>-5.364627</td>\n",
       "      <td>-5.389050</td>\n",
       "      <td>-2.391150</td>\n",
       "      <td>-2.391150</td>\n",
       "      <td>-2.497491</td>\n",
       "      <td>-7.275712</td>\n",
       "      <td>-7.275712</td>\n",
       "      <td>-5.923299</td>\n",
       "      <td>-6.710426</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.352317</td>\n",
       "      <td>0.509844</td>\n",
       "      <td>0.188443</td>\n",
       "      <td>0.143825</td>\n",
       "      <td>0.089402</td>\n",
       "      <td>0.111278</td>\n",
       "      <td>0.307617</td>\n",
       "      <td>0.312690</td>\n",
       "      <td>0.389198</td>\n",
       "      <td>0.331207</td>\n",
       "      <td>2.146012</td>\n",
       "      <td>3.417696</td>\n",
       "      <td>1.553117</td>\n",
       "      <td>3.497509</td>\n",
       "      <td>3.497509</td>\n",
       "      <td>3.488768</td>\n",
       "      <td>0.528852</td>\n",
       "      <td>0.528852</td>\n",
       "      <td>2.050237</td>\n",
       "      <td>2.396472</td>\n",
       "      <td>-4.407169</td>\n",
       "      <td>-3.920889</td>\n",
       "      <td>-4.087314</td>\n",
       "      <td>-1.891085</td>\n",
       "      <td>-1.891085</td>\n",
       "      <td>-1.964027</td>\n",
       "      <td>-5.059037</td>\n",
       "      <td>-5.059037</td>\n",
       "      <td>-4.223137</td>\n",
       "      <td>-5.095089</td>\n",
       "      <td>3.158082</td>\n",
       "      <td>4.636039</td>\n",
       "      <td>2.179199</td>\n",
       "      <td>4.430997</td>\n",
       "      <td>4.430997</td>\n",
       "      <td>4.420446</td>\n",
       "      <td>0.868211</td>\n",
       "      <td>0.868211</td>\n",
       "      <td>3.020928</td>\n",
       "      <td>3.220925</td>\n",
       "      <td>-6.276412</td>\n",
       "      <td>-5.191124</td>\n",
       "      <td>-5.500988</td>\n",
       "      <td>-2.484262</td>\n",
       "      <td>-2.484262</td>\n",
       "      <td>-2.632834</td>\n",
       "      <td>-7.140369</td>\n",
       "      <td>-7.140369</td>\n",
       "      <td>-5.934493</td>\n",
       "      <td>-6.825925</td>\n",
       "      <td>2013</td>\n",
       "      <td>213</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Solution data frame saved to data/Solution_pp_2018_10_21_00_41_10.feather\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Preprocess the solution file. This is the testing data set with TARGETVAR included.\n",
    "Preprocess it in the same way as the training data set, then output it in pickle and feather formats.\n",
    "'''\n",
    "# Load and preprocess data\n",
    "solution_csv = \"data/Solution.csv\"\n",
    "solution_df  = preProcess(solution_csv, Nroll=1, inclusive=False, keepID=False)\n",
    "print('\\nDataframe solution_df for pickle:\\n')\n",
    "display_html(solution_df.head())\n",
    "\n",
    "# Output dataframe in pickle format\n",
    "fname          = strftime(\"%Y_%m_%d_%H_%M_%S\")\n",
    "solution_fname = \"data/Solution_pp_\" + fname + \".pkl\"\n",
    "try:\n",
    "    solution_df.to_pickle(solution_fname)\n",
    "    print('\\nSolution data frame saved to {0}'.format(solution_fname))\n",
    "except:\n",
    "    print('\\nError saving solution data frame to {0}'.format(solution_fname))\n",
    "    \n",
    "\n",
    "# Rename dataframe columns for easier handling in R\n",
    "fields  = [item[0]+'_'+str(item[1]) for item in solution_df.columns.values]\n",
    "fields  = [field if field[-1]!='_' else field[:-1] for field in fields]\n",
    "solution_df.columns = fields\n",
    "print('\\nDataframe solution_df for feather:\\n')\n",
    "display_html(solution_df.head())\n",
    "    \n",
    "# Output data frame in feather format\n",
    "solution_fname = \"data/Solution_pp_\" + fname + \".feather\"\n",
    "try:\n",
    "    feather.write_dataframe(solution_df, solution_fname)\n",
    "    print('\\nSolution data frame saved to {0}'.format(solution_fname))\n",
    "except:\n",
    "    print('\\nError saving solution data frame to {0}'.format(solution_fname))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}