Using DL beta.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><ul class=\"toc-item\"><li><span><a href=\"#Load-Dfs\" data-toc-modified-id=\"Load-Dfs-0.1\"><span class=\"toc-item-num\">0.1&nbsp;&nbsp;</span>Load Dfs</a></span></li></ul></li><li><span><a href=\"#Preprocessing\" data-toc-modified-id=\"Preprocessing-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Preprocessing</a></span><ul class=\"toc-item\"><li><span><a href=\"#Convert-date-cols\" data-toc-modified-id=\"Convert-date-cols-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Convert date cols</a></span></li><li><span><a href=\"#Remove-some-low-importance-cols\" data-toc-modified-id=\"Remove-some-low-importance-cols-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Remove some low importance cols</a></span></li><li><span><a href=\"#Pick-highly-important-features\" data-toc-modified-id=\"Pick-highly-important-features-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Pick highly important features</a></span></li><li><span><a href=\"#Get-validation-idx\" data-toc-modified-id=\"Get-validation-idx-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Get validation idx</a></span></li><li><span><a href=\"#Get-categorical-&amp;-continous-fields\" data-toc-modified-id=\"Get-categorical-&amp;-continous-fields-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>Get categorical &amp; continous fields</a></span></li><li><span><a href=\"#Intiate-model--&amp;-train\" data-toc-modified-id=\"Intiate-model--&amp;-train-1.6\"><span class=\"toc-item-num\">1.6&nbsp;&nbsp;</span>Intiate model  &amp; train</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai1/lib/python3.6/site-packages/dask/config.py:168: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  data = yaml.load(f.read()) or {}\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from fastai.imports import *\n",
    "from fastai.torch_imports import *\n",
    "from fastai.dataset import *\n",
    "from fastai.learner import *\n",
    "from fastai.structured import *\n",
    "from fastai.column_data import *\n",
    "import feather\n",
    "import gc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'data/elo/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep = 'target'\n",
    "df_raw = feather.read_dataframe('train_df_alpha')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = feather.read_dataframe('test_df_alpha')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df3 = feather.read_dataframe('train_df_alpha')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert date cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [df_raw, df_test]:\n",
    "    for f in ['purchase_date_max','purchase_date_min','purchase_date_max_old',\\\n",
    "                     'purchase_date_min_old', 'observation_date_old']:\n",
    "        df[f] = df[f].astype(np.int64) * 1e-9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove some low importance cols\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_excluded = ['purchase_date_max', 'purchase_date_max_old', 'card_id', 'first_active_month',\n",
    "                 'target','outliers','card_id_size', 'card_id_size_old', \n",
    "                 'purchase_date_min', 'purchase_date_min_old','first_active_monthYear',\n",
    "                 'first_active_monthMonth',\n",
    "                 'first_active_monthWeek',\n",
    "                 'first_active_monthDay',\n",
    "                 'first_active_monthDayofweek',\n",
    "                 'first_active_monthDayofyear',\n",
    "                 'first_active_monthIs_month_end',\n",
    "                 'first_active_monthIs_month_start',\n",
    "                 'first_active_monthIs_quarter_end',\n",
    "                 'first_active_monthIs_quarter_start',\n",
    "                 'first_active_monthIs_year_end',\n",
    "                 'Black_Friday_2017_mean',\n",
    "                 'amount_month_ratio_max',\n",
    "                 'purchase_Month_mean_old',\n",
    "                 'purchase_amount_total_max',\n",
    "                 'first_active_monthIs_year_start']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pick highly important features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_included = ['feature_1','feature_2','feature_3','transactions_count','subsector_id_nunique','merchant_id_nunique','merchant_category_id_nunique','purchase_Month_mean','purchase_Month_min','purchase_Month_max','purchase_Week_nunique','purchase_Week_mean','purchase_Week_min','purchase_Week_max','purchase_Dayofweek_mean','purchase_Dayofweek_min','purchase_Dayofweek_max','purchase_Day_nunique','purchase_Day_mean','purchase_Day_min','purchase_Day_max','purchase_Hour_nunique','purchase_Hour_mean','purchase_Hour_min','purchase_Hour_max','purchase_amount_sum','purchase_amount_max','purchase_amount_min','purchase_amount_mean','purchase_amount_var','purchase_amount_skew','installments_sum','installments_max','installments_mean','installments_var','installments_skew','month_lag_max','month_lag_min','month_lag_mean','month_lag_var','month_lag_skew','month_diff_mean','month_diff_var','month_diff_skew','purchased_on_weekend_mean','category_1_mean','category_2_mean','category_3_mean','card_id_count','price_mean','price_max','price_min','price_var','Christmas_Day_2017_mean','Children_day_2017_mean','Black_Friday_2017_mean','Mothers_Day_2018_mean','duration_mean','duration_min','duration_max','duration_var','duration_skew','amount_month_ratio_mean','amount_month_ratio_min','amount_month_ratio_max','amount_month_ratio_var','amount_month_ratio_skew','category_2_mean_mean','category_3_mean_mean','purchase_date_diff','purchase_date_average','purchase_date_uptonow','purchase_date_uptomin','transactions_count_old','subsector_id_nunique_old','merchant_id_nunique_old','merchant_category_id_nunique_old','purchase_Month_nunique','purchase_Month_mean_old','purchase_Month_min_old','purchase_Month_max_old','purchase_Week_nunique_old','purchase_Week_mean_old','purchase_Week_min_old','purchase_Week_max_old','purchase_Dayofweek_mean_old','purchase_Day_nunique_old','purchase_Day_mean_old','purchase_Day_min_old','purchase_Hour_nunique_old','purchase_Hour_mean_old','purchase_Hour_min_old','purchase_Hour_max_old','purchase_amount_sum_old','purchase_amount_max_old','purchase_amount_min_old','purchase_amount_mean_old','purchase_amount_var_old','purchase_amount_skew_old','installments_sum_old','installments_max_old','installments_mean_old','installments_var_old','installments_skew_old','month_lag_max_old','month_lag_min_old','month_lag_mean_old','month_lag_var_old','month_lag_skew_old','month_diff_max','month_diff_min','month_diff_mean_old','month_diff_var_old','month_diff_skew_old','authorized_flag_mean','purchased_on_weekend_mean_old','category_1_mean_old','category_2_mean_old','category_3_mean_old','card_id_count_old','price_sum','price_mean_old','price_max_old','price_min_old','price_var_old','Christmas_Day_2017_mean_old','Mothers_Day_2017_mean','fathers_day_2017_mean','Children_day_2017_mean_old','Valentine_Day_2017_mean','Black_Friday_2017_mean_old','Mothers_Day_2018_mean_old','duration_mean_old','duration_min_old','duration_max_old','duration_var_old','duration_skew_old','amount_month_ratio_mean_old','amount_month_ratio_min_old','amount_month_ratio_max_old','amount_month_ratio_var_old','amount_month_ratio_skew_old','category_2_mean_mean_old','category_3_mean_mean_old','purchase_date_diff_old','purchase_date_average_old','purchase_date_uptonow_old','purchase_date_uptomin_old','quarter','observed_elapsed_time','days_feature1','days_feature2','days_feature3','days_feature1_ratio','days_feature2_ratio','days_feature3_ratio','feature_sum','feature_mean','feature_max','feature_min','feature_var','card_id_total','card_id_count_total','card_id_count_ratio','purchase_amount_total','purchase_amount_total_mean','purchase_amount_total_max','purchase_amount_total_min','purchase_amount_sum_ratio','hist_first_buy','new_first_buy','hist_last_buy','new_last_buy','month_diff_ratio','installments_total','installments_ratio','price_total','CLV','CLV_old','CLV_ratio']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_columns = [c for c in cols_included if c not in cols_excluded]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_cols = ['merchant_address_id_nunique', 'merchant_rating_nunique']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_columns = df_train_columns + exp_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "178"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_train_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw_copy = df_raw.copy()\n",
    "df_test_copy = df_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw = df_raw[df_train_columns]\n",
    "df_test = df_test[df_train_columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get validation idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_valid = 12000\n",
    "n_trn = len(df_raw)-n_valid\n",
    "val_idx = list(range(n_trn, len(df_raw)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get categorical & continous fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'feature_1 feature_2 feature_3 transactions_count subsector_id_nunique merchant_id_nunique merchant_category_id_nunique purchase_Month_min purchase_Month_max purchase_Week_nunique purchase_Week_min purchase_Week_max purchase_Dayofweek_min purchase_Dayofweek_max purchase_Day_nunique purchase_Day_min purchase_Day_max purchase_Hour_nunique purchase_Hour_min purchase_Hour_max installments_max month_lag_max month_lag_min card_id_count purchase_date_diff purchase_date_uptonow purchase_date_uptomin subsector_id_nunique_old merchant_category_id_nunique_old purchase_Month_nunique purchase_Month_min_old purchase_Month_max_old purchase_Week_nunique_old purchase_Week_min_old purchase_Week_max_old purchase_Day_nunique_old purchase_Day_min_old purchase_Hour_nunique_old purchase_Hour_min_old purchase_Hour_max_old installments_max_old month_lag_max_old month_lag_min_old month_diff_max month_diff_min quarter feature_sum feature_mean feature_max feature_min feature_var merchant_address_id_nunique merchant_rating_nunique'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_flds = [n for n in df_raw.columns.values if (df_raw[n].nunique()<100) & (n != 'outliers')]\n",
    "' '.join(cat_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[n for n in df_raw.drop(cat_flds,axis=1).columns if not is_numeric_dtype(df_raw[n])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'purchase_Month_mean purchase_Week_mean purchase_Dayofweek_mean purchase_Day_mean purchase_Hour_mean purchase_amount_sum purchase_amount_max purchase_amount_min purchase_amount_mean purchase_amount_var purchase_amount_skew installments_sum installments_mean installments_var installments_skew month_lag_mean month_lag_var month_lag_skew month_diff_mean month_diff_var month_diff_skew purchased_on_weekend_mean category_1_mean category_2_mean category_3_mean price_mean price_max price_min price_var Christmas_Day_2017_mean Children_day_2017_mean Mothers_Day_2018_mean duration_mean duration_min duration_max duration_var duration_skew amount_month_ratio_mean amount_month_ratio_min amount_month_ratio_var amount_month_ratio_skew category_2_mean_mean category_3_mean_mean purchase_date_average transactions_count_old merchant_id_nunique_old purchase_Week_mean_old purchase_Dayofweek_mean_old purchase_Day_mean_old purchase_Hour_mean_old purchase_amount_sum_old purchase_amount_max_old purchase_amount_min_old purchase_amount_mean_old purchase_amount_var_old purchase_amount_skew_old installments_sum_old installments_mean_old installments_var_old installments_skew_old month_lag_mean_old month_lag_var_old month_lag_skew_old month_diff_mean_old month_diff_var_old month_diff_skew_old authorized_flag_mean purchased_on_weekend_mean_old category_1_mean_old category_2_mean_old category_3_mean_old card_id_count_old price_sum price_mean_old price_max_old price_min_old price_var_old Christmas_Day_2017_mean_old Mothers_Day_2017_mean fathers_day_2017_mean Children_day_2017_mean_old Valentine_Day_2017_mean Black_Friday_2017_mean_old Mothers_Day_2018_mean_old duration_mean_old duration_min_old duration_max_old duration_var_old duration_skew_old amount_month_ratio_mean_old amount_month_ratio_min_old amount_month_ratio_max_old amount_month_ratio_var_old amount_month_ratio_skew_old category_2_mean_mean_old category_3_mean_mean_old purchase_date_diff_old purchase_date_average_old purchase_date_uptonow_old purchase_date_uptomin_old observed_elapsed_time days_feature1 days_feature2 days_feature3 days_feature1_ratio days_feature2_ratio days_feature3_ratio card_id_total card_id_count_total card_id_count_ratio purchase_amount_total purchase_amount_total_mean purchase_amount_total_min purchase_amount_sum_ratio hist_first_buy new_first_buy hist_last_buy new_last_buy month_diff_ratio installments_total installments_ratio price_total CLV CLV_old CLV_ratio'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for n in cat_flds: df_raw[n] = df_raw[n].astype('category').cat.as_ordered()\n",
    "\n",
    "cont_flds = [n for n in df_raw.columns if n not in cat_flds and n!= 'outliers']\n",
    "' '.join(cont_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw.replace(np.inf, 0, inplace=True)\n",
    "df_raw.replace(-np.inf, 0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai1/lib/python3.6/site-packages/pandas/core/frame.py:4042: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  method=method)\n"
     ]
    }
   ],
   "source": [
    "df_test.replace(np.inf, 0, inplace=True)\n",
    "df_test.replace(-np.inf, 0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 178), (123623, 178))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c for c in df_raw.columns.values if c not in df_test.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in cat_flds: df_raw[n] = df_raw[n].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai1/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.32 s, sys: 4.5 s, total: 10.8 s\n",
      "Wall time: 10.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for n in cont_flds: df_raw[n] = df_raw[n].fillna(df_raw[n].median()).astype('float32')\n",
    "for n in cont_flds: df_test[n] = df_test[n].fillna(df_test[n].median()).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_raw[cat_flds].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "??apply_cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/private/elo/fastai/structured.py:175: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df[n] = c.astype('category').cat.as_ordered()\n"
     ]
    }
   ],
   "source": [
    "apply_cats(df_test, df_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 178), (123623, 178))"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all((df_raw.dtypes.sort_index() == df_test.dtypes.sort_index()).sort_values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['feature_1',\n",
       " 'feature_2',\n",
       " 'feature_3',\n",
       " 'transactions_count',\n",
       " 'subsector_id_nunique',\n",
       " 'merchant_id_nunique',\n",
       " 'merchant_category_id_nunique',\n",
       " 'purchase_Month_min',\n",
       " 'purchase_Month_max',\n",
       " 'purchase_Week_nunique',\n",
       " 'purchase_Week_min',\n",
       " 'purchase_Week_max',\n",
       " 'purchase_Dayofweek_min',\n",
       " 'purchase_Dayofweek_max',\n",
       " 'purchase_Day_nunique',\n",
       " 'purchase_Day_min',\n",
       " 'purchase_Day_max',\n",
       " 'purchase_Hour_nunique',\n",
       " 'purchase_Hour_min',\n",
       " 'purchase_Hour_max',\n",
       " 'installments_max',\n",
       " 'month_lag_max',\n",
       " 'month_lag_min',\n",
       " 'card_id_count',\n",
       " 'purchase_date_diff',\n",
       " 'purchase_date_uptonow',\n",
       " 'purchase_date_uptomin',\n",
       " 'subsector_id_nunique_old',\n",
       " 'merchant_category_id_nunique_old',\n",
       " 'purchase_Month_nunique',\n",
       " 'purchase_Month_min_old',\n",
       " 'purchase_Month_max_old',\n",
       " 'purchase_Week_nunique_old',\n",
       " 'purchase_Week_min_old',\n",
       " 'purchase_Week_max_old',\n",
       " 'purchase_Day_nunique_old',\n",
       " 'purchase_Day_min_old',\n",
       " 'purchase_Hour_nunique_old',\n",
       " 'purchase_Hour_min_old',\n",
       " 'purchase_Hour_max_old',\n",
       " 'installments_max_old',\n",
       " 'month_lag_max_old',\n",
       " 'month_lag_min_old',\n",
       " 'month_diff_max',\n",
       " 'month_diff_min',\n",
       " 'quarter',\n",
       " 'feature_sum',\n",
       " 'feature_mean',\n",
       " 'feature_max',\n",
       " 'feature_min',\n",
       " 'feature_var',\n",
       " 'merchant_address_id_nunique',\n",
       " 'merchant_rating_nunique']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_flds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c for c in df_test.columns.values if c not in df_raw.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5,  3,  2, 87, 24, 86, 39, 12, 12, 10, 52, 52,  7,  7, 30, 31, 31, 23, 24, 24, 13,  2,  2, 87, 62,\n",
       "       63, 63, 34, 92, 11, 11, 11, 52, 48, 48, 31, 30, 24, 23, 22, 13, 12, 13, 14, 14,  4,  7,  7,  5,  2,\n",
       "        9, 86, 14])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw[cat_flds].nunique().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5,  3,  2, 74, 23, 73, 37, 12, 12, 10, 52, 52,  7,  7, 30, 31, 31, 22, 24, 24, 13,  2,  2, 74, 62,\n",
       "       63, 63, 34, 86, 11, 11, 11, 52, 48, 48, 31, 30, 24, 23, 20, 13, 12, 13, 14, 14,  4,  7,  7,  5,  2,\n",
       "        9, 73, 14])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test[cat_flds].nunique().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df_raw.dtypes.sort_index() == df_test.dtypes.sort_index()).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(53, 125)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds), len(cont_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai1/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "df_raw['target'] = df_raw_copy['target']\n",
    "df_test['target'] = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "??proc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "??scale_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/numpy/lib/nanfunctions.py:1431: RuntimeWarning: overflow encountered in multiply\n",
      "  sqr = np.multiply(arr, arr, out=arr)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.19 s, sys: 1.6 s, total: 10.8 s\n",
      "Wall time: 9.25 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_raw_copy = df_raw[cat_flds+cont_flds+[dep]].copy()\n",
    "df, y, nas, mapper_t = proc_df(df_raw_copy, 'target', do_scale=True)\n",
    "\n",
    "val_idx = list(range(n_trn, len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 178), (201917, 179))"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.49 s, sys: 944 ms, total: 6.43 s\n",
      "Wall time: 5.01 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_test_copy = df_test[cat_flds+cont_flds+[dep]].copy()\n",
    "df_t, _, nas, mapper = proc_df(df_test_copy,'target', do_scale=True, mapper=mapper_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123623, 179)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_copy.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 178), (123623, 178))"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c for c in df_t.columns.values if c not in df.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.dtypes == df_t.dtypes).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply_cats(df_t, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((123623, 178), (201917, 178))"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_t.shape, df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy2 = df.copy()\n",
    "df_t_copy2 = df_t.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx = list(range(n_trn, len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 178), (123623, 178), (201917,))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(x,y): return math.sqrt(((x-y)**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'feature_1': 6,\n",
       " 'feature_2': 4,\n",
       " 'feature_3': 3,\n",
       " 'transactions_count': 88,\n",
       " 'subsector_id_nunique': 25,\n",
       " 'merchant_id_nunique': 87,\n",
       " 'merchant_category_id_nunique': 40,\n",
       " 'purchase_Month_min': 13,\n",
       " 'purchase_Month_max': 13,\n",
       " 'purchase_Week_nunique': 11,\n",
       " 'purchase_Week_min': 53,\n",
       " 'purchase_Week_max': 53,\n",
       " 'purchase_Dayofweek_min': 8,\n",
       " 'purchase_Dayofweek_max': 8,\n",
       " 'purchase_Day_nunique': 31,\n",
       " 'purchase_Day_min': 32,\n",
       " 'purchase_Day_max': 32,\n",
       " 'purchase_Hour_nunique': 24,\n",
       " 'purchase_Hour_min': 25,\n",
       " 'purchase_Hour_max': 25,\n",
       " 'installments_max': 14,\n",
       " 'month_lag_max': 3,\n",
       " 'month_lag_min': 3,\n",
       " 'card_id_count': 88,\n",
       " 'purchase_date_diff': 63,\n",
       " 'purchase_date_uptonow': 64,\n",
       " 'purchase_date_uptomin': 64,\n",
       " 'subsector_id_nunique_old': 35,\n",
       " 'merchant_category_id_nunique_old': 93,\n",
       " 'purchase_Month_nunique': 12,\n",
       " 'purchase_Month_min_old': 12,\n",
       " 'purchase_Month_max_old': 12,\n",
       " 'purchase_Week_nunique_old': 53,\n",
       " 'purchase_Week_min_old': 49,\n",
       " 'purchase_Week_max_old': 49,\n",
       " 'purchase_Day_nunique_old': 32,\n",
       " 'purchase_Day_min_old': 31,\n",
       " 'purchase_Hour_nunique_old': 25,\n",
       " 'purchase_Hour_min_old': 24,\n",
       " 'purchase_Hour_max_old': 23,\n",
       " 'installments_max_old': 14,\n",
       " 'month_lag_max_old': 13,\n",
       " 'month_lag_min_old': 14,\n",
       " 'month_diff_max': 15,\n",
       " 'month_diff_min': 15,\n",
       " 'quarter': 5,\n",
       " 'feature_sum': 8,\n",
       " 'feature_mean': 8,\n",
       " 'feature_max': 6,\n",
       " 'feature_min': 3,\n",
       " 'feature_var': 10,\n",
       " 'merchant_address_id_nunique': 87,\n",
       " 'merchant_rating_nunique': 15}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_c = {n: len(c.cat.categories)+1 for n,c in df_raw[cat_flds].items()}\n",
    "emb_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_szs = [(c, min(10, (c+1)//2)) for _,c in emb_c.items()]\n",
    "metrics=[rmse]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_range=(np.min(y)*1,np.max(y)*1.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(6, 3),\n",
       " (4, 2),\n",
       " (3, 2),\n",
       " (88, 10),\n",
       " (25, 10),\n",
       " (87, 10),\n",
       " (40, 10),\n",
       " (13, 7),\n",
       " (13, 7),\n",
       " (11, 6),\n",
       " (53, 10),\n",
       " (53, 10),\n",
       " (8, 4),\n",
       " (8, 4),\n",
       " (31, 10),\n",
       " (32, 10),\n",
       " (32, 10),\n",
       " (24, 10),\n",
       " (25, 10),\n",
       " (25, 10),\n",
       " (14, 7),\n",
       " (3, 2),\n",
       " (3, 2),\n",
       " (88, 10),\n",
       " (63, 10),\n",
       " (64, 10),\n",
       " (64, 10),\n",
       " (35, 10),\n",
       " (93, 10),\n",
       " (12, 6),\n",
       " (12, 6),\n",
       " (12, 6),\n",
       " (53, 10),\n",
       " (49, 10),\n",
       " (49, 10),\n",
       " (32, 10),\n",
       " (31, 10),\n",
       " (25, 10),\n",
       " (24, 10),\n",
       " (23, 10),\n",
       " (14, 7),\n",
       " (13, 7),\n",
       " (14, 7),\n",
       " (15, 8),\n",
       " (15, 8),\n",
       " (5, 3),\n",
       " (8, 4),\n",
       " (8, 4),\n",
       " (6, 3),\n",
       " (3, 2),\n",
       " (10, 5),\n",
       " (87, 10),\n",
       " (15, 8)]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_szs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(53, 125, 178)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds), len(cont_flds), len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_feather('dl_df_beta')\n",
    "df_t.to_feather('dl_df_t_beta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_t = feather.read_dataframe('dl_df_t_beta')\n",
    "df = feather.read_dataframe('dl_df_beta')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Intiate model  & train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train_df3['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "201917"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx = list(range(41000, len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = ColumnarModelData.from_data_frame(PATH, val_idx, df, y.astype(np.float32), cat_flds=cat_flds, bs=128, test_df=df_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "??ColumnarDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(next(iter(md.trn_ds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1,l2,l3 = next(iter(md.trn_ds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(l1[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((53,), (125,), (1,))"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l1.shape, l2.shape, l3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.2514 , -0.24076,  0.05885,  0.14364, -0.21976,  1.08686, -0.08475, -0.13903, -0.07369, -0.03511,\n",
       "       -0.51744, -0.57234, -0.7556 , -0.22209, -0.41728, -0.00174,  0.34604,  0.02931, -0.50046,  0.76717,\n",
       "       -0.11306, -0.10089, -0.28106, -0.7342 , -1.03658, -0.26878, -0.24468, -0.12299, -0.0224 , -0.24394,\n",
       "       -0.19696,  0.17509, -0.1039 , -0.14891, -0.10517, -0.03782, -0.52032, -0.04773, -0.12888, -0.03191,\n",
       "       -0.48677,  0.33716, -0.39381, -0.59586,  1.6319 ,  1.96457,  0.82615,  0.36434, -0.14179, -0.43228,\n",
       "       -0.00393, -0.00479, -0.1023 , -0.00278, -0.00223,  3.11806, -0.55079, -0.83411, -0.43968,  1.29591,\n",
       "       -0.03322, -0.23354,  0.24687, -0.49285,  0.46469,  0.32697,  0.49497,  0.50097, -0.48584, -0.77694,\n",
       "       -1.02985,  1.6319 , -0.02032, -0.00795, -0.01102, -0.00586, -0.00297, -0.12421, -0.70753, -0.20729,\n",
       "        1.54226, -0.72712,  0.36126, -0.02406, -0.00261, -0.10985, -0.00398, -0.     ,  3.14493, -0.00304,\n",
       "       -0.09518, -0.00603, -0.00224,  3.08426,  0.33137,  1.02164, -0.01419, -0.74195, -0.35653, -0.04318,\n",
       "       -0.31151,  0.22487, -0.1492 ,  0.17118,  0.53288,  0.01429,  0.51454,  1.79217,  1.79217, -0.23035,\n",
       "       -0.00337, -0.00268, -0.15363, -0.04877, -0.4138 , -0.35166, -0.32929, -0.25987, -0.3806 , -0.55941,\n",
       "       -0.24171, -0.00165,  1.33   , -0.00534, -0.04329], dtype=float32)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.12 s, sys: 448 ms, total: 1.56 s\n",
      "Wall time: 2.26 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_flds),\n",
    "                   0.1, 1, [1000, 500, 50], [0.5, 0.5, 0.01], y_range=y_range, use_bn=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "??md.get_learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[125, 1000, 500, 50]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(df.columns)-len(cat_flds)]+[1000, 500, 50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c6235d8edd44c918b7b2957c98ac78a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 951/1262 [00:06<00:02, 141.39it/s, loss=18.7]\n",
      "                                                              \r"
     ]
    }
   ],
   "source": [
    "m.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VfX9+PHX+2aTCWRACJCwZMkSBBQHuPeso2qdVTvttFrr1/Gz1bauqm0V96jVVq22blFAQFZAliA77JGElQQy7/v3xzkJSci4gXtz7819Px+P+8g9555z7juHcN/3s0VVMcYYE7k8wQ7AGGNMcFkiMMaYCGeJwBhjIpwlAmOMiXCWCIwxJsJZIjDGmAhnicAYYyKcJQJjjIlwlgiMMSbCWSIwxpgIFx3sAHyRnp6uubm5wQ7DGGPCyoIFC4pUNaO148IiEeTm5pKfnx/sMIwxJqyIyAZfjrOqIWOMiXCWCIwxJsJZIjDGmAhnicAYYyKcJQJjjIlwlgiMMSbCdehEoKos27IXW47TGGOa16ETwSffbOfcJ2fy7qItwQ7FGGNCVodOBCu2lQAwY3VRkCMxxpjQ1aETQUW1F4Bte8qDHIkxxoSusJhi4nDdcdZAiksrmLqyMNihGGNMyOrQJQKA3PREikorePmrgmCHYowxIanDJ4JxfboC8PiUVUGOxBhjQlOHTwTH9O7MFWN6cqCqhorqmmCHY4wxIafDJwKA0wZnUV7l5ctV1nvIGGMaC3giEJEoEflaRN53t18SkfUissh9jAh0DCf0d9Zl+P4r+RSXVgT67YwxJqy0R4ngNmBFo32/VtUR7mNRoAOIjfZw3vBsAAqK9wf67YwxJqwENBGISA5wDvBcIN/HF7ec2AfASgTGGNNIoEsEjwO3A95G+38vIktE5DERiQtwDAB0TYoFoKi0sj3ezhhjwkbAEoGInAvsVNUFjV66ExgIjAG6AL9p5vybRSRfRPILC498QFjXRCffWInAGGMaCmSJ4HjgfBEpAN4AJonIa6q6TR0VwIvAsU2drKqTVXW0qo7OyMg44mBioz2kxEdTZInAGGMaCFgiUNU7VTVHVXOBK4AvVPVqEekOICICXAgsC1QMjaUnx1nVkDHGNBKMcQT/EJGlwFIgHXigvd64c6dYPli6jSWb97TXWxpjTMhrl0SgqtNU9Vz3+SRVPVpVh6rq1apa2h4xAFS6s5He+mrjZgtjjIlcETGyuFZ0lAQ7BGOMCTkRlQhq00C115auNMaYWhGVCPpkJAFQWdN4WIMxxkSuiEoE918whKS4aOKjo4IdijHGhIyISgSdYqP50cR+bN9Xzr7yqmCHY4wxISGiEgFA/0ynemj1jnbrrGSMMSEt4hLBgKxkAFbvKAlyJMYYExoiLhHkdE4gPsbDKisRGGMMEIGJwOMR+mUmsXqnlQiMMQYiMBEADMhMtjYCY4xxRWQi6J+VzPZ95ew9YD2HjDEmMhOB23NozU4rFRhjTEQmgtqeQ/kFu4IciTHGBF9EJoIenRMAePCjb9lZUh7kaIwxJrgiMhFEeQ7OQnrs7z8PYiTGGBN8EZkIAN790fHBDsEYY0JCxCaCET3TuHZ8bwDKKqqDHI0xxgRPxCYCgAn9MwBYadNNGGMiWEQngn5uN9J1hWVBjsQYY4InohNB99R4ALbtORDkSIwxJngCnghEJEpEvhaR993tPBGZKyKrReRNEYkNdAzNiY+JIj0plq17LREYYyJXe5QIbgNW1Nv+I/CYqvYHdgM3tkMMzcpOS2DLHhtLYIyJXAFNBCKSA5wDPOduCzAJeMs95GXgwkDG0Jrs1AS2WtWQMSaCBbpE8DhwO1C7WnxXYI+q1vbX3Az0aOpEEblZRPJFJL+wsDBgAWanJbBmZynfbN0bsPcwxphQFrBEICLnAjtVdUH93U0cqk2dr6qTVXW0qo7OyMgISIwA2WlOg/E5T8wM2HsYY0woiw7gtY8HzheRs4F4IAWnhJAmItFuqSAH2BrAGFoVF30wF6oqTu2VMcZEjoCVCFT1TlXNUdVc4ArgC1W9CpgKXOoedi3wXqBi8MWwnLS650WllUGMxBhjgiMY4wh+A/xCRNbgtBk8H4QY6gzvmcbfrxoFYMtXGmMiUiCrhuqo6jRgmvt8HXBse7yvr0b26gzA2p2lHNc3PcjRGGNM+4rokcW1slLiSIqLZq1NNWGMiUCWCAARITM5jsLSimCHYowx7c4SgatrUiy7rLHYGBOBLBG4uiTGsmn3/mCHYYwx7c4SgSsrJZ7Nuw+wfOu+YIdijDHtyhKB68YJeQC8tWAzSzfbdBPGmMhhicDVu2siGclxvDBrPec9NZNtNjW1MSZCWCKoZ2C35Lrn4x/8IoiRGGNM+7FEUE9KfEyD7cIS605qjOn4LBHUc8GI7Abb1ovIGBMJLBHUc/qQbiy77wz+5s49tGOvrVxmjOn4LBE0khQXzbg+XQHYYiuXGWMigCWCJnTuFENWShzvL9lGVY239ROMMSaMWSJogojQLSWeRZv28Nepa4IdjjHGBJQlgmZMGpgFwBIbXGZMSPtX/iamLN8R7DDCmiWCZvxoYl9EICEmKtihGGOasaG4jNvfWsJNr+Szsdh6+R0uSwTNiI7yMKFfujUYGxPCyqsOtuHlb9gVxEjCmyWCFmSnJrBo0x6qrcHYmJBUvzPHThsAethaTQQikigiHvf5ABE5X0RiWjuvI/C4d+fl2RuCG4gxpkn1E8FzM9YHMZLw5kuJ4EsgXkR6AJ8D1wMvtXaSiMSLyDwRWSwi34jIfe7+l0RkvYgsch8jjuQXCKSrx/UGYOHG3UGOxBjTlGqv1j0vKq2gstpK74fDl0QgqrofuBh4UlUvAgb7cF4FMElVhwMjgDNFZJz72q9VdYT7WHRYkbeDIdmpnDook5XbS4IdijGmCbUlgjOHdANg1pqiYIYTtnxKBCIyHrgK+MDdF93aSeoodTdj3Ie2cEpIGp6TxtrCUvaVVwU7FGNMI9U1zkfKgKwkAK5/aX4wwwlbviSCnwF3Av9R1W9EpA8w1ZeLi0iUiCwCdgKfqepc96Xfi8gSEXlMROIOK/J2MrxnGqqwzMYTGBNyaksEE/pnADCoe0owwwlbrSYCVZ2uquer6h/dRuMiVf2pLxdX1RpVHQHkAMeKyFCcpDIQGAN0AX7T1LkicrOI5ItIfmFhoa+/j98Ny0kFYNHmPUGLwRjTtCq3RJAYF8Vpg7PwesOu0iEk+NJr6HURSRGRRGA5sFJEft2WN1HVPcA04ExV3eZWG1UALwLHNnPOZFUdraqjMzIy2vJ2fpXWKZa89EQWbbREYEyoqfY6JYKYKA8ZyXEUFJdRYtW4beZL1dBgVd0HXAh8CPQCrmntJBHJEJE093kCcCrwrYh0d/eJe81lhxl7uzm6RypLt1jVkDGhpraNINojnDIwk4pqL7PXFgc5qvDjSyKIcccNXAi8p6pV+Nbo2x2YKiJLgPk4bQTvA/8QkaXAUiAdeODwQm8/Y3I7s21vOcu37gt2KMaYeiprDpYIhuWkAbBxl0010Va+JIJngAIgEfhSRHoDrX4iquoSVR2pqsNUdaiq3u/un6SqR7v7rq7XsyhkneF2TZu9zr5pGBNKaksEMVEeuibGAvDABytYZiX4NvGlsfgJVe2hqme7dfsbgIntEFvIyEyJJz0pjhXbrERgTCipbSOIjhI8HqFHWgIAHyzdFsywwo4vjcWpIvJobQ8eEXkEp3QQUQZ1T7ZEYEyIqR1JHOPOB/PJz08kMTaK1TtsEGhb+FI19AJQAlzmPvbh9PaJKIO6p7B6R6l1TzMmhNROMRETLYCz1OzEgZnMXbfLVhdsA18SQV9VvUdV17mP+4A+gQ4s1OR0TqCyxktRmc1waEyoOFBZA0Bc9MF1Q3I6d6Kkopq7/rM0WGGFHV8SwQERmVC7ISLHAxE3SX/3VKfucdue8iBHYoyptfdAFcnx0UR5pG7f2LwuAExdGbyBqOGm1TmDgB8AL4tIKiDALuC6QAYVirqnxgOwbW85w3sGORhjDOAkgtSEhrPiTxyYydAeKXhEmjnLNObL5HGLgOEikuJuR2SL6cFEEHGFIWNCVlOJAOCYXp15a8FmvF7F47GE0JpmE4GI/KKZ/QCo6qMBiikkdUmMJTbaw7a9VjVkTKjYs7+yyUQwsHsKZZU1PPnFGm47tX8QIgsvLbURJLfyiCgiQv/MJD5fscN6DhkTIiqqvSTERB2yf2A35yPqsSmr2juksNRsicDtHWTquWZcb+54Zyl9fvshBQ+dE+xwjIl4NV5t0FBc66huB7+r7i6rpLM76tg0zRavb4Ozhnave77dqoiMCbrmEkGn2GjuO38IAN/aCoOtskTQBqmdYvjLFc4SyzbK2Jjgq9HmG4PPHOrMEWbzDrXOEkEbHd8vHYD1RWVBjsQY4/Uq0c0kgqyUeAZ3T7F5h3zQavdRdynJS4Dc+sfXziYaabomxpIcF01BsSUCY4Kt2qtEtTBe4JRBmfxt2lrKKqpJjPNl2FRk8qVE8B5wAVANlNV7RCQRITc90UoExoQAbzNtBLUGd0+hxqtsKLY1ClriS4rMUdUzAx5JGMlLT2TBht3BDsOYiFejLSeCnM6dANi8ez+Ds21h++b4UiL4SkSODngkYeSobsls2XOAeet3BTsUYyJaTSsjh3M6O3OEbd5tMwK0xJdEMAFYICIrRWSJiCx1l5+MWKN6dQbgsmdms3OfdSM1JlhqWmgsBkjrFEOn2ChLBK3wpWrorIBHEWbG9enCGUOy+OSbHazcUUJmSnywQzImIlV7tcXJ5USEnM4JbNptbQQt8WWpyg1AGnCe+0hz97VIROJFZJ6ILBaRb0TkPnd/nojMFZHVIvKmiITdkD8R4e5zBwOwxb5pGBM0LXUfrdWzcyc22YL2LfJlqcrbgH8Ame7jNRH5iQ/XrgAmqepwYARwpoiMA/4IPKaq/YHdwI2HG3wwdUuJJzbaw+LNe4IdijERq7XGYoDc9EQKistQtTnCmuNLG8GNwFhV/T9V/T9gHPD91k5yF7ovdTdj3IcCk4C33P0vAxe2OeoQEB3l4bTBWUy3xS+MCZrWGosB+mQkUl7lZaOVCprlSyIQoKbedo27r/UTRaJEZBGwE/gMWAvsUdVq95DNQA/fww0tA7OS2bq3nP2V1a0fbIzxu9YaiwHG5Dorls21Xn7N8iURvAjMFZF7ReReYA7wvC8XV9UaVR0B5ADHAoOaOqypc0XkZhHJF5H8wsLQ/Nadl5EI2HQTxgSDquJVWl2JrH9mEl0SY5m9tridIgs/vjQWPwpcj7NE5W7gelV9vC1voqp7gGk41UppIlLbWykH2NrMOZNVdbSqjs7IyGjL27WbPulJAKwrtERgTHurcdcFaa1EICKcOiiTj5Zto7zKqdyw9oKGmk0EtUtTikgXoAB4DXgV2ODua5GIZIhImvs8ATgVWAFMBS51D7sWZwqLsJSX7pQILBEY0/5q3A9zX5ainDQwi/IqL8/PXM+zX64j784PWbTJOnrUamkcwevAucACGlbfiLvdp5Vrd8dZ9D4KJ+H8S1XfF5HlwBsi8gDwNT5WM4WihNgoeqQlsL6otPWDjTF+5fU6P1vrNQQHVyz78ycr6/Z9+s12RvRMC0hs4aalFcrOdX/mHc6FVXUJMLKJ/etw2gs6hL6ZSby7aCud4qL5w0U2E4cx7aXazQStVQ0BZKclHLJvg/UiquPLOILPfdkXqS4bnQPA63M3BjkSYyJLbYmgtcZigNjoQz/qltgYoDottRHEu20B6SLSWUS6uI9cILu9Agx15xzdnWN6O3MP3fRyfpCjMSZy1JUIonzqzc5FI3uQ7K5JkJ0az6ZdB5ixOjR7JLa3ltoIbgF+hvOhv4CDYwf2AX8NcFxhQ0S4ZlxvFmzYzZQVOygsqSAjOS7YYRnT4dU1FvtQIgB47HJnmdn5BbvwiHDJ37/ib1PXckL/0OyV2J5aaiP4C/AXEfmJqj7ZjjGFneH1Gpy27DlgicCYdtCWxuL6ageY5Xbt1OZzOypfxhE8KSJDReQyEfle7aM9ggsXeemJPHDhUADWFVoPImPaQ0W1MyYgNurwll4fnJ3CZpuVFPCtsfge4En3MRH4E3B+gOMKO5eN7kl2ajzvLNwS7FCMiQjFZZUAdE06vAmMj+ndhYLi/SzbstefYYUlX1LppcApwHZVvR4YDljdRyOx0R5OH9KN/A27KK2wuYeMCbTiUjcRJB7ex9Glx+SQEBPFS18V+DGq8ORLIjigql6g2h1tvJPWB5NFpHOHdae8ysvnK3YEOxRjOrzi0grg8EsEqQkxnDEki+mrrOeQL4kg350q4lmc3kMLgXkBjSpMDe2RiggUFFm9ozGBVls11CXx8Ne26t01kcKSirr2hkjV6lKVqvpD9+nTIvIxkOKOGjaNxMdEkZUcz4ZdNveQMYFWVFpBUlw08TFRh32NHu6I4x17K+jVtZO/Qgs7zSYCERnV0muqujAwIYW3/llJfLutJNhhGNPhFZdWHna1UK3aqSe27j1giaAZj7g/44HRwGKcQWXDgLnAhMCGFp6G5aTy9PR1VFZ7mxzWbozxj+KyiiOqFgLonhYPwNY9kb32eLOfVKo6UVUnAhuAUe7aAMfgTCS3pr0CDDe9uyZS41W27y0PdijGdGjlVV4SY1ut3W5RdqpbIrBE0KqBqrq0dkNVl+EsRm+akNPZ+cPaZANVjAkoryo+zi7RrITYKLokxrI1wr+4+ZIIVojIcyJysoicJCLP4iwwY5rQP9OZ93z51n1BjsSYjs2XZSp9UV5Vw+tzN9atXtZYcWkFSzd37EFnviSC64FvgNtwJqFb7u4zTchIjqNvRiKvzd1gy+EZE0Ber+KPqYL2VzoJYHETK5btr6zm0qdnc95TM3n5qwKqarxH/oYhyJe5hspV9TFVvch9PKaqkV2OasV1x+WyoXg/G4qtesiYQPGq+mXSuLduHQ/A5ZPnsHd/VYPXPl62nfVFTnfwe/77Db/7z7Ijfr9Q1NJ6BP9yfy4VkSWNH+0XYvgZ2ctZn2D5NqseMiZQvOpMA3+kav+/AizctLvBa/kbdpMSH81PJ/UD4M38TR2ypN9SieA29+e5wHlNPEwzahe1f3/J1g5ft2hMsKj6p2ooyiMsvfd0gEP+v27Z7Ywv+MXpR/GbMwcCMOzeTyksqTjyNw4hLXUf3eb+3NDUo7ULi0hPEZkqIitE5BsRuc3df6+IbBGRRe7jbP/9OqEhMS6arJQ4Ply6nfOemtkhv0EYE2w1XvVLYzFAcnwMfTISWdIoEWzdc6Bu9PG1x/UmISaKkopq7nxnKeVVNRyo7BhTU7RUNVQiIvuaeJSIiC91HtXAL1V1EDAO+JGIDHZfe0xVR7iPD/3we4Scc4cdXM1zlzsnijHGf7zqv0QAMKhbClNW7GjQe2hXWSVdk5zZTTvFRjPvrlO45cQ+TFmxg4F3f8yv31rst/cPppZKBMmqmtLEI1lVU1q7sKpuq52GQlVLcLqc9vBf6KHttlP7c1SW05X01TmtFqCMMW2kCh5/rjDmXuqPH39bt6ukorpunWNwSg6/OH1A3fbHy7b77/2DyOc5EEQkU0R61T7a8ibugvcjcaamAPix2+j8goh0bvbEMJYSH8PHPzuBMbmdeW3OBvYeqGr9JGOMz7x+aiOodeOEPABenFVAQVEZldVeKqu9JMU1HL0cFx3F1F+dzGmDs6j2Kl+tKfJfEEHiywpl54vIamA9MB0oAD7y9Q1EJAl4G/iZqu4D/g70xRmdvI2Dcxo1Pu9mEckXkfzCwvCcL1xE+NXpR1FUWslFf53VYfsgGxMM/hpQVmtUr848fbUz1+bT09dS5i4wlRR/6DQWeemJnDYoC4DvPjf3kNfDjS8lgv+HU8e/SlXzcFYrm+XLxUUkBicJ/ENV3wFQ1R2qWuMudvMscGxT56rqZHd+o9EZGRm+vF1Iql0oe11RGQPv/rjuj8sYc2RqvEc+xURjZw7tzmWjc3hj/iZW7nBmEU6Ma3o+o1G9D1ZmhPv/a18SQZWqFgMeEfGo6lR8mGtInA6+zwMrVPXRevu71zvsIqBjjtBweTzCBz91Jmqt8Sr/nLcxyBEZ0zGoKlH+zgTAtcflAnDF5DkADdoI6uuXmcSvzzgKgAc+WO73ONqTL4lgj1u98yXwDxH5C06PoNYcD1wDTGrUVfRPtYPUgInAzw83+HAxJDu17vkDH6yoW2LPGHP4/F01VGtQt4Z9YTJTml8TeWyeU+L/57xNTU5RES58SQQXAPtxPrA/Btbiw4AyVZ2pqqKqw+p3FVXVa1T1aHf/+bXjFTq6+g1Ony63NY2NOVJeVTwBWPLD4xH+7U47ATDA7f3XlJG9OnOdW4K44K+zwraHoC+38WYgW1WrVfVlVX3CrSoybTDzNxOZ8osTAbjznaW8YVVExhwRZxpq/5cIwGnb++RnJ/L7i4aSHB/T7HFRHuGe8wbXbT/+2aqAxBNoviSCFOATEZkhIj8SkaxAB9URpXWKpV9mMr26OMvh3fHOUutSaswRcKqGAnf9o7olc9XY3q0eJyL88OS+ZCTHUVJeTY03/GYS8GX20ftUdQjwIyAbmC4iUwIeWQf10W0n1D2fsTo8u8UaEwq8AWosPhy3nzmQX50+gMoaLws27G79hBDTlhq2ncB2oBjIDEw4HV9iXDTL7juD+BgPX3y7M9jhGBO2vN7AVQ0djqN7pAHw+tzwayfwZUDZD0RkGvA5kA58X1WHBTqwjiwpLprTB3dj+spCvGFYjDQmFGiAeg0drsHZKQzLSaUgDNch8aVE0BtnVPAQVb1HVcO7w2yImDQwk+KySr5aa+3uxhyOGj9PMeEPI3umsWZnadjNOOxLG8EdqrqoPYKJJJMGZZISH83LswuCHYoxYclfK5T5U/+sZEorqpmyIryqfQPQC9f4IiU+hhP6Z7DKHcZujGkbf61Q5k+1Yw6+/0p+kCNpG0sEQdQvM4lNu/Y3mP/cGOMbf61Q5k8DspLqnp/zxAz2lYdHF3FLBEHULzMJr8JXa8N/Gltj2ps/Vyjzl7ROsYzJdSaj+2brPobd+yn3vBf606lZIgiiET2d7mYPvL8iyJEYE34CPaDscP371uNY94eDK/C+PDv0u5NaIgiinl06cfuZR7GuqIzh933KhuKyYIdkTFio7ZXj1xXK/MjjEe6/YEjddqhPNGmJIMjG9ekKwN4DVbw4qyC4wRgTJmqH34Ra1VB93xufy9s/OA6AOet2BTmallkiCLJRvTrz4nVjSE+K46WvCvzWuDR3XTHPTF/rl2sZE2q8tSWC0M0DAAzLSSXaI3yzdW+wQ2mRJYIQMHFgJheNzAbg/v/5Z7ze5ZPn8OBH31qPJNMh1U7sFmrdRxuLifKQl57Iqh2lwQ6lRZYIQsTtZw7krKHdeGvBZiY9Mu2IrrVlz4G653/8+FumLN8RdiMdjWlJ7Z9zqA0oa8qoXp2Zu644pNcst0QQImKiPHWrHa0rLKO6jX80NV6tO+fUR6bX7X9xVgE3vZLPFZPnsDEM50AxpinhUjUEcFy/rpRUVLM6hEsFlghCyKSBB5d6WLHN9xHHe/dX0fe3H3L2EzNQVQ40UR00d/0uXpi13i9xGhNsBxNB6GeCo3s4S9W+vXAzr84uCGoszWl6VWYTFL26duLru0/j2D9M4bynZgLw44n9+MVpA5rsJrd08172HKjkjXmbAFi1o7RuwW2Aqb86mYKiMt5auJkPlmxjxbZ97fOLGBNgXrfAHOptBAB56YnkpSfy/Ezni9iJAzLo3TUxyFE1ZCWCENM5MZaJRx1c7uGpqWtYuLHphS4unzyba56fx6x6I5Pnrne6qb18w7HkpScycWAmT105kltO6sOCDbvDZsi7MU2p8Sqz1hRRUuH8HYdD1ZCIcNMJeXXbJ/15Gn/4MLQGkQYsEYhITxGZKiIrROQbEbnN3d9FRD4TkdXuz86BiiFcnTOse4Pt2g/3xvZXOlVAe/Y7/ynuOntQ3WvZqfF1z0WE4/umU+1Vlm0J7W5sxrTks+U7uOq5uTz8yUogPBqLAS4Y0aPB9uQv14VUl9JAlgiqgV+q6iBgHPAjERkM3AF8rqr9cRa7uSOAMYSlgd1SABjaI4VeXTrx9cY9hxxTWe2t+zZ0dI9UZtw+kdMGH2xjyKqXCACGZDvXzC8Iv2X0jKlVXOaM0H130VaAujXAQ11SXDR/u2oUL14/ht9fNBSA+c18wQuGgCUCVd2mqgvd5yXACqAHcAHwsnvYy8CFgYohXPXPTOLGCXk88p0RnDEkiykrdjDpkWkNFsWesmIHXoWHvzOc//1kAj27dCI3PZF7zhvMhH7pJMc1bP7pmhTHcX278ub8TWG5uLYxAKXl1Q22T+yfEaRI2u7so7sz8ahMvntsLzp3iuHb7aEzBX27tBGISC4wEpgLZKnqNnCSBbb+8SE8HuHucwdzVLdkbpjg1C2uKyxjbWEpc9YVc8FTM/nhPxYCh34juv74PF67aWyTjWgXj8phy54DrN4ZOn+AxrRFaUXDRBCqcw21RESIjfbwxvxN7NhXHuxwgHZIBCKSBLyNs9ylz91WRORmEckXkfzCwsLABRjiuqcmMO1XJwPw+JRVXDF5Dos3H6xbHNkrzedrDctxurEt3Rw6dZPGtEVpRTXJcdFMPCqDE/qnBzucwzY2z5ljbNaa0JiCPqCJQERicJLAP1T1HXf3DhHp7r7eHWhyTTdVnayqo1V1dEZG+BT/AiE3PZGRvdL4cOn2un2XjMrh67tPIybK93/CvhlJ5HRO4N8LNgciTGMCrrS8mqT4aF68/lhevXFssMM5bI9cNpxoj7BsS2h06Q5kryEBngdWqOqj9V76L3Ct+/xa4L1AxdCR1Db2Alw+uiePXDaczomxbbpGlEe4/vg85q3fxX++tmRgwk9ZZTWdYqOCHcYRi4nyMKF/Oi/MWh8SXboDWSI4HrgGmCQii9zH2cBDwGkisho4zd02rbhxQh8AxvfpykOXHH3Y17nuuFyG56TyxOdrbP4hE3bKq7wkdIBEAHDRSKdL6eOfrW72mP2V1c0dQOhcAAAZy0lEQVS+5k+B7DU0U1VFVYep6gj38aGqFqvqKara3/0ZOn2oQlheeiLz7jqF564dfUSjKaM8wiXH5LC+qKzB5HTGhIPKai9x0R0jEZw/3JlxeNWOpjtvrNxewnEPfcGXqwLfRmoji8NIZnI8iXFHPivIUHfuk4VNjE8wJpRVVNcQ24Z2sVAmIlw1thcLN+6mrOLQb/6vzdlARZW3QbVwoHSMO2ra5OgeqWSnxvPe11uCHYoxbVJR7SUupuN8bF00sgf7K2v4YMk2wOlFtGCDU0myYts+hmSn0DUpLuBx2KRzESgmysMJ/TN4M38TU7/dycSBNpTDhIeKKi9x0R0nERzTuzN9MhK5/e0lPPHFajbvdqprfzyxH/kbdnPrSX3bJY6Oc0dNm5wx1JmO4vqX5tcVS2u8ytrC0J0z3ZjKmo7TRgBO9dDlo3sC1CUBcCabBLjh+Nx2icMSQYSaNDCLP1zk9D566KNvUVXu+e8yTnlkOnPXFbf5el98u8OWxTQBV1FV06FKBOCM+G/KcX27kpkS3+Rr/tax7qhpk++O7cWNE/J4dc4G3l+yjTfnO+safLRseytnNrSxeD83vJTPwLs/tmRgAqqi2ktsB0sEGclxfPrzEwG45aQ+pHWKAZxBo+3F2ggi3C9OG8BLXxXwu3eXUVXjjCv4am0RXq/6PI9L/QExn3yz/ZApd43xhxqvUlxW2aGqhmoNyEpm1h2TyEqO4/i+6cxaW8TZR3dv/UQ/6Vip1bRZYlw05w7rzt4Dzof5baf0Z9WOUqat2sn6ojK89WYq3bu/ijVNTFhXey7Aok3WJdUExhOfOwOvvt7UMadS75GWQHSUhxMHZHDnWYPadeCcJQLDOfW+eVw8yvk2f8NL+Ux8eBq/emtx3WtXPz+XUx/9kuoaZ53AnSXl/PSfXzPdHfASG+3hxVkFTfaJNqYpXq82+LLRkm+2OvPy/KCdetJEEksEhpPrLY3Zu2sivz7jqLrtdxZu4cevL0RVWequbtbvro94cdZ6rpg8h/8u3srkL9cBkNM5AYB/ztvYjtGbcFVZ7eXoez/hmAc+8+n48qoaRvVK4/Qh3QIcWeSxNgJDbLSHpLjournefzSxH51io1hfVMYrs52G5Mb1svf9b/kh13n2e6M55ZHpPPDBCuJiorhmXO92id+Epyc+X01ZZQ1llTVMX1XISQNanmV4X3kVnTu1baJF4xsrERgAvrx9Yt26B+AscHP/BUPpkeZ8y3974cHZSuu3IX902wmAM4dR34wk7jhrIAB3v7uMzbv3Bz5wE5bKKqp5evrauu1Xvipo9ZyS8mpSEmICGFXkskRgAOiSGEtueuIh+7+8fWKDxW/m33UqU+sljEHdU5h/16nM/e0pAHz/hD7ce95gYqKE33+wIuBxm9ZVVNfwnae/Ysbq0Fng6ern51LtVX552gBOGZjJt9tLWu16XFJeRXK8VWIEgiUC06Ioj3CxO13u7y8aSkZyHL26dOLiUT145YZjAacfdLo7H0qUR7ju+DxuO6U/Hy3bzrItthpae9pVVsk3Wxve89U7SplfsJtrnp/Ha3M2BCmyhr52JzzsnBjLJcc4S6iOuP/Tuo4IjS3etIei0koy2mHenUhkicC06prxuXz68xO5ckwvwBkW/+hlIzixhTrdS49xhs3PW2+zjLenq56byzlPzGyQDBZvPtil93fvLgv6Qiher5LszqJ72eienDmkGz3SEiiv8jZYhhWcBuK7313GBX+dRXpSLN8bb+1OgWCJwPhkQFZymxYK75YaT9+MRN5bvDWAUZnGVmxzulie88RMtu45wGmPTueu/ywjMTaqrmT3V3cem2BZvHkPJRXV/OnSYcRGe/B4hHd+eBzQcA3fjcX7GXj3x7zqlmIuG92zXWbijESWCEzAnDc8m8Wb9nDVc3N87ituDl+NV4mql6yPe+gLVu90JhG8+cS+PHr5CC4e1YMXZxUcsvLV+qKydhkMuGd/JRf97SsAzhx6sBtoVko84/t05d16U6Mvq1eqmTQwk5+e0j/g8UUqSwQmYGoHqs1aU8y/F2xixbZ9rNnZ9OymawtLbSDaEdpVVkmNV7n73MGHvHb9hFwAzhuWTWW1t+5D/415G/m/95Zx7QvzuPCvs9hVVtnm91V1BoX9b/FWDlQ23eD7r/mbeGvBZi57ZjYAvbp0IiW+YQ+gEwdksK6ojNfnOuNQtror6D1/7WheuG4M8TEdb2qJUGFN8CZg+mcls/7Bsxn34Of85u2ldfvn/vYUsurNqlhZ7eWUR6Yzomca7/7o+GCEGtZUlfcWbeV37y4DIDs1npm/mcj6ojJenb2Bxy4fUbey3ahenRGBT7/ZwWfLd/DirIIG15q1pojz3CUUW1Jd46Wssobh930KgAiowvCcVP561ShyOndi2Za97CuvYnyfrtz+9pIG579249hDrnnhyGz++PG3/PY/SxnaI4UH3F5npwzKavM9MW0TsEQgIi8A5wI7VXWou+9e4PtAbT+236rqh4GKwQSfiHD12N488tmqun0zVxdxyTHOzIoHKmu48eX5gDNPUXlVjX3za6MFG3bzszcX1W0P6JZMTudO5HTuxAn9Gzbop3aKYViPVF5qpt/+Xz5fzdlHd29QxVTfnv2VvPRVAU9+sYaaetV96j5dvHkvE/44tcV4Y6M99Ora6ZD93VMT6p6f/9QsAE4+quVBZsY/AlkieAl4Cnil0f7HVPXhAL6vCTE/OaU/kwZl8tnyHbz0VQH/W7KV7mnxPPC+841vudvACfDBkm11ScL4Zn1RWYPtPk2MB6nvjrMGceWzcwD42an96ZGWQHZaAtv3lvPLfy/m6427GZ3bhWVb9pIcH03vroks37qPs5+YcdgxDu6ewvJt+4iL9jQYh9LY2z8YzyV/n123XbtmhgmsgCUCVf1SRHIDdX0TXoZkpzIkO5UDVTU8M30d01Y2HNw09Vcn84PXFnD3e8uY0D+9QdWRaVlt99Arj+3FAxcORaTl3l3j+3blT5cMIy8jkTG5Xer279lfSWyUh9fmbGB0bhfOfXImAAUPncPjU1Ydcp2/XzWK4T3TKCmvprLai6I8N2M9x+Z1YcnmPaQmxPDZ8h3cclJfLhmVg1cVj0iL6wkc07sLf7joaN5dtAWPQDf7O2gXwWgj+LGIfA/IB36pqh1zTlnTpKvH9uaZ6esa7PvLFSPIS0/kwYuP5qK/fcW0lTu5bHTPVj/Qam3fW05mclyburd2FPe8t4zX5jiNqw9e7Pu358vG9DxkX1qnWG46IY+/TVvb4PWXZq3n0+U7Djn+rCbmy3/iypHuM6e//13nHNpw3Zrvju3Fd8f2avN55vC1d6+hvwN9gRHANuCR5g4UkZtFJF9E8gsLQ2dovDkyPbt04k+XDAPg1EFZrH/w7LqFbIbnpNEnI5HfvL2UvDs/pLCk4pDz/++9ZVz2zGxW7SihpLyKV+dsYNyDn/PK7IJ2/C18t23vgSbXcPCXl2c7fezvOnuQX653zjDnw/27z86t2/enT1YCzqjxdX84m4e/M9xv72dCQ7uWCFS17muFiDwLvN/CsZOByQCjR4+2TugdyGVjejb5jdTjEU4f3K1uMrJpK3fyndEHj6uq8fKK+8F3+mNfNjj3sSmrufiYnEO6JAZTYUkF4x/8AoAXrhvNyu2lXHlsT9L8NIPm3v3OCOE7zhrI90/s45drDuyW0mAmWoD9lTUkx0Xzr1vH4/EIl1obTofTriUCEalflrwIWNae729C3w9OPrjoyK/fWsKXqwp5Z+FmarzKusKyJs+59aS+lFVUc/u/lzT5ejBUVnsZ8/spdds3vJTPHz/+loc/Xem391i5wylpHJWV7LdrRnmkbpLBIdkpZCQ7I3k//cWJDOqe4rf3MaElkN1H/wmcDKSLyGbgHuBkERkBKFAA3BKo9zfhKTUhhpUPnMkFT83i2+0lfO+FeYAz6di7i5zpKmbcPpHoKOGn//yaa8bncv7wbOJjPDw+ZTUbi/c32TWxPW3atZ/Xm1mcp3aVLX94dobT1nJUN/8lAoBjc7swY3URmclx/PPmccRFezrkOsHmoED2Grqyid3PB+r9TMcRFx3FY5eP4Ky/HOyuWFsX/pNJ/ejZxfmg//etx9W9fu6w7jw+ZTWz1hbx4lcldIqN4lenH4WI4PVqQBuSV2zbh1eVIdmpAPz2P0uZsboIEVhyz+nUeJXb3lhEWUU1+Rt2M3ttMeP7dqXGq5RX1dQN9mqLbXsP8NnyHaQnxZKdltD6CW1w9bjebNy1nxtPyAupqjYTOKIa+tXvo0eP1vz8/GCHYdpZVY2X/yzcwrfbS3hh1nrA6crYFFVl3IOfEyXC1r3lAKTER3PLSX15bc4GJg7M5L7zhxATdWS1oarK3gNVdfX8m3bt54Q/TSXaI1x5bC+GZKfw8KcrKSqt5MXrxzCx3jKgM1cXcfXzTiPs7DsncenfZ3OgqoYvfnkSlz8zh5U7Snj88hFc6E4O15RpK3dy3/+Wc0L/dF6ZvYF/3DSW4/ulH9HvZDouEVmgqqNbO86mmDAhKybKw2VjeuL1Ki/MWs/AFqpARITj+6XzzsKDk5btK6/mz26Pl9fnbqR3l07ccoQLn89eV8x3n53LAxcO5epxvfnhPxYCkBATVTdLJsD9FwxpkAQAxvY52Ge/thEZYMT9B9fs/fMnK+mWGs9NL+fzxJUjmDTw4PQK1TVernvRGYVdO4jsmN6dj+j3MQZs0jkTBjweYcbtE3nz5vEtHnfjhDz6ZCRy+5lHNVhO86yh3RiSncJTX6xh74Ejm4t/uVvH/7t3l1FZ7WWpu/DOxz8/kSvq9YSa0MS39JgoD//54XGH7K91/vBsikoreHXOBkorqvnr1LUNJnGbva74kHNsOg7jD1YiMGGhtl2gJUOyU/nilycDcPKATAqKyxjVqzPdUuOZ+u1Orn9pPk9PX8tvzhzI3v1VpCRE+zxordbOemMbJj48DXAGUfVIS+ChS4YxcWAm2/YcoE9GUpPnj+iZxi0n9qF/VjI90hL4cnUhf5/mdJe95Jgc/rt4Kx8s2QY4cwi9MX8j1x+fR2lFdd3+eXedwkdLt1MS5AVmTMdhicB0SIOzUxicfbC740kDMhjXpwuvzd7Ajr3lvPP1Fi4e2YObTujT4LiWlFZUs66wlLz0RHbuK2eLO03y0T1S6445Y0i35k4HnCqsO+sNxhrftyvH5nahoLiMsXkHq47++t1RPPrZSp6evparxvbmgqdmsrawjLz0RDKT47n2uFyfYjbGF1Y1ZCKCxyPcNKEPJRXVvOMufvLO11s4+4kZhyzS0piqoqoMvecTpqzYSd+MxLoeQgC5R9hddeLATK4/Po/4mCjG5Dp1/iN7pXHbqQPYsa+CKybPZq07huI6SwAmAKxEYCLGhP5N966ZtaaYUwdlUu1VSsur2b2/skHVzouzCrj//eV122cM6ca4Pl157LNV3HXOoDZXL7XklRvGsnXvAbLTEjjLHcy10F3o/brjcm3NXhMQlghMxIiPieJPlw5j7rpd/O6cQfz0ja+ZsbqIX/xrESXlDUsFf7liRN0cSPkbdtXtv2lCXt20F49ePsLvMSbERtHXTUIxUR5evuFYrnUH1d16Ul+/Jh1jatk4AhPRlm3Zy8OfrjxkWmyAmCght2ti3bq/Y/O68OSVI8ls56mRC4rK2L6vnHF9urbr+5rw5+s4AmsjMBFtaI9UJl9z8P9JelIsmW6VTFWN1iWB0wdn8eYt49s9CQDkpidaEjABZVVDJuLFRnv45r4ziI+JIsoj7D1Qxf+9t4z33LmNAH51xlFBjNCYwLJEYAw0mO8nNSGGv1wxkj9eMowZq4t4e8Fm+mc2PS7AmI7AEoExzYiPieK0wVmcNjir9YONCWPWRmCMMRHOEoExxkQ4SwTGGBPhLBEYY0yEs0RgjDERzhKBMcZEOEsExhgT4SwRGGNMhAuLSedEpBDYA+yttzu1he10oMiPITR+L38c39wxvu5vy7bdD7sfdj8i8370VtWMVt+xdtGNUH8Ak33dBvID+d7+OL65Y3zdb/fD7ofdD7sfR/petY9wqhr6Xxu3A/ne/ji+uWN83W/348i2/cnux5Fd2+5H249v6/1oUVhUDbWViOSrD3NwRwq7Hw3Z/WjI7kdDkXg/wqlE0BaTgx1AiLH70ZDdj4bsfjQUcfejQ5YIjDHG+K6jlgiMMcb4yBKBMcZEOEsExhgT4SIuEYjIySIyQ0SeFpGTgx1PKBCRRBFZICLnBjuWYBORQe7fxlsi8oNgxxNsInKhiDwrIu+JyOnBjifYRKSPiDwvIm8FOxZ/CqtEICIviMhOEVnWaP+ZIrJSRNaIyB2tXEaBUiAe2ByoWNuDn+4HwG+AfwUmyvbjj/uhqitU9VbgMiCsuxD66X68q6rfB64DLg9guAHnp/uxTlVvDGyk7S+seg2JyIk4H+KvqOpQd18UsAo4DeeDfT5wJRAFPNjoEjcARarqFZEs4FFVvaq94vc3P92PYThD6uNx7s377RO9//njfqjqThE5H7gDeEpVX2+v+P3NX/fDPe8R4B+qurCdwvc7P9+Pt1T10vaKPdDCavF6Vf1SRHIb7T4WWKOq6wBE5A3gAlV9EGipqmM3EBeIONuLP+6HiEwEEoHBwAER+VBVvQENPED89fehqv8F/isiHwBhmwj89PchwEPAR+GcBMDvnx8dSlglgmb0ADbV294MjG3uYBG5GDgDSAOeCmxoQdGm+6GqdwGIyHW4paWARtf+2vr3cTJwMc6XhA8DGllwtOl+AD8BTgVSRaSfqj4dyOCCoK1/H12B3wMjReRON2GEvY6QCKSJfc3Wd6nqO8A7gQsn6Np0P+oOUH3J/6GEhLb+fUwDpgUqmBDQ1vvxBPBE4MIJurbej2Lg1sCFExxh1VjcjM1Az3rbOcDWIMUSCux+NGT3oyG7Hw3Z/aBjJIL5QH8RyRORWOAK4L9BjimY7H40ZPejIbsfDdn9IMwSgYj8E5gNHCUim0XkRlWtBn4MfAKsAP6lqt8EM872YvejIbsfDdn9aMjuR/PCqvuoMcYY/wurEoExxhj/s0RgjDERzhKBMcZEOEsExhgT4SwRGGNMhLNEYIwxEc4SgfE7ESlth/c438cptv35nieLyHGHcd5IEXnOfX6diITEHFciktt4SuYmjskQkY/bKyYTHJYITMhypwhukqr+V1UfCsB7tjT/1slAmxMB8FvgycMKKMhUtRDYJiLHBzsWEziWCExAicivRWS+iCwRkfvq7X9XnFXRvhGRm+vtLxWR+0VkLjBeRApE5D4RWSgiS0VkoHtc3TdrEXlJRJ4Qka9EZJ2IXOru94jI39z3eF9EPqx9rVGM00TkDyIyHbhNRM4Tkbki8rWITBGRLHf64luBn4vIIhE5wf22/Lb7+81v6sNSRJKBYaq6uInXeovI5+69+VxEern7+4rIHPea9zdVwhJnVbkPRGSxiCwTkcvd/WPc+7BYROaJSLL7zX+Gew8XNlWqEZEoEflzvX+rW+q9/C4Qtut2GB+oqj3s4dcHUOr+PB2YjDPDowd4HzjRfa2L+zMBWAZ0dbcVuKzetQqAn7jPfwg85z6/DmfhGICXgH+77zEYZ355gEtxppL2AN1w1qC4tIl4pwF/q7fdmYOj7m8CHnGf3wv8qt5xrwMT3Oe9gBVNXHsi8Ha97fpx/w+41n1+A/Cu+/x94Er3+a2197PRdS8Bnq23nQrEAuuAMe6+FJwZhjsB8e6+/kC++zwXWOY+vxn4nfs8DsgH8tztHsDSYP9d2SNwj44wDbUJXae7j6/d7SScD6IvgZ+KyEXu/p7u/mKgBni70XVqpw1fgLNWQFPeVWctheXirD4HMAH4t7t/u4hMbSHWN+s9zwHeFJHuOB+u65s551RgsEjdTMYpIpKsqiX1jukOFDZz/vh6v8+rwJ/q7b/Qff468HAT5y4FHhaRPwLvq+oMETka2Kaq8wFUdR84pQfgKREZgXN/BzRxvdOBYfVKTKk4/ybrgZ1AdjO/g+kALBGYQBLgQVV9psFOZ/GXU4HxqrpfRKbhLJUJUK6qNY2uU+H+rKH5v9mKes+l0U9flNV7/iTOMqb/dWO9t5lzPDi/w4EWrnuAg79ba3ye+EtVV4nIMcDZwIMi8ilOFU5T1/g5sAMY7sZc3sQxglPy+qSJ1+Jxfg/TQVkbgQmkT4AbRCQJQER6iEgmzrfN3W4SGAiMC9D7zwQucdsKsnAae32RCmxxn19bb38JkFxv+1OcmSsBcL9xN7YC6NfM+3yFM+0xOHXwM93nc3Cqfqj3egMikg3sV9XXcEoMo4BvgWwRGeMek+w2fqfilBS8wDU46/E29gnwAxGJcc8d4JYkwClBtNi7yIQ3SwQmYFT1U5yqjdkishR4C+eD9GMgWkSWAP8P54MvEN7GWXhkGfAMMBfY68N59wL/FpEZQFG9/f8DLqptLAZ+Cox2G1eX08TKVar6Lc4yj8mNX3PPv969D9cAt7n7fwb8QkTm4VQtNRXz0cA8EVkE3AU8oKqVwOXAkyKyGPgM59v834BrRWQOzod6WRPXew5YDix0u5Q+w8HS10TggybOMR2ETUNtOjQRSVLVUnHWmp0HHK+q29s5hp8DJar6nI/HdwIOqKqKyBU4DccXBDTIluP5EmdB993BisEElrURmI7ufRFJw2n0/X/tnQRcfwe+04bjj8Fp3BVgD06PoqAQkQyc9hJLAh2YlQiMMSbCWRuBMcZEOEsExhgT4SwRGGNMhLNEYIwxEc4SgTHGRDhLBMYYE+H+P4nTqT5XvyT2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "m.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=5e-5; wd=1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36f45107dfa04eba99377cc9d17f54e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.781235  14.807194  3.748016  \n",
      "    1      12.958119  14.762188  3.742506                      \n",
      "    2      14.384175  14.414407  3.69296                       \n",
      "    3      14.071619  14.339529  3.68361                       \n",
      "    4      13.856909  14.229776  3.667675                      \n",
      "    5      11.761227  14.234771  3.668712                      \n",
      "\n",
      "CPU times: user 57 s, sys: 739 ms, total: 57.7 s\n",
      "Wall time: 56.6 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([14.23477]), 3.6687118778333327]"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 2, cycle_len=2, cycle_mult=2, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = m.predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917,), (160917, 1))"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape, l.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5095"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8457"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "operands could not be broadcast together with shapes (201917,) (123623,) ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-92-89a8e7bfdcd5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrmse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-55-e7df4b972056>\u001b[0m in \u001b[0;36mrmse\u001b[0;34m(x, y)\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mrmse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/ops.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(left, right)\u001b[0m\n\u001b[1;32m   1581\u001b[0m             \u001b[0mrvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1582\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1583\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msafe_na_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1584\u001b[0m         return construct_result(left, result,\n\u001b[1;32m   1585\u001b[0m                                 index=left.index, name=res_name, dtype=None)\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/ops.py\u001b[0m in \u001b[0;36msafe_na_op\u001b[0;34m(lvalues, rvalues)\u001b[0m\n\u001b[1;32m   1527\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1528\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrstate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1529\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mna_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1530\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1531\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_object_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/ops.py\u001b[0m in \u001b[0;36mna_op\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomputation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpressions\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexpressions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1504\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1505\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexpressions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr_rep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0meval_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1506\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1507\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmasked_arith_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/computation/expressions.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(op, op_str, a, b, use_numexpr, **eval_kwargs)\u001b[0m\n\u001b[1;32m    206\u001b[0m     \u001b[0muse_numexpr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0muse_numexpr\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0m_bool_arith_check\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0muse_numexpr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 208\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0meval_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    209\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m_evaluate_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/computation/expressions.py\u001b[0m in \u001b[0;36m_evaluate_numexpr\u001b[0;34m(op, op_str, a, b, truediv, reversed, **eval_kwargs)\u001b[0m\n\u001b[1;32m    121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_evaluate_standard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/computation/expressions.py\u001b[0m in \u001b[0;36m_evaluate_standard\u001b[0;34m(op, op_str, a, b, **eval_kwargs)\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0m_store_test_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrstate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/ops.py\u001b[0m in \u001b[0;36mrsub\u001b[0;34m(left, right)\u001b[0m\n\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrsub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mleft\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mright\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mright\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mleft\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: operands could not be broadcast together with shapes (201917,) (123623,) "
     ]
    }
   ],
   "source": [
    "rmse(l, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 178), (123623, 178))"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_feather('dl_df_beta')\n",
    "df_t.to_feather('dl_df_t_beta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_t = feather.read_dataframe('dl_df_t_beta')\n",
    "df = feather.read_dataframe('dl_df_beta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 178), (123623, 178))"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123623, 179)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "634612cbd4634b3580d1641536dd15f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.791351  15.126971  3.80196   \n",
      "    1      15.088356  15.08216   3.79615                       \n",
      "    2      14.005444  14.609645  3.740064                      \n",
      "    3      12.851993  14.550278  3.730572                      \n",
      "    4      14.876751  14.39631   3.712769                      \n",
      "    5      12.745191  14.421658  3.715282                      \n",
      "\n",
      "CPU times: user 40.6 s, sys: 785 ms, total: 41.4 s\n",
      "Wall time: 40.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([14.42166]), 3.7152817555270703]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 2, wd, cycle_len=2, cycle_mult=2, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.934302457848325"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l = m.predict()\n",
    "rmse(l, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d371c4ff406948bdad2f84dc7a62eb50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                         \n",
      "    0      12.313839  14.685268  3.626335  \n",
      "    1      11.636162  14.866773  3.656945                     \n",
      "    2      14.60143   15.627731  3.78566                      \n",
      "CPU times: user 1min 51s, sys: 2.14 s, total: 1min 53s\n",
      "Wall time: 1min 52s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([15.62773]), 3.7856603576887675]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 3, wd, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddc32b8da05c441a82aae7ef5386cbd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                       \n",
      "    0      12.772112  14.705375  3.792834  \n",
      "    1      11.019245  15.128058  3.844349                   \n",
      "CPU times: user 25.1 s, sys: 543 ms, total: 25.7 s\n",
      "Wall time: 24.8 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([15.12806]), 3.8443493809803377]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 1, wd, cycle_len=2, cycle_mult=3, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=4590)\n",
    "oof = np.zeros(len(df_raw))\n",
    "predictions = np.zeros(len(df_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliers = train_df3['outliers']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = train_df3['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[41000, 41001, 41002, 41003, 41004]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_idx[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Get the val_idx\n",
    "2. Get the model data set \n",
    "`ColumnarModelData.from_data_frame(PATH, val_idx, df, y, cat_flds=cat_flds, bs=64)`\n",
    "3. Get the learner from model `md.get_learner(`\n",
    "4. Fit the model `m.fit`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8a2421a5df74e8e9bf2502783dd7a5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      15.04418   14.080989  3.655247  \n",
      "    1      13.357478  13.860406  3.633134                      \n",
      "    2      15.261212  13.722315  3.614915                      \n",
      "    3      12.620456  13.537075  3.590767                      \n",
      "    4      13.00645   13.456646  3.582054                      \n",
      "    5      13.12599   13.428988  3.579914                      \n",
      "\n",
      "fold 1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e77ac302862f47679cf7027249377bcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      13.617827  13.965022  3.6414    \n",
      "    1      13.079861  13.731323  3.614611                      \n",
      "    2      11.972027  13.74923   3.614342                      \n",
      "    3      14.52318   13.552962  3.598553                      \n",
      "    4      12.174224  13.598773  3.600115                      \n",
      "    5      12.668827  13.580135  3.600397                      \n",
      "\n",
      "fold 2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91f179b44407433aacb1b9946dda7708",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.155871  14.268029  3.673169  \n",
      "    1      14.234623  14.129975  3.654891                      \n",
      "    2      14.202703  13.75935   3.605025                      \n",
      "    3      13.852974  13.683855  3.605496                      \n",
      "    4      13.446477  13.576355  3.584531                      \n",
      "    5      12.311568  13.582815  3.584924                      \n",
      "\n",
      "fold 3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0ea519a22cc4d9393797e2af125bc10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.932624  13.909347  3.629735  \n",
      "    1      14.032097  13.797103  3.615603                      \n",
      "    2      13.875641  13.587077  3.590105                      \n",
      "    3      13.241488  13.546938  3.582971                      \n",
      "    4      12.881615  13.527927  3.581848                      \n",
      "    5      13.862915  13.511095  3.582175                      \n",
      "\n",
      "fold 4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe2913b1cca34a3d8ababa3f01f6ce38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.480497  13.755276  3.603879  \n",
      "    1      13.010159  13.693718  3.595021                      \n",
      "    2      13.706165  13.587178  3.584129                      \n",
      "    3      11.35896   13.613993  3.594746                      \n",
      "    4      14.656499  13.512122  3.581542                      \n",
      "    5      12.654803  13.513496  3.57855                       \n",
      "\n",
      "3.6774048153824945\n",
      "CPU times: user 5min 18s, sys: 4.85 s, total: 5min 23s\n",
      "Wall time: 5min 16s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(df, outliers.values)):\n",
    "    print(\"fold {}\".format(fold_))\n",
    "    md = ColumnarModelData.from_data_frame(PATH, val_idx, df, target.astype(np.float32), \n",
    "                                           cat_flds=cat_flds, bs=128, test_df=df_t)\n",
    "    m = md.get_learner(emb_szs, len(df.columns)-len(cat_flds),\n",
    "                    0.04, 1, [1000, 500, 50, 10], [0.5, 0.5, 0.1, 0.1], y_range=y_range, use_bn=True)\n",
    "    m.fit(lr, 2, wd, cycle_len=2, cycle_mult=2, metrics=[rmse])\n",
    "    oof[val_idx] = m.predict().reshape(1, -1)[0]\n",
    "    predictions += m.predict(is_test=True).reshape(1, -1)[0] / folds.n_splits\n",
    "\n",
    "print(rmse(oof, target))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Round 1 - 3.6829778404491664\n",
    "    - 178 cols \n",
    "    - [1000,500], [0.001,0.01]\n",
    "    - 2, cycle_len=2, cycle_mult=1\n",
    "    - 0.04\n",
    "1. Round 2 - 3.6918684314349646\n",
    "    - 178 cols \n",
    "    - [1000,500], [0.01,0.01]\n",
    "    - 2, cycle_len=2, cycle_mult=2\n",
    "        \n",
    "1. Round 3 - 3.8706636257473663\n",
    "    - 178 cols \n",
    "    - [1000,500], [0.001,0.01]\n",
    "    - 3, cycle_len=2, cycle_mult=2\n",
    "    \n",
    "1. Round 4 - 3.6897606081657064\n",
    "    - 178 cols \n",
    "    - [1000,500,100], [0.001,0.001, 0.01]\n",
    "    - 2, cycle_len=2, cycle_mult=2\n",
    "\n",
    "1. Round 5 - 3.7120662242321023\n",
    "    - 178 cols \n",
    "    - [1000,500,1000], [0.001,0.001, 0.01]\n",
    "    - 2, cycle_len=2, cycle_mult=2\n",
    "    - 0.04\n",
    "1. Round 5 - 3.7179195061868584\n",
    "    - 178 cols \n",
    "    - [1000,500,1000], [0.001,0.001, 0.01]\n",
    "    - 2, cycle_len=2, cycle_mult=2\n",
    "    - 0.01\n",
    "1. Round 6 - 3.7066968715711557\n",
    "    - 178 cols \n",
    "    - [1000,500,1000], [0.001,0.001, 0.01]\n",
    "    - 2, cycle_len=2, cycle_mult=2\n",
    "    - 0.1 \n",
    "1. Round 7 - 3.6924910735895273\n",
    "    - 178 cols \n",
    "    - [1000,500], [0.001,0.01]\n",
    "    - 2, cycle_len=2, cycle_mult=2\n",
    "    - 0.1    \n",
    "1. Round 8 - 3.8299337395345825\n",
    "    - 178 cols \n",
    "    - [100, 1, 100, 10], [0.5, 0.0, 0.01, 0.0]\n",
    "    - 2, cycle_len=2, cycle_mult=2\n",
    "    - 0.01    \n",
    "1. Round 8 - 3.6698599137831964\n",
    "    - 178 cols \n",
    "    - [1000, 500], [0.5, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04   \n",
    "1. Round 9 - 3.6847662491064166\n",
    "    - 178 cols \n",
    "    - [1000, 500], [0.5, 0.1]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04   \n",
    "1. Round 10 - 3.6810496643272974\n",
    "    - 178 cols \n",
    "    - [1500, 500], [0.5, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04   \n",
    "1. Round 11 - 3.6728591602118903\n",
    "    - 178 cols \n",
    "    - [1000, 250], [0.5, 0.01], y_range=y_range)\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04\n",
    "1. Round 11 - 3.6963961816811954\n",
    "    - 178 cols \n",
    "    - 0.04, 1, [1000, 500], [0.5, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04\n",
    "1. Round 12 - 3.7332198713513396\n",
    "    - 178 cols \n",
    "    - 0.04, 1, [1000, 500, 250], [0.5, 0.1, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04\n",
    "    - bs 64\n",
    "1. Round 13 - 3.6926874253656465\n",
    "    - 178 cols \n",
    "    - 0.04, 1, [1000, 500, 250], [0.5, 0.01, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04\n",
    "    - bs 128\n",
    "1. Round 14 - 3.725xxx\n",
    "    - 178 cols \n",
    "    - 0.04, 1, [1000, 500, 250], [0.5, 0.01, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04\n",
    "    - bs 128\n",
    "    - use_bn = False\n",
    "1. Round 15 - 3.682071482200128\n",
    "    - 178 cols \n",
    "    - 0.04, 1, [1000, 500, 250], [0.5, 0.01, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04\n",
    "    - bs 128\n",
    "    - use_bn = True\n",
    "1. Round 16 - 3.6829853042918717\n",
    "    - 178 cols \n",
    "    - 0.04, 1, [1000, 500], [0.5, 0.01]\n",
    "    - 2, wd, cycle_len=2, cycle_mult=2,\n",
    "    - 0.04\n",
    "    - bs 128\n",
    "    - use_bn = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917,), (123623,))"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oof.shape, predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('oof_dl', oof)\n",
    "np.save('predictions_dl', predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.6882473181358773"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1.9204810121779068**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((123623,), (201917,), (201917,))"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape, oof.shape, target.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"></ul></div>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}