regressor/dnn/nn_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.utils import shuffle\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load datasets\n",
    "train = pd.read_feather(\"./data/train_feather\")\n",
    "val = pd.read_feather(\"./data/val_feather\")\n",
    "test = pd.read_feather(\"./data/test_feather\")\n",
    "ids = pd.read_feather(\"./data/test_ids\")\n",
    "\n",
    "# Shuffle time\n",
    "train = shuffle(train)\n",
    "val = shuffle(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneHotEncoder(categorical_features='all', dtype=<class 'numpy.float32'>,\n",
       "       handle_unknown='error', n_values='auto', sparse=True)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "num_steps = 500\n",
    "batch_size = 256\n",
    "display_step = 5\n",
    "\n",
    "# Indices to traverse dataset\n",
    "train_ptr = 0\n",
    "val_ptr = 0\n",
    "\n",
    "# Network Parameters\n",
    "n_hidden_1 = 128 # 1st layer number of neurons\n",
    "n_hidden_2 = 128 # 2nd layer number of neurons\n",
    "n_hidden_3 = 128 # 2nd layer number of neurons\n",
    "num_input = 4207 # one-hot encoding of featuers\n",
    "num_classes = 1 # regression\n",
    "\n",
    "# Sizes of datasets\n",
    "num_samples = train.shape[0]\n",
    "num_samples_val = val.shape[0]\n",
    "num_samples_test = test.shape[0]\n",
    "\n",
    "# For one-hot encoding of the features\n",
    "encoding_vals = np.load(\"./data/encoding_vals.npy\")\n",
    "enc = OneHotEncoder(dtype=np.float32)\n",
    "enc.fit(encoding_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Traverses dataset using begin/end\n",
    "# One-hot encodes all the features as they are all categorical\n",
    "def get_batch_train(begin, end):\n",
    "    batch = train.iloc[begin:end,:].values\n",
    "    labels = batch[:,-1]\n",
    "    features = batch[:,:-1]\n",
    "    one_hots = enc.transform(features).toarray()\n",
    "\n",
    "    return (tf.convert_to_tensor(one_hots), tf.convert_to_tensor(labels))\n",
    "\n",
    "def get_batch_val(begin, end):\n",
    "    batch = val.iloc[begin:end,:].values\n",
    "    labels = batch[:,-1]\n",
    "    features = batch[:,:-1]\n",
    "    one_hots = enc.transform(features).toarray()\n",
    "\n",
    "    return (tf.convert_to_tensor(one_hots), tf.convert_to_tensor(labels))\n",
    "\n",
    "def get_batch_test(begin, end):\n",
    "    batch = test.iloc[begin:end,:].values\n",
    "    one_hots = enc.transform(batch).toarray()\n",
    "\n",
    "    return tf.convert_to_tensor(one_hots)\n",
    "\n",
    "def variable_summaries(var):\n",
    "  \"\"\"Attach a lot of summaries to a Tensor (for TensorBoard visualization).\"\"\"\n",
    "  with tf.name_scope('summaries'):\n",
    "    mean = tf.reduce_mean(var)\n",
    "    tf.summary.scalar('mean', mean)\n",
    "    with tf.name_scope('stddev'):\n",
    "      stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))\n",
    "    tf.summary.scalar('stddev', stddev)\n",
    "    tf.summary.scalar('max', tf.reduce_max(var))\n",
    "    tf.summary.scalar('min', tf.reduce_min(var))\n",
    "    tf.summary.histogram('histogram', var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "\n",
    "# tf Graph input\n",
    "X = tf.placeholder(\"float\")\n",
    "Y = tf.placeholder(\"float\")\n",
    "\n",
    "# Store layers weight & bias\n",
    "# h10 = tf.get_variable('h10', [num_input, n_hidden_1], initalizer=tf.random_normal_initializer)\n",
    "with tf.name_scope('weights'):\n",
    "    weights = {\n",
    "        'h1': tf.get_variable('h1', [num_input, n_hidden_1], tf.float32, tf.random_normal_initializer()),\n",
    "        'h2': tf.get_variable('h2', [n_hidden_1, n_hidden_2], tf.float32, tf.random_normal_initializer()),\n",
    "        'h3': tf.get_variable('h3', [n_hidden_2, n_hidden_3], tf.float32, tf.random_normal_initializer()),\n",
    "        'out': tf.get_variable('out', [n_hidden_3, num_classes], tf.float32, tf.random_normal_initializer())  \n",
    "    }\n",
    "    \n",
    "with tf.name_scope('biases'):\n",
    "    biases = {\n",
    "        'b1': tf.get_variable('b1', [n_hidden_1], tf.float32, tf.random_normal_initializer()),\n",
    "        'b2': tf.get_variable('b2', [n_hidden_2], tf.float32, tf.random_normal_initializer()),\n",
    "        'b3': tf.get_variable('b3', [n_hidden_3], tf.float32, tf.random_normal_initializer()),\n",
    "        'outb': tf.get_variable('outb', [num_classes], tf.float32, tf.random_normal_initializer())\n",
    "    }\n",
    "\n",
    "global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "\n",
    "# should make a loop\n",
    "# variable_summaries(weights['h1'])\n",
    "# variable_summaries(weights['h2'])\n",
    "# variable_summaries(weights['h3'])\n",
    "# variable_summaries(weights['out'])\n",
    "# variable_summaries(biases['b1'])\n",
    "# variable_summaries(biases['b2'])\n",
    "# variable_summaries(biases['b3'])\n",
    "# variable_summaries(biases['outb'])\n",
    "\n",
    "# Create model\n",
    "with tf.name_scope('nn'):\n",
    "    def neural_net(x):\n",
    "        # Hidden fully connected layer with 128 neurons\n",
    "        layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])\n",
    "        tf.summary.histogram(\"L1pre-activations\", layer_1)\n",
    "        l1_act = tf.nn.relu(layer_1)\n",
    "        tf.summary.histogram(\"L1activated\", l1_act)\n",
    "        # Hidden fully connected layer with 128 neurons\n",
    "        layer_2 = tf.add(tf.matmul(l1_act, weights['h2']), biases['b2'])\n",
    "        tf.summary.histogram(\"L2pre-activations\", layer_2)\n",
    "        l2_act = tf.nn.relu(layer_2)\n",
    "        tf.summary.histogram(\"L2activated\", l2_act)\n",
    "        # Hidden fully connected layer with 128 neurons\n",
    "        layer_3 = tf.add(tf.matmul(l2_act, weights['h3']), biases['b3'])\n",
    "        tf.summary.histogram(\"L3pre-activations\", layer_3)\n",
    "        l3_act = tf.nn.relu(layer_3)\n",
    "        tf.summary.histogram(\"L3activated\", l3_act)\n",
    "        # Output fully connected layer with a neuron for prediction\n",
    "        out_layer = tf.matmul(l3_act, weights['out']) + biases['outb']\n",
    "        return out_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward prop\n",
    "prediction = neural_net(X)\n",
    "tf.summary.histogram(\"output\", prediction)\n",
    "\n",
    "# Define loss and optimizer\n",
    "loss_op = tf.losses.mean_squared_error(labels=Y, predictions=prediction)\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)\n",
    "train_op = optimizer.minimize(loss_op, global_step=global_step)\n",
    "\n",
    "# Evaluation\n",
    "mae = tf.metrics.mean_absolute_error(labels=Y, predictions=prediction)\n",
    "\n",
    "# Tensorboard\n",
    "tb_loss = tf.summary.scalar(\"loss\", loss_op)\n",
    "tb_mae = tf.summary.scalar(\"MAE\", tf.reduce_mean(mae))\n",
    "merged = tf.summary.merge_all()\n",
    "\n",
    "saver = tf.train.Saver(tf.global_variables())\n",
    "\n",
    "# Initialize the variables (i.e. assign their default value)\n",
    "init = tf.global_variables_initializer()\n",
    "init_l = tf.local_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring and Intializing\n",
      "INFO:tensorflow:Restoring parameters from ./model_checks/model.ckpt\n",
      "Begin datetime: 2018-01-09 05:55:12.506118\n",
      "Step 1/1401, Minibatch Loss = 1907.03, Training MAE = (0.0, 27.485388), Eval MAE = (27.485388, 27.21974)\n",
      "Step 5/1405, Minibatch Loss = 2645.16, Training MAE = (27.21974, 32.626328), Eval MAE = (32.626328, 35.367386)\n",
      "Step 10/1410, Minibatch Loss = 3741.49, Training MAE = (35.367386, 36.539783), Eval MAE = (36.539783, 36.838596)\n",
      "Step 15/1415, Minibatch Loss = 1287.7, Training MAE = (36.838596, 35.746689), Eval MAE = (35.746689, 35.707558)\n",
      "Step 20/1420, Minibatch Loss = 1362.81, Training MAE = (35.707558, 35.017857), Eval MAE = (35.017857, 34.469757)\n",
      "Step 25/1425, Minibatch Loss = 1494.15, Training MAE = (34.469757, 34.102634), Eval MAE = (34.102634, 33.977554)\n",
      "Step 30/1430, Minibatch Loss = 1826.51, Training MAE = (33.977554, 34.09267), Eval MAE = (34.09267, 34.137104)\n",
      "Step 35/1435, Minibatch Loss = 1479.84, Training MAE = (34.137104, 33.823254), Eval MAE = (33.823254, 33.383312)\n",
      "Step 40/1440, Minibatch Loss = 2198.42, Training MAE = (33.383312, 33.37323), Eval MAE = (33.37323, 33.150528)\n",
      "Step 45/1445, Minibatch Loss = 2710.7, Training MAE = (33.150528, 33.759464), Eval MAE = (33.759464, 34.352253)\n",
      "Step 50/1450, Minibatch Loss = 1350.64, Training MAE = (34.352253, 34.082336), Eval MAE = (34.082336, 33.920124)\n",
      "Step 55/1455, Minibatch Loss = 1806.38, Training MAE = (33.920124, 33.916466), Eval MAE = (33.916466, 33.77227)\n",
      "Step 60/1460, Minibatch Loss = 1455.56, Training MAE = (33.77227, 33.615627), Eval MAE = (33.615627, 33.398102)\n",
      "Step 65/1465, Minibatch Loss = 2231.85, Training MAE = (33.398102, 33.487469), Eval MAE = (33.487469, 33.621307)\n",
      "Step 70/1470, Minibatch Loss = 1376.38, Training MAE = (33.621307, 33.544003), Eval MAE = (33.544003, 33.474842)\n",
      "Step 75/1475, Minibatch Loss = 2653.24, Training MAE = (33.474842, 33.373192), Eval MAE = (33.373192, 33.210228)\n",
      "Step 80/1480, Minibatch Loss = 2775.34, Training MAE = (33.210228, 33.503304), Eval MAE = (33.503304, 33.681892)\n",
      "Step 85/1485, Minibatch Loss = 1585.94, Training MAE = (33.681892, 33.567348), Eval MAE = (33.567348, 33.453896)\n",
      "Step 90/1490, Minibatch Loss = 1414.03, Training MAE = (33.453896, 33.266251), Eval MAE = (33.266251, 33.04813)\n",
      "Step 95/1495, Minibatch Loss = 2813.82, Training MAE = (33.04813, 33.384418), Eval MAE = (33.384418, 33.583)\n",
      "Step 100/1500, Minibatch Loss = 963.248, Training MAE = (33.583, 33.356148), Eval MAE = (33.356148, 33.145412)\n",
      "Step 105/1505, Minibatch Loss = 2250.03, Training MAE = (33.145412, 33.294212), Eval MAE = (33.294212, 33.443729)\n",
      "Step 110/1510, Minibatch Loss = 1486.68, Training MAE = (33.443729, 33.314777), Eval MAE = (33.314777, 33.106514)\n",
      "Step 115/1515, Minibatch Loss = 855.32, Training MAE = (33.106514, 32.867126), Eval MAE = (32.867126, 32.746799)\n",
      "Step 120/1520, Minibatch Loss = 1025.99, Training MAE = (32.746799, 32.575466), Eval MAE = (32.575466, 32.443935)\n",
      "Step 125/1525, Minibatch Loss = 2386.08, Training MAE = (32.443935, 32.642933), Eval MAE = (32.642933, 32.798767)\n",
      "Step 130/1530, Minibatch Loss = 1120.16, Training MAE = (32.798767, 32.66954), Eval MAE = (32.66954, 32.639641)\n",
      "Step 135/1535, Minibatch Loss = 956.502, Training MAE = (32.639641, 32.502384), Eval MAE = (32.502384, 32.394718)\n",
      "Step 140/1540, Minibatch Loss = 2071.08, Training MAE = (32.394718, 32.484028), Eval MAE = (32.484028, 32.533306)\n",
      "Step 145/1545, Minibatch Loss = 1397.23, Training MAE = (32.533306, 32.5144), Eval MAE = (32.5144, 32.452728)\n",
      "Step 150/1550, Minibatch Loss = 934.515, Training MAE = (32.452728, 32.301933), Eval MAE = (32.301933, 32.224483)\n",
      "Step 155/1555, Minibatch Loss = 3719.8, Training MAE = (32.224483, 32.530712), Eval MAE = (32.530712, 32.789303)\n",
      "Step 160/1560, Minibatch Loss = 887.016, Training MAE = (32.789303, 32.629997), Eval MAE = (32.629997, 32.533295)\n",
      "Step 165/1565, Minibatch Loss = 968.689, Training MAE = (32.533295, 32.4249), Eval MAE = (32.4249, 32.361332)\n",
      "Step 170/1570, Minibatch Loss = 2104.75, Training MAE = (32.361332, 32.437222), Eval MAE = (32.437222, 32.517994)\n",
      "Step 175/1575, Minibatch Loss = 1249.07, Training MAE = (32.517994, 32.467815), Eval MAE = (32.467815, 32.446747)\n",
      "Step 180/1580, Minibatch Loss = 863.609, Training MAE = (32.446747, 32.313004), Eval MAE = (32.313004, 32.200054)\n",
      "Step 185/1585, Minibatch Loss = 2124.68, Training MAE = (32.200054, 32.298508), Eval MAE = (32.298508, 32.383602)\n",
      "Step 190/1590, Minibatch Loss = 1114.41, Training MAE = (32.383602, 32.315746), Eval MAE = (32.315746, 32.232849)\n",
      "Step 195/1595, Minibatch Loss = 1112.91, Training MAE = (32.232849, 32.17886), Eval MAE = (32.17886, 32.147087)\n",
      "Step 200/1600, Minibatch Loss = 3129.34, Training MAE = (32.147087, 32.278843), Eval MAE = (32.278843, 32.350155)\n",
      "Step 205/1605, Minibatch Loss = 827.614, Training MAE = (32.350155, 32.241985), Eval MAE = (32.241985, 32.151951)\n",
      "Step 210/1610, Minibatch Loss = 1822.5, Training MAE = (32.151951, 32.189018), Eval MAE = (32.189018, 32.213959)\n",
      "Step 215/1615, Minibatch Loss = 1923.63, Training MAE = (32.213959, 32.1884), Eval MAE = (32.1884, 32.171577)\n",
      "Step 220/1620, Minibatch Loss = 2381.28, Training MAE = (32.171577, 32.22929), Eval MAE = (32.22929, 32.286522)\n",
      "Step 225/1625, Minibatch Loss = 965.409, Training MAE = (32.286522, 32.218678), Eval MAE = (32.218678, 32.17485)\n",
      "Step 230/1630, Minibatch Loss = 1490.33, Training MAE = (32.17485, 32.162384), Eval MAE = (32.162384, 32.132069)\n",
      "Step 235/1635, Minibatch Loss = 1013.93, Training MAE = (32.132069, 32.06633), Eval MAE = (32.06633, 32.020969)\n",
      "Step 240/1640, Minibatch Loss = 2499.9, Training MAE = (32.020969, 32.140766), Eval MAE = (32.140766, 32.270885)\n",
      "Step 245/1645, Minibatch Loss = 1000.37, Training MAE = (32.270885, 32.190411), Eval MAE = (32.190411, 32.085045)\n",
      "Step 250/1650, Minibatch Loss = 1153.52, Training MAE = (32.085045, 32.01123), Eval MAE = (32.01123, 31.966026)\n",
      "Step 255/1655, Minibatch Loss = 1797.79, Training MAE = (31.966026, 32.011875), Eval MAE = (32.011875, 32.065689)\n",
      "Step 260/1660, Minibatch Loss = 905.351, Training MAE = (32.065689, 31.982391), Eval MAE = (31.982391, 31.900682)\n",
      "Step 265/1665, Minibatch Loss = 1398.38, Training MAE = (31.900682, 31.894039), Eval MAE = (31.894039, 31.869568)\n",
      "Step 270/1670, Minibatch Loss = 1731.33, Training MAE = (31.869568, 31.900673), Eval MAE = (31.900673, 31.962416)\n",
      "Step 275/1675, Minibatch Loss = 1375.45, Training MAE = (31.962416, 31.956602), Eval MAE = (31.956602, 31.93112)\n",
      "Step 280/1680, Minibatch Loss = 992.024, Training MAE = (31.93112, 31.848438), Eval MAE = (31.848438, 31.763634)\n",
      "Step 285/1685, Minibatch Loss = 1808.22, Training MAE = (31.763634, 31.812317), Eval MAE = (31.812317, 31.861784)\n",
      "Step 290/1690, Minibatch Loss = 1428.56, Training MAE = (31.861784, 31.807621), Eval MAE = (31.807621, 31.756001)\n",
      "Step 295/1695, Minibatch Loss = 907.337, Training MAE = (31.756001, 31.677385), Eval MAE = (31.677385, 31.610447)\n",
      "Step 300/1700, Minibatch Loss = 1694.8, Training MAE = (31.610447, 31.632658), Eval MAE = (31.632658, 31.665178)\n",
      "Step 305/1705, Minibatch Loss = 2101.96, Training MAE = (31.665178, 31.686123), Eval MAE = (31.686123, 31.694527)\n",
      "Step 310/1710, Minibatch Loss = 1060.4, Training MAE = (31.694527, 31.646296), Eval MAE = (31.646296, 31.610144)\n",
      "Step 315/1715, Minibatch Loss = 1224.17, Training MAE = (31.610144, 31.59058), Eval MAE = (31.59058, 31.574804)\n",
      "Step 320/1720, Minibatch Loss = 2160.24, Training MAE = (31.574804, 31.609812), Eval MAE = (31.609812, 31.652426)\n",
      "Step 325/1725, Minibatch Loss = 1176.94, Training MAE = (31.652426, 31.606718), Eval MAE = (31.606718, 31.584902)\n",
      "Step 330/1730, Minibatch Loss = 1439.81, Training MAE = (31.584902, 31.561941), Eval MAE = (31.561941, 31.524258)\n",
      "Step 335/1735, Minibatch Loss = 1066.06, Training MAE = (31.524258, 31.490843), Eval MAE = (31.490843, 31.43885)\n",
      "Step 340/1740, Minibatch Loss = 1663.15, Training MAE = (31.43885, 31.43343), Eval MAE = (31.43343, 31.403072)\n",
      "Step 345/1745, Minibatch Loss = 1171.21, Training MAE = (31.403072, 31.38238), Eval MAE = (31.38238, 31.343842)\n",
      "Step 350/1750, Minibatch Loss = 937.423, Training MAE = (31.343842, 31.291313), Eval MAE = (31.291313, 31.253658)\n",
      "Step 355/1755, Minibatch Loss = 1388.47, Training MAE = (31.253658, 31.258055), Eval MAE = (31.258055, 31.281105)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 360/1760, Minibatch Loss = 1097.83, Training MAE = (31.281105, 31.236235), Eval MAE = (31.236235, 31.187193)\n",
      "Step 365/1765, Minibatch Loss = 1543.09, Training MAE = (31.187193, 31.200178), Eval MAE = (31.200178, 31.203926)\n",
      "Step 370/1770, Minibatch Loss = 1115.63, Training MAE = (31.203926, 31.156057), Eval MAE = (31.156057, 31.114252)\n",
      "Step 375/1775, Minibatch Loss = 1013.54, Training MAE = (31.114252, 31.080017), Eval MAE = (31.080017, 31.060255)\n",
      "Step 380/1780, Minibatch Loss = 1417.49, Training MAE = (31.060255, 31.064083), Eval MAE = (31.064083, 31.046402)\n",
      "Step 385/1785, Minibatch Loss = 1274.73, Training MAE = (31.046402, 31.030586), Eval MAE = (31.030586, 31.020708)\n",
      "Step 390/1790, Minibatch Loss = 1348.08, Training MAE = (31.020708, 30.987225), Eval MAE = (30.987225, 30.942005)\n",
      "Step 395/1795, Minibatch Loss = 753.848, Training MAE = (30.942005, 30.88925), Eval MAE = (30.88925, 30.843739)\n",
      "Step 400/1800, Minibatch Loss = 1199.74, Training MAE = (30.843739, 30.829979), Eval MAE = (30.829979, 30.827055)\n",
      "Step 405/1805, Minibatch Loss = 1216.57, Training MAE = (30.827055, 30.811003), Eval MAE = (30.811003, 30.791319)\n",
      "Step 410/1810, Minibatch Loss = 1087.37, Training MAE = (30.791319, 30.755606), Eval MAE = (30.755606, 30.717789)\n",
      "Step 415/1815, Minibatch Loss = 669.518, Training MAE = (30.717789, 30.655005), Eval MAE = (30.655005, 30.590837)\n",
      "Step 420/1820, Minibatch Loss = 2223.91, Training MAE = (30.590837, 30.599066), Eval MAE = (30.599066, 30.605024)\n",
      "Step 425/1825, Minibatch Loss = 776.657, Training MAE = (30.605024, 30.558119), Eval MAE = (30.558119, 30.530657)\n",
      "Step 430/1830, Minibatch Loss = 718.168, Training MAE = (30.530657, 30.452148), Eval MAE = (30.452148, 30.382259)\n",
      "Step 435/1835, Minibatch Loss = 1639.1, Training MAE = (30.382259, 30.411741), Eval MAE = (30.411741, 30.447523)\n",
      "Step 440/1840, Minibatch Loss = 3507.09, Training MAE = (30.447523, 30.410954), Eval MAE = (30.410954, 30.354006)\n",
      "Step 445/1845, Minibatch Loss = 1044.22, Training MAE = (30.354006, 30.334024), Eval MAE = (30.334024, 30.325104)\n",
      "Step 450/1850, Minibatch Loss = 1005.69, Training MAE = (30.325104, 30.295712), Eval MAE = (30.295712, 30.277401)\n",
      "Step 455/1855, Minibatch Loss = 1068.2, Training MAE = (30.277401, 30.255598), Eval MAE = (30.255598, 30.228165)\n",
      "Step 460/1860, Minibatch Loss = 1371.25, Training MAE = (30.228165, 30.229769), Eval MAE = (30.229769, 30.216534)\n",
      "Step 465/1865, Minibatch Loss = 1189.92, Training MAE = (30.216534, 30.213037), Eval MAE = (30.213037, 30.218311)\n",
      "Step 470/1870, Minibatch Loss = 949.936, Training MAE = (30.218311, 30.178734), Eval MAE = (30.178734, 30.129587)\n",
      "Step 475/1875, Minibatch Loss = 1327.72, Training MAE = (30.129587, 30.132544), Eval MAE = (30.132544, 30.126394)\n",
      "Step 480/1880, Minibatch Loss = 1303.68, Training MAE = (30.126394, 30.10688), Eval MAE = (30.10688, 30.081684)\n",
      "Step 485/1885, Minibatch Loss = 852.173, Training MAE = (30.081684, 30.050434), Eval MAE = (30.050434, 30.021955)\n",
      "Step 490/1890, Minibatch Loss = 1350.22, Training MAE = (30.021955, 30.025904), Eval MAE = (30.025904, 30.04282)\n",
      "Step 495/1895, Minibatch Loss = 988.402, Training MAE = (30.04282, 30.02611), Eval MAE = (30.02611, 30.002991)\n",
      "Step 500/1900, Minibatch Loss = 758.27, Training MAE = (30.002991, 29.946743), Eval MAE = (29.946743, 29.88101)\n",
      "End datetime: 2018-01-09 05:56:00.700356\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "GraphDef cannot be larger than 2GB.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-f772c3bd62f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0;31m# Save model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;31m#     print(\"h1 : %s\" % weights['h1'].eval())\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m     \u001b[0msave_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"./model_checks/model.ckpt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Model saved at: \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, sess, save_path, global_step, latest_filename, meta_graph_suffix, write_meta_graph, write_state)\u001b[0m\n\u001b[1;32m   1494\u001b[0m           checkpoint_file, meta_graph_suffix=meta_graph_suffix)\n\u001b[1;32m   1495\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1496\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexport_meta_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeta_graph_filename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1497\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1498\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_empty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mexport_meta_graph\u001b[0;34m(self, filename, collection_list, as_text, export_scope, clear_devices, clear_extraneous_savers)\u001b[0m\n\u001b[1;32m   1526\u001b[0m     return export_meta_graph(\n\u001b[1;32m   1527\u001b[0m         \u001b[0mfilename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1528\u001b[0;31m         \u001b[0mgraph_def\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_graph_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0madd_shapes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1529\u001b[0m         \u001b[0msaver_def\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaver_def\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1530\u001b[0m         \u001b[0mcollection_list\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollection_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mas_graph_def\u001b[0;34m(self, from_version, add_shapes)\u001b[0m\n\u001b[1;32m   2483\u001b[0m     \"\"\"\n\u001b[1;32m   2484\u001b[0m     \u001b[0;31m# pylint: enable=line-too-long\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2485\u001b[0;31m     \u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_as_graph_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfrom_version\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0madd_shapes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2486\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_as_graph_def\u001b[0;34m(self, from_version, add_shapes)\u001b[0m\n\u001b[1;32m   2444\u001b[0m           \u001b[0mbytesize\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mByteSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2445\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0mbytesize\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m<<\u001b[0m \u001b[0;36m31\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mbytesize\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2446\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"GraphDef cannot be larger than 2GB.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2447\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_functions\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2448\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: GraphDef cannot be larger than 2GB."
     ]
    }
   ],
   "source": [
    "# Start training\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    print(\"Restoring and Intializing\")\n",
    "    saver.restore(sess, \"./model_checks/model.ckpt\")\n",
    "#     print(\"h1 : %s\" % weights['h1'].eval())\n",
    "\n",
    "    # Run the initializer\n",
    "#     sess.run(init)\n",
    "    sess.run(init_l)\n",
    "#     print(\"h1 : %s\" % weights['h1'].eval())\n",
    "\n",
    "\n",
    "    # Tensorboard writers\n",
    "    train_writer = tf.summary.FileWriter(\"./log_tb\" + '/train', sess.graph)\n",
    "    test_writer = tf.summary.FileWriter(\"./log_tb\" + '/test')\n",
    "\n",
    "    print(\"Begin datetime: \" + str(datetime.datetime.now()))\n",
    "    # TRAINING ///\n",
    "    for step in range(1, num_steps+1):\n",
    "        try:\n",
    "            # Get batch and use begin/end indices to traverse dataset\n",
    "            batch_x, batch_y = sess.run(get_batch_train(train_ptr, train_ptr+batch_size))\n",
    "            train_ptr+=batch_size\n",
    "            \n",
    "            # Run optimization op (backprop)\n",
    "            sess.run(train_op, feed_dict={X: batch_x, Y: batch_y})\n",
    "        except IndexError:\n",
    "            print(\"Finished an epoch, shuffling dataset\")\n",
    "            shuffle(train)\n",
    "            train_ptr=0\n",
    "\n",
    "        # VALIDATING and printing stats\n",
    "        if step % display_step == 0 or step == 1:\n",
    "            # Calculate batch loss and accuracy\n",
    "            summary, loss, mean_abs_train = sess.run([merged, loss_op, mae], feed_dict={X: batch_x, Y: batch_y})\n",
    "            train_writer.add_summary(summary, tf.train.global_step(sess, global_step))\n",
    "\n",
    "            # Validation accuracy\n",
    "            try:\n",
    "                val_batch_x, val_batch_y = sess.run(get_batch_val(val_ptr, val_ptr+batch_size))\n",
    "                val_ptr+=batch_size\n",
    "            except StopIteration:\n",
    "                print(\"Reshuffling validation set\")\n",
    "                shuffle(val)\n",
    "                val_ptr=0\n",
    "\n",
    "                val_batch_x, val_batch_y = sess.run(get_batch_val(val_ptr, val_ptr+batch_size))\n",
    "                val_ptr+=batch_size\n",
    "\n",
    "            summary, mean_abs_val = sess.run([merged, mae], feed_dict={X: val_batch_x, Y: val_batch_y})\n",
    "            test_writer.add_summary(summary, tf.train.global_step(sess, global_step))\n",
    "\n",
    "            print(\"Step \" + str(step) + \"/\" + str(tf.train.global_step(sess, global_step)) + \", Minibatch Loss = \" + \\\n",
    "                  str(loss) + \", Training MAE = \" + \\\n",
    "                  str(mean_abs_train) + \", Eval MAE = \" + str(mean_abs_val))\n",
    "\n",
    "\n",
    "    print(\"End datetime: \" + str(datetime.datetime.now()))\n",
    "    # Save model\n",
    "#     print(\"h1 : %s\" % weights['h1'].eval())\n",
    "    save_path = saver.save(sess, \"./model_checks/model.ckpt\")\n",
    "    print(\"Model saved at: \" + str(save_path))\n",
    "\n",
    "    # Predict values for submission\n",
    "#     submission = np.array([[0,0]])\n",
    "#     counter = 0\n",
    "#     test_ptr = 0\n",
    "    \n",
    "#     while(True):\n",
    "#         if (test_ptr > num_samples_test):\n",
    "#             np.savetxt(\"./submission.txt\", submission, delimiter=',')\n",
    "#             print(\"Saved Submissions!\")\n",
    "#             break\n",
    "            \n",
    "#         try:\n",
    "#             if(counter % 10000 == 0):\n",
    "#                 print(\"batch # \" + str(counter))\n",
    "#             counter+=1\n",
    "\n",
    "#             test_batch = sess.run(get_batch_test(test_ptr, test_ptr+batch_size))\n",
    "#             test_ids = ids.iloc[test_ptr:test_ptr+batch_size].values.reshape((batch_size,-1))\n",
    "#             test_ptr+=batch_size\n",
    "\n",
    "#             predictions = sess.run([prediction], feed_dict={X: test_batch})\n",
    "#             preds = sess.run(tf.squeeze(preds)).reshape((batch_size, -1))\n",
    "#             new_subs = sess.run(tf.squeeze(np.array([test_ids, preds]).T))\n",
    "#             submission = np.concatenate((submission, new_subs))\n",
    "#             print(\"Concatenated!\")\n",
    "\n",
    "#         except IndexError:\n",
    "#             np.savetxt(\"./submission.txt\", submission, delimiter=',')\n",
    "#             print(\"Saved Submissions!\")\n",
    "#             break\n",
    "\n",
    "    print(\"Optimization Finished!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}