notebooks/regression_in_tensorflow_with_tensorboard.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regression Example in TensorFlow (with TensorBoard)\n",
    "Predicting house prices in Boston, Massachusetts while logging results for TensorBoard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/jonkrohn/DLTFpT/blob/master/notebooks/regression_in_tensorflow_with_tensorboard.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow.keras.datasets import boston_housing \n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout \n",
    "from tensorflow.keras.layers import BatchNormalization \n",
    "from tensorflow.keras.callbacks import ModelCheckpoint \n",
    "from tensorflow.keras.callbacks import TensorBoard # new!  \n",
    "import os "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_valid, y_valid) = boston_housing.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404, 13)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(102, 13)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1.23247,   0.     ,   8.14   ,   0.     ,   0.538  ,   6.142  ,\n",
       "        91.7    ,   3.9769 ,   4.     , 307.     ,  21.     , 396.9    ,\n",
       "        18.72   ])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15.2"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Dense(32, input_dim=13, activation='relu'))\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "model.add(Dense(16, activation='relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Dense(1, activation='linear'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 32)                448       \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 32)                128       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 16)                528       \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 16)                64        \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 16)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 17        \n",
      "=================================================================\n",
      "Total params: 1,185\n",
      "Trainable params: 1,089\n",
      "Non-trainable params: 96\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='mean_squared_error', optimizer='adam', )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = 'model_output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_name = 'regression_baseline'\n",
    "output_path = output_dir + run_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(output_path):\n",
    "    os.makedirs(output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(output_path + '/weights.{epoch:02d}.hdf5', # decimal integers\n",
    "                                  save_weights_only=True) # otherwise full model is saved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = TensorBoard(log_dir='logs/' + run_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 404 samples, validate on 102 samples\n",
      "Epoch 1/32\n",
      "404/404 [==============================] - 1s 2ms/sample - loss: 587.6668 - val_loss: 680.7983\n",
      "Epoch 2/32\n",
      "404/404 [==============================] - 0s 323us/sample - loss: 565.8801 - val_loss: 659.5214\n",
      "Epoch 3/32\n",
      "404/404 [==============================] - 0s 322us/sample - loss: 548.8434 - val_loss: 631.0423\n",
      "Epoch 4/32\n",
      "404/404 [==============================] - 0s 342us/sample - loss: 531.8849 - val_loss: 561.6263\n",
      "Epoch 5/32\n",
      "404/404 [==============================] - 0s 317us/sample - loss: 509.8884 - val_loss: 539.1380\n",
      "Epoch 6/32\n",
      "404/404 [==============================] - 0s 312us/sample - loss: 486.0127 - val_loss: 499.1526\n",
      "Epoch 7/32\n",
      "404/404 [==============================] - 0s 309us/sample - loss: 452.2261 - val_loss: 468.7240\n",
      "Epoch 8/32\n",
      "404/404 [==============================] - 0s 295us/sample - loss: 415.7714 - val_loss: 438.4499\n",
      "Epoch 9/32\n",
      "404/404 [==============================] - 0s 295us/sample - loss: 380.2730 - val_loss: 418.0454\n",
      "Epoch 10/32\n",
      "404/404 [==============================] - 0s 258us/sample - loss: 336.4345 - val_loss: 398.4512\n",
      "Epoch 11/32\n",
      "404/404 [==============================] - 0s 215us/sample - loss: 299.1088 - val_loss: 319.1686\n",
      "Epoch 12/32\n",
      "404/404 [==============================] - 0s 248us/sample - loss: 258.3119 - val_loss: 264.6858\n",
      "Epoch 13/32\n",
      "404/404 [==============================] - 0s 218us/sample - loss: 220.0767 - val_loss: 203.3719\n",
      "Epoch 14/32\n",
      "404/404 [==============================] - 0s 206us/sample - loss: 180.3471 - val_loss: 219.1501\n",
      "Epoch 15/32\n",
      "404/404 [==============================] - 0s 196us/sample - loss: 145.8800 - val_loss: 103.0927\n",
      "Epoch 16/32\n",
      "404/404 [==============================] - 0s 227us/sample - loss: 127.1186 - val_loss: 72.2505\n",
      "Epoch 17/32\n",
      "404/404 [==============================] - 0s 263us/sample - loss: 97.9500 - val_loss: 56.8262\n",
      "Epoch 18/32\n",
      "404/404 [==============================] - 0s 240us/sample - loss: 88.1242 - val_loss: 47.5330\n",
      "Epoch 19/32\n",
      "404/404 [==============================] - 0s 299us/sample - loss: 68.4377 - val_loss: 41.7972\n",
      "Epoch 20/32\n",
      "404/404 [==============================] - 0s 344us/sample - loss: 60.5731 - val_loss: 44.2351\n",
      "Epoch 21/32\n",
      "404/404 [==============================] - 0s 359us/sample - loss: 60.5693 - val_loss: 36.1036\n",
      "Epoch 22/32\n",
      "404/404 [==============================] - 0s 334us/sample - loss: 50.9928 - val_loss: 29.6428\n",
      "Epoch 23/32\n",
      "404/404 [==============================] - 0s 295us/sample - loss: 52.3127 - val_loss: 29.8090\n",
      "Epoch 24/32\n",
      "404/404 [==============================] - 0s 290us/sample - loss: 49.0510 - val_loss: 25.9772\n",
      "Epoch 25/32\n",
      "404/404 [==============================] - 0s 308us/sample - loss: 43.6791 - val_loss: 24.5425\n",
      "Epoch 26/32\n",
      "404/404 [==============================] - 0s 280us/sample - loss: 53.2793 - val_loss: 104.4379\n",
      "Epoch 27/32\n",
      "404/404 [==============================] - 0s 256us/sample - loss: 49.2610 - val_loss: 158.6787\n",
      "Epoch 28/32\n",
      "404/404 [==============================] - 0s 236us/sample - loss: 45.6568 - val_loss: 21.4948\n",
      "Epoch 29/32\n",
      "404/404 [==============================] - 0s 284us/sample - loss: 46.4782 - val_loss: 26.2483\n",
      "Epoch 30/32\n",
      "404/404 [==============================] - 0s 345us/sample - loss: 37.8259 - val_loss: 73.2823\n",
      "Epoch 31/32\n",
      "404/404 [==============================] - 0s 357us/sample - loss: 55.7427 - val_loss: 35.7386\n",
      "Epoch 32/32\n",
      "404/404 [==============================] - 0s 351us/sample - loss: 38.9442 - val_loss: 42.0964\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f2ef445a0f0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, \n",
    "          batch_size=8, epochs=32, verbose=1, \n",
    "          validation_data=(X_valid, y_valid),\n",
    "          callbacks=[modelcheckpoint, tensorboard]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Performing inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(output_path + '/weights.28.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  9.32909,   0.     ,  18.1    ,   0.     ,   0.713  ,   6.185  ,\n",
       "        98.7    ,   2.2616 ,  24.     , 666.     ,  20.2    , 396.9    ,\n",
       "        18.13   ])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid[42]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14.1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid[42]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[14.776713]], dtype=float32)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(np.reshape(X_valid[42], [1, 13]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}