test_Deep3D.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple Toy TS Training for Deep3D+"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Simple tester for the deep3d\n",
    "\"\"\"\n",
    "import tensorflow as tf\n",
    "import Deep3D_Final as deep3d\n",
    "import utils\n",
    "import numpy as np\n",
    "import os\n",
    "import os.path\n",
    "\n",
    "# for auto-reloading external modules\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "batchsize = 50\n",
    "num_epochs = 20\n",
    "print_step = 1\n",
    "\n",
    "\n",
    "left_dir = \"/a/data/deep3d_data/frames/train/left/\"\n",
    "right_dir = \"/a/data/deep3d_data/frames/train/right/\"\n",
    "\n",
    "with tf.device('/cpu:0'):\n",
    "    left_image_queue = tf.train.string_input_producer(\n",
    "      left_dir + tf.convert_to_tensor(os.listdir(left_dir)),\n",
    "      shuffle=False, num_epochs=num_epochs)\n",
    "    right_image_queue = tf.train.string_input_producer(\n",
    "      right_dir + tf.convert_to_tensor(os.listdir(right_dir)),\n",
    "      shuffle=False, num_epochs=num_epochs)\n",
    "\n",
    "    # use reader to read file\n",
    "    image_reader = tf.WholeFileReader()\n",
    "\n",
    "    _, left_image_raw = image_reader.read(left_image_queue)\n",
    "    left_image = tf.image.decode_jpeg(left_image_raw)\n",
    "    left_image = tf.cast(left_image, tf.float32)/255.0\n",
    "\n",
    "    _, right_image_raw = image_reader.read(right_image_queue)\n",
    "    right_image = tf.image.decode_jpeg(right_image_raw)\n",
    "    right_image = tf.cast(right_image, tf.float32)/255.0\n",
    "\n",
    "    left_image.set_shape([160,288,3])\n",
    "    right_image.set_shape([160,288,3])\n",
    "\n",
    "    # preprocess image\n",
    "    batch = tf.train.shuffle_batch([left_image, right_image], \n",
    "                                   batch_size = batchsize,\n",
    "                                   capacity = 12*batchsize,\n",
    "                                   num_threads = 1,\n",
    "                                   min_after_dequeue = 4*batchsize)\n",
    "\n",
    "\n",
    "# Define config for GPU memory debugging \n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True  # Switch to True for dynamic memory allocation instead of TF hogging BS\n",
    "config.gpu_options.per_process_gpu_memory_fraction= 1  # Cap TF mem usage\n",
    "config.allow_soft_placement=True\n",
    "\n",
    "\n",
    "# Session\n",
    "sess = tf.Session(config=config)\n",
    "\n",
    "# Placeholders\n",
    "images = tf.placeholder(tf.float32, [None, 160, 288, 3], name='input_batch')\n",
    "true_out = tf.placeholder(tf.float32, [None, 160, 288, 3] , name='ground_truth')\n",
    "train_mode = tf.placeholder(tf.bool, name='train_mode')\n",
    "\n",
    "# Building Net based on VGG weights \n",
    "net = deep3d.Deep3Dnet('./vgg19.npy', dropout = 1.0)\n",
    "net.build(images, train_mode)\n",
    "\n",
    "# Print number of variables used: 143667240 variables, i.e. ideal size = 548MB\n",
    "print 'Variable count:'\n",
    "print(net.get_var_count())\n",
    "\n",
    "# Define Training Objectives\n",
    "with tf.variable_scope(\"Loss\"):\n",
    "    cost = tf.reduce_sum(tf.abs(net.prob - true_out))/batchsize\n",
    "    \n",
    "update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "with tf.control_dependencies(update_ops): \n",
    "    train = tf.train.AdamOptimizer(learning_rate=0.00025).minimize(cost)\n",
    "\n",
    "# Run initializer \n",
    "sess.run(tf.global_variables_initializer())\n",
    "sess.run(tf.local_variables_initializer()) \n",
    "coord = tf.train.Coordinator()\n",
    "queue_threads = tf.train.start_queue_runners(coord=coord, sess=sess)\n",
    "\n",
    "# Track Cost    \n",
    "tf.summary.scalar('cost', cost)\n",
    "# tensorboard operations to compile summary and then write into logs\n",
    "merged = tf.summary.merge_all()\n",
    "writer = tf.summary.FileWriter('./tensorboard_logs/', graph = sess.graph)\n",
    "\n",
    "\n",
    "# Training Loop\n",
    "print \"\"\n",
    "print \"== Start training ==\"\n",
    "\n",
    "#base case\n",
    "next_batch = sess.run(batch)\n",
    "i=0\n",
    "try:\n",
    "    while not coord.should_stop():\n",
    "        \n",
    "            # Traing Step\n",
    "        _, cost_val, next_batch, summary, up_conv = sess.run([train, cost, batch, merged, net.up_conv],\n",
    "                                                    feed_dict={images: next_batch[0],\n",
    "                                                               true_out: next_batch[1],\n",
    "                                                               train_mode: True})\n",
    "        writer.add_summary(summary, i)\n",
    "\n",
    "        # No longer needed: cost_hist.append(cost_val)\n",
    "        if i%print_step == 0:\n",
    "            print str(i) + ' | Cost: ' + str(cost_val) + \" | UpConv Max: \" + str(np.mean(up_conv, axis =(0,1,2)).max())\n",
    "        i+=1\n",
    "\n",
    "except tf.errors.OutOfRangeError:\n",
    "    print('Done training -- epoch limit reached')\n",
    "\n",
    "#finally:\n",
    "    # When done, ask the threads to stop.\n",
    "    #coord.request_stop()\n",
    "    \n",
    "\n",
    "print \"\"\n",
    "print \"Training Completed, storing weights\"\n",
    "# Store Traing Output\n",
    "net.save_npy(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#termination stuff\n",
    "coord.request_stop()\n",
    "coord.join(queue_threads)\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "inria_file = '/a/data/deep3d_data/inria_data.h5'\n",
    "# inria_file = 'data/inria_data.h5'\n",
    "h5f = h5py.File(inria_file,'r')\n",
    "\n",
    "# X_train_0 = h5f['X_0'][:,10:170,16:304,:]\n",
    "# Y_train_0 = h5f['Y_0'][:,10:170,16:304,:]\n",
    "# X_train_1 = h5f['X_1'][:,10:170,16:304,:]\n",
    "# Y_train_1 = h5f['Y_1'][:,10:170,16:304,:]\n",
    "# X_train_2 = h5f['X_2'][:,10:170,16:304,:]\n",
    "# Y_train_2 = h5f['Y_2'][:,10:170,16:304,:]\n",
    "# X_train_3 = h5f['X_3'][:,10:170,16:304,:]\n",
    "# Y_train_3 = h5f['Y_3'][:,10:170,16:304,:]\n",
    "# X_train_4 = h5f['X_4'][:,10:170,16:304,:]\n",
    "# Y_train_4 = h5f['Y_4'][:,10:170,16:304,:]\n",
    "# X_train_5 = h5f['X_5'][:,10:170,16:304,:]\n",
    "# Y_train_5 = h5f['Y_5'][:,10:170,16:304,:]\n",
    "# X_train_6 = h5f['X_6'][:,10:170,16:304,:]\n",
    "# Y_train_6 = h5f['Y_6'][:,10:170,16:304,:]\n",
    "# #X_train_7 = h5f['X_7'][:,10:170,16:304,:]\n",
    "# #Y_train_7 = h5f['Y_7'][:,10:170,16:304,:]\n",
    "\n",
    "X_val = h5f['X_7'][:,10:170,16:304,:]\n",
    "Y_val = h5f['Y_7'][:,10:170,16:304,:]\n",
    "  \n",
    "h5f.close()\n",
    "\n",
    "# # ------------------------------------------#\n",
    "# X_train = np.concatenate([X_train_0,X_train_1,X_train_2,X_train_3,X_train_4,X_train_5,X_train_6])\n",
    "# Y_train = np.concatenate([Y_train_0,Y_train_1,Y_train_2,Y_train_3,Y_train_4,Y_train_5,Y_train_6])\n",
    "\n",
    "# print \"Training Size:\" + str(X_train.shape)\n",
    "print \"Validation Size:\" + str(X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Test\n",
    "test_img = np.expand_dims(X_val[365], axis = 0)\n",
    "test_ans = Y_val[365]\n",
    "\n",
    "with tf.device(\"/gpu:0\"):\n",
    "    res, mask, up_conv = sess.run([net.prob, net.mask, net.up_conv], \n",
    "                                  feed_dict={images: test_img, train_mode: False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "print \"--- Input ---\"\n",
    "plt.imshow(test_img[0])\n",
    "plt.show()\n",
    "\n",
    "print \"--- GT ---\"\n",
    "plt.imshow(test_ans)\n",
    "plt.show()\n",
    "\n",
    "print \"--- Our result ---\"\n",
    "plt.imshow(res[0])\n",
    "plt.show()\n",
    "\n",
    "#pyplot.imsave('1.jpeg', test_img[0])\n",
    "#pyplot.imsave('2.jpeg', res[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Data from H5 Format for fast loading\n",
    "- Will eventually unit test dynamic CPU data loading pipeline here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Looking at Disparity Maps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "f, axs = plt.subplots(6, 6, sharex='col', sharey='row',figsize=(14,14))\n",
    "\n",
    "for i in range(33):\n",
    "    axs[i/6][i%6].imshow(mask[0,:,:,i],cmap=\"gray\", vmin=0, vmax=1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_shift_channel = np.argmax(mask,axis = 3)\n",
    "max_shift_channel[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "channel_mean = np.mean(mask[0], axis =(0,1))\n",
    "channel_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(channel_mean)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "channel_act_mean = np.mean(up_conv[0], axis =(0,1))\n",
    "plt.plot(channel_act_mean)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.hist(up_conv[0,:,:,16].ravel(), bins=100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for layer in range(33):\n",
    "    print layer\n",
    "    plt.imshow(up_conv[0,:,:,layer], cmap=\"gray\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}