cnns.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Import inference graph, extend it to a training graph and train network:\n",
    "\n",
    "To extend the graph, we have to add:\n",
    "    - input data structure\n",
    "    - loss function\n",
    "    - optimization op\n",
    "\n",
    "- outdated: To plot all graphs directly in this notebook, run jupyter form\n",
    "  terminal like this:\n",
    "      jupyter notebook --NotebookApp.iopub_data_rate_limit=10000000000\n",
    "- continously tracking nvidia gpu: nvidia-smi -l 10\n",
    "- converting a jupyter notebook file to python script: \n",
    "      jupyter nbconvert --to script cnns.ipynb\n",
    "\"\"\"\n",
    "\n",
    "import os   \n",
    "import sys\n",
    "sys.path.append('./util')\n",
    "sys.path.append('./scripts_cnn_models_slim')\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import slim\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import input_queues as iq\n",
    "import cnn_model\n",
    "import plot_helpers as plt_help\n",
    "import general_helpers as ghelp\n",
    "import nn_helpers as nnhelp\n",
    "import cnn_helpers as chlp\n",
    "import download\n",
    "import resnet_v1 as resnet\n",
    "import cnn_model as cnnm\n",
    "\n",
    "__author__ = \"ud\"\n",
    "__copyright__ = \"Copyright 2017\"\n",
    "__credits__ = [\"phthalo@mailbox.org\"]\n",
    "__license__ = \"\"\n",
    "__version__ = \"0.1\"\n",
    "__maintainer__ = \"ud\"\n",
    "__email__ = \"phthalo@mailbox.org\"\n",
    "__status__ = \"Development\"\n",
    "\n",
    "__all__ = ['train_network_sintel', 'train_network_iiw', 'train_network']\n",
    "\n",
    "\n",
    "# make only 'gpu:0' visible, so that only one gpu is used not both, see also\n",
    "# https://github.com/tensorflow/tensorflow/issues/5066\n",
    "# https://github.com/tensorflow/tensorflow/issues/3644#issuecomment-237631171\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "\n",
    "def train_network_sintel(log_dir, data_dir, path_inference_graph, \n",
    "                         checkpoint_path, restore_scope, image_shape, \n",
    "                         initial_learning_rate, loss_opt, batch_size, \n",
    "                         num_epochs, display_step, save_step, nodes_name_dict,\n",
    "                         logger, dataset='sintel', is_sample=False, norm=True, \n",
    "                         plot_inference_graph=False):\n",
    "    \"\"\"\n",
    "    :param log_dir: path to directory for saving summary/log files\n",
    "    :type log_dir: str\n",
    "    :param data_dir: path to directory with training and validation data (in \n",
    "        this directory the training and validation csv files have to be\n",
    "        located.)\n",
    "    :type data_dir: str\n",
    "    :param path_inference_graph: path to inference graph (graph without \n",
    "        'training' ops)\n",
    "    :type path_inference_graph: str\n",
    "    :param checkpoint_path: path to model parameters (checkpoint files\n",
    "        e.g. 'logs/2/tfmodel-5' or None)\n",
    "    :type checkpoint_path: str\n",
    "    :param restore_scope: a scope name which has to be defined to find \n",
    "        parameters to restore (e.g. 'vgg_16'). Usually this is needed to\n",
    "        find a pre-trained (tf slim) model within another model.\n",
    "    :type restore_scope: str\n",
    "    :param image_shape: Shape of images that should be used for training \n",
    "        (shape of cnn input tensor)\n",
    "    :type image_shape: list with len(image_shape)==3\n",
    "    :param initial_learning_rate: hyper-parameters for initial learning rate\n",
    "    :type initial_learning_rate: float\n",
    "    :param loss_opt: loss function used for optimization \n",
    "        ('berhu', 'l2', 'l2_inv', 'l2_avg')\n",
    "    :type loss_opt: str\n",
    "    :param batch_size: nr of data which is put through the network before \n",
    "        updating it, as default use: 16, 32 or 64. \n",
    "        batch_size determines how many data samples are loaded in the memory \n",
    "        (be careful with memory space)\n",
    "    :type batch_size: int\n",
    "    :param num_epochs: nr of times the training process loops through the \n",
    "        complete training data set (how often is the tr set 'seen')\n",
    "        if you have 1000 training examples, and your batch size is 500, then it\n",
    "        will take 2 iterations to complete 1 epoch.\n",
    "    :type num_epochs: int\n",
    "    :param display_step: every display_step'th training iteration information is\n",
    "        printed to stdout and file training.log in log_dir (default: 100)\n",
    "    :type display_step: int\n",
    "    :param save_step: every save_step'th training iteration a summary file is \n",
    "        written to log_dir and checkpoint files are saved\n",
    "    :type save_step: int\n",
    "    :param nodes_name_dict: dictionary that contains name of input, albedo and \n",
    "        shading output in form {'input': '', \n",
    "                                'output_albedo': '',\n",
    "                                'output_shading': ''}\n",
    "    :type nodes_name_dict: dict\n",
    "    :param dataset: which dataset to use for training (default: 'sintel')\n",
    "    :type dataset: str, must be \\elem {'iiw', 'sintel'}\n",
    "    :param is_sample: flag, if True only a smaller sample size is used for \n",
    "        training and validation (default: False).\n",
    "    :type is_sample: boolean\n",
    "    :param norm: flag, if True image pixels are scaled to range [0, 1]\n",
    "        (default: True)\n",
    "    :type norm: boolean\n",
    "    :param plot_inference_graph: flag, True if inference graph should be \n",
    "        plotted (default: False).\n",
    "    :type plot_inference_graph: boolean\n",
    "    \"\"\"\n",
    "    ############################################################################\n",
    "    logger.info('Training on images of shape: {}'.format(image_shape))\n",
    "    logger.info('Training on [0, 1] normalized pixel values: {}'.format(norm))\n",
    "    logger.info('Initial learning rate: {}'.format(initial_learning_rate))\n",
    "    logger.info('Loss function used for optimization: {}'.format(loss_opt))\n",
    "    logger.info('Batch size: {}'.format(batch_size))\n",
    "    logger.info('# epochs: {}'.format(num_epochs))\n",
    "    logger.info('Write summary and checkpoints to file (in directory ' +\n",
    "                '{}) every '.format(log_dir) +\n",
    "                '{} iterations.'.format(save_step))\n",
    "\n",
    "    # load meta graph (inference graph)\n",
    "    # how to work with restored models:\n",
    "    # https://www.tensorflow.org/programmers_guide/meta_graph\n",
    "    # http://cv-tricks.com/tensorflow-tutorial/save-restore-tensorflow-models-quick-complete-tutorial/\n",
    "    saver_restore = tf.train.import_meta_graph(path_inference_graph, \n",
    "                                               clear_devices=True)\n",
    "\n",
    "    logger.debug('Restored inference graph from\\n' +\n",
    "                 '    {}'.format(path_inference_graph))\n",
    "    variables_to_restore = slim.get_variables_to_restore(include=[restore_scope],\n",
    "                                                         exclude=None)\n",
    "    logger.info('# of parameters that can be restored: ' + \n",
    "                '{}.'.format(len(variables_to_restore)))\n",
    "\n",
    "    ############################################################################\n",
    "\n",
    "    # save default graph in variable:\n",
    "    graph = tf.get_default_graph()\n",
    "    if plot_inference_graph:\n",
    "        # plot imported inference graph:\n",
    "        plt_help.show_graph(graph.as_graph_def())\n",
    "\n",
    "    # lets get the input\n",
    "    x = graph.get_tensor_by_name(name=nodes_name_dict['input'])\n",
    "\n",
    "    # setup target output classes (ground truth):\n",
    "    y_albedo_label = tf.placeholder(dtype=tf.float32, \n",
    "                                    shape=[None] + image_shape, \n",
    "                                    name='out_albedo')\n",
    "    y_shading_label = tf.placeholder(dtype=tf.float32, \n",
    "                                     shape=[None] + image_shape, \n",
    "                                     name='out_shading')\n",
    "\n",
    "    # bool variable that indicates if we are in training mode (training=True) or\n",
    "    # valid/test mode (training=False) this indicator is important if dropout \n",
    "    # or/and batch normalization is used.\n",
    "    try:\n",
    "        # try importing training node (is needed for models that use batch \n",
    "        # normalization etc.)\n",
    "        try:\n",
    "            training = graph.get_tensor_by_name(name='is_training_1:0')\n",
    "        except KeyError:\n",
    "            training = graph.get_tensor_by_name(name='is_training:0')\n",
    "        logger.debug('Was able to catch is_training node!')\n",
    "    except KeyError:\n",
    "        # elsewise just define a placeholder wich is used as dummy variable\n",
    "        # and won't be used later:\n",
    "        training = tf.placeholder(dtype=tf.bool, name='is_training_1')\n",
    "    # define new global step variable:\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "    # get the last global training step if we continue training:\n",
    "    try:\n",
    "        last_global_step = int(checkpoint_path.split('/')[-1].split('-')[-1])\n",
    "    except (ValueError, AttributeError):\n",
    "        last_global_step = 0\n",
    "\n",
    "    invalid_px_mask = tf.placeholder(dtype=tf.float32, \n",
    "                                     shape=[None] + image_shape, \n",
    "                                     name='invalid_px_mask')\n",
    "\n",
    "    # get graph output nodes:\n",
    "    y_albedo_pred = graph.get_tensor_by_name(name=nodes_name_dict['output_albedo'])\n",
    "    y_shading_pred = graph.get_tensor_by_name(name=nodes_name_dict['output_shading'])\n",
    "    # y_albedo_pred = tf.clip_by_value(t=y_albedo, clip_value_min=0, \n",
    "    #                                  clip_value_max=1, \n",
    "    #                                  name='0_1_clipping_albedo')\n",
    "    # y_shading_pred = tf.clip_by_value(t=y_shading, clip_value_min=0,\n",
    "    #                                   clip_value_max=1, \n",
    "    #                                   name='0_1_clipping_shading')\n",
    "\n",
    "    ############################################################################\n",
    "    ############################################################################\n",
    "\n",
    "    with tf.name_scope('loss'):\n",
    "        valid_mask = chlp.get_valid_pixels(image=x, \n",
    "                                           invalid_mask=invalid_px_mask)\n",
    "        d = {'label_albedo': y_albedo_label,\n",
    "             'label_shading': y_shading_label,\n",
    "             'prediction_albedo': y_albedo_pred, \n",
    "             'prediction_shading': y_shading_pred,\n",
    "             'valid_mask': valid_mask}\n",
    "\n",
    "        loss_dict = {'berhu': chlp.sintel_loss_fct(**d, **{'loss_type': 'berhu', \n",
    "                                                           'lambda_': None}),\n",
    "                     'l1': chlp.sintel_loss_fct(**d, **{'loss_type': 'l1',\n",
    "                                                        'lambda_': None}),\n",
    "                     'l2': chlp.sintel_loss_fct(**d, **{'loss_type': 'l2', \n",
    "                                                        'lambda_': 0}),\n",
    "                     'l2_inv': chlp.sintel_loss_fct(**d, **{'loss_type': 'l2',\n",
    "                                                            'lambda_': 1}),\n",
    "                     'l2_avg': chlp.sintel_loss_fct(**d, **{'loss_type': 'l2',\n",
    "                                                            'lambda_': 0.5})\n",
    "                    }\n",
    "        if loss_opt not in ('berhu', 'l1', 'l2', 'l2_inv', 'l2_avg'):\n",
    "            raise ValueError('{} is not a valid loss '.format(loss_opt) + \n",
    "                             'function. Set parameter loss_opt to one of the ' +\n",
    "                             \"following: ('berhu', 'l1', 'l2', 'l2_inv', \" +\n",
    "                             \"'l2_avg')\")\n",
    "\n",
    "        loss = loss_dict[loss_opt]\n",
    "    logger.debug('Defined training losses.')\n",
    "\n",
    "    ############################################################################\n",
    "\n",
    "    # Use an AdamOptimizer to train the network:\n",
    "    with tf.name_scope('optimization'):\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=initial_learning_rate)\n",
    "        \n",
    "        # use slim optimizaton op only if a batch normalization is in the \n",
    "        # tf graph.\n",
    "        use_slim_train_opt = False\n",
    "        for v in graph.as_graph_def().node:\n",
    "            if 'batchnorm' in v.name.lower():\n",
    "                use_slim_train_opt = True\n",
    "                break\n",
    "        # Many networks utilize modules, like BatchNorm, that require \n",
    "        # performing a series of non-gradient updates during training. \n",
    "        # slim.learning.create_train_op allows a user to pass in a list of \n",
    "        # update_ops to call along with the gradient updates.\n",
    "        #   train_op = slim.learning.create_train_op(total_loss, optimizer,\n",
    "        #                                            update_ops)\n",
    "        # By default, slim.learning.create_train_op includes all update ops \n",
    "        # that are part of the `tf.GraphKeys.UPDATE_OPS` collection. \n",
    "        # Additionally, TF-Slim's slim.batch_norm function adds the moving mean\n",
    "        # and moving variance updates to this collection. Consequently, users \n",
    "        # who want to use slim.batch_norm will not need to take any additional\n",
    "        # steps in order to have the moving mean and moving variance updates be\n",
    "        # computed.\n",
    "        # (see https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/slim/python/slim/learning.py)\n",
    "        if use_slim_train_opt:\n",
    "            opt_step = slim.learning.create_train_op(total_loss=loss, \n",
    "                                                     optimizer=optimizer,\n",
    "                                                     global_step=global_step) \n",
    "            logger.debug('Using slim optimizaton op because a batch ' +\n",
    "                         'normalization is apparently in the tf graph.')\n",
    "        else:\n",
    "            opt_step = optimizer.minimize(loss, global_step=global_step)\n",
    "            logger.debug('Using default tf optimizaton op because there is ' +\n",
    "                         'apparently no batch normalization in the tf ' +\n",
    "                         'graph.')\n",
    "\n",
    "    logger.debug('Defined optimization method.')\n",
    "    \n",
    "    \n",
    "    ############################################################################\n",
    "\n",
    "    # to get every summary defined above we merge them to get one target:\n",
    "    merge_train_summaries = tf.summary.merge_all()\n",
    "    # define a FileWriter op which writes summaries defined above to disk:\n",
    "    summary_writer = tf.summary.FileWriter(log_dir)\n",
    "    # Create a saver for writing training checkpoints.\n",
    "    saver = tf.train.Saver(max_to_keep=None)\n",
    "\n",
    "    ############################################################################\n",
    "\n",
    "    # introduce some validation set specific summaries\n",
    "    # These summaries need to be defined blow the function \n",
    "    # merge_train_summaries = tf.summary.merge_all()\n",
    "    # because the validation set summaries are added to the summary writer at \n",
    "    # different times. If they had been summarized with the training summaries \n",
    "    # they would have to be defined at times where merge_train_summaries are \n",
    "    # added to the summary writer\n",
    "    with tf.name_scope('loss/valid/'):\n",
    "        valid_dict = {key: tf.placeholder(dtype=tf.float32, name=key) for \\\n",
    "                      key in loss_dict.keys()}\n",
    "        valid_summaries = [tf.summary.scalar(name=key, tensor=val) for \\\n",
    "                           key, val in valid_dict.items()]\n",
    "        valid_sums_merged = tf.summary.merge(inputs=valid_summaries,\n",
    "                                             collections=None, name=None)\n",
    "    logger.debug('Defined validation losses.')\n",
    "    logger.debug('Finished building training graph.')\n",
    "    logger.info('Total parameters of network: ' +\n",
    "                '{}'.format(nnhelp.network_params()))\n",
    "\n",
    "    ############################################################################\n",
    "    ############################################################################\n",
    "\n",
    "    # import data:\n",
    "    if is_sample:\n",
    "        sample = 'sample_'\n",
    "    else:\n",
    "        sample = ''\n",
    "    \n",
    "    # import training data:\n",
    "    file = sample + 'data_sintel_shading_train.csv'\n",
    "    df_train = pd.read_csv(filepath_or_buffer=data_dir + file, \n",
    "                           sep=',', header=None,\n",
    "                           names=['img', 'alb', 'shad', 'invalid'])\n",
    "    # complete image paths:\n",
    "    df_train = data_dir + df_train\n",
    "\n",
    "    # # enable this line to train on only one image:\n",
    "    # df_train1 = df_train.loc[[0]]\n",
    "    # # replicate this row 100 times:\n",
    "    # df_train = pd.concat([df_train1]*100).reset_index(drop=True)\n",
    "\n",
    "    # instantiate a data queue for feeding data in (mini) batches to cnn:\n",
    "    data_train = iq.DataQueue(df=df_train, batch_size=batch_size,\n",
    "                              num_epochs=num_epochs)\n",
    "\n",
    "    logger.debug('Imported {} training data '.format(dataset) + \n",
    "                 '(#: {}) '.format(data_train.df.shape[0]) + \n",
    "                 'from\\n    {}'.format(data_dir + file))\n",
    "    \n",
    "    # import validation data set: \n",
    "    # why not using the whole validation set for validation at once? \n",
    "    # - limited memory space.\n",
    "    #  -> After each training epoch we will use the complete validation dataset\n",
    "    #     to calculate the error/accuracy on the validation set\n",
    "    file = sample + 'data_sintel_shading_valid.csv'\n",
    "    df_valid = pd.read_csv(filepath_or_buffer=data_dir + file, \n",
    "                           sep=',', header=None,\n",
    "                           names=['img', 'alb', 'shad', 'invalid'])\n",
    "    # complete image paths:\n",
    "    df_valid = data_dir + df_valid\n",
    "    # instantiate a data queue for feeding data in (mini) batches to cnn:\n",
    "    data_valid = iq.DataQueue(df=df_valid, batch_size=batch_size,\n",
    "                              num_epochs=num_epochs)\n",
    "\n",
    "    logger.debug('Imported {} validation data '.format(dataset) +\n",
    "                 '(#: {}) '.format(data_valid.df.shape[0]) + \n",
    "                 'from\\n    {}'.format(data_dir + file))\n",
    "\n",
    "    ############################################################################\n",
    "    ############################################################################\n",
    "\n",
    "    logger.info('Start training:')\n",
    "    # Initialization:\n",
    "    # Op that initializes global variables in the graph:\n",
    "    init_global = tf.global_variables_initializer()\n",
    "    # Op that initializes local variables in the graph:\n",
    "    init_local = tf.local_variables_initializer()\n",
    "\n",
    "    config = tf.ConfigProto(device_count = {'GPU': 1},\n",
    "                            intra_op_parallelism_threads=4\n",
    "    #                        allow_soft_placement = True,\n",
    "    #                        log_device_placement=False\n",
    "                           )\n",
    "    with tf.Session(config=config) as sess: \n",
    "        \n",
    "        ########################################################################\n",
    "        \n",
    "        # initialize all variables:\n",
    "        sess.run([init_global, init_local])\n",
    "\n",
    "        # assign the last true global step to to global step:\n",
    "        sess.run(global_step.assign(last_global_step))\n",
    "        logger.info('Assigned last global training step: ' +\n",
    "                    '{}'.format(global_step.eval()))\n",
    "\n",
    "        if checkpoint_path:\n",
    "            try:\n",
    "                # restore saved model parameters (weights, biases, etc):\n",
    "                saver_restore.restore(sess, checkpoint_path)\n",
    "                logger.info('Restoring parameters from ' +\n",
    "                            '{}'.format(checkpoint_path))\n",
    "            except (tf.errors.InvalidArgumentError, tf.errors.NotFoundError):\n",
    "                # in the worst case parameters are loaded twice.\n",
    "                # restore the parameters:\n",
    "                init_fn = slim.assign_from_checkpoint_fn(checkpoint_path,\n",
    "                                                         variables_to_restore)\n",
    "                init_fn(sess)\n",
    "                logger.info('Restoring parameters from ' +\n",
    "                            '{}'.format(checkpoint_path))\n",
    "\n",
    "        # Adds a Graph to the event file.\n",
    "        # create summary that give output (TensorFlow op that output protocol \n",
    "        # buffers containing 'summarized' data) of the built Tensorflow graph:\n",
    "        summary_writer.add_graph(sess.graph)\n",
    "\n",
    "        # start timer for total training time:\n",
    "        start_total_time = time.time()\n",
    "        # set timer to measure the displayed training steps:\n",
    "        start_time = start_total_time\n",
    "\n",
    "        ########################################################################\n",
    "        \n",
    "        # Training:\n",
    "        # train loop\n",
    "        # train until all data is processed (queue empty),\n",
    "        # number of iterations depends on number of data, number of epochs and \n",
    "        # batch size:\n",
    "        iter_start = data_train.iter_left\n",
    "        logger.info('For training it takes {}\\n'.format(iter_start) +\n",
    "                    '    (= # data / batch_size * epochs) iterations to loop ' +\n",
    "                    'through {} samples of\\n  '.format(data_train.df.shape[0]) +\n",
    "                    '  training data over {} '.format(data_train.num_epochs) +\n",
    "                    'epochs summarized in batches of size ' + \n",
    "                    '{}.\\n'.format(data_train.batch_size) +\n",
    "                    '    So, there are # data / batch_size = ' +\n",
    "                    '{}'.format(int(data_train.df.shape[0]/data_train.batch_size))+\n",
    "                    ' iterations per epoch.')\n",
    "\n",
    "        while data_train.iter_left >= 0:\n",
    "            try:\n",
    "                # take a (mini) batch of the training data:\n",
    "                deq_train = data_train.dequeue()\n",
    "                img_b, alb_b, shad_b, inv_b = iq.next_batch_sintel(deq=deq_train,\n",
    "                                                                   output_shape=image_shape,\n",
    "                                                                   is_scale=True,\n",
    "                                                                   is_flip=True,\n",
    "                                                                   is_rotated=True,\n",
    "                                                                   norm=norm)\n",
    "                # run training/optimization step:\n",
    "                # Run one step of the model.  The return values are the \n",
    "                # activations from the `train_op` (which is discarded) and the \n",
    "                # `loss` Op.  To inspect the values of your Ops or variables, \n",
    "                # you may include them in the list passed to sess.run() and the \n",
    "                # value tensors will be returned in the tuple from the call.\n",
    "                feed_dict_tr = {x: img_b,\n",
    "                                y_albedo_label: alb_b,\n",
    "                                y_shading_label: shad_b,\n",
    "                                invalid_px_mask: inv_b,\n",
    "                                training: True}\n",
    "                sess.run(opt_step, feed_dict=feed_dict_tr)\n",
    "\n",
    "                ################################################################\n",
    "\n",
    "                # report training set accuracy every display_step-th step:\n",
    "                if (data_train.num_iter) % display_step == 0:\n",
    "                    # console output:\n",
    "                    train_loss_dict = {}\n",
    "                    for key, val in loss_dict.items():\n",
    "                        train_loss_dict[key] = sess.run(val, \n",
    "                                                        feed_dict=feed_dict_tr)\n",
    "\n",
    "                    dur_time = time.time() - start_time\n",
    "                    dur_time = ghelp.get_time_format(time_in_sec=dur_time)\n",
    "                    dur_time = ghelp.time_tuple_to_str(time_tuple=dur_time)\n",
    "                    logger.info('step {}: '.format(data_train.num_iter) +\n",
    "                                'training ({}) loss '.format(loss_opt) +\n",
    "                                '{:.4f}'.format(train_loss_dict[loss_opt]) + \n",
    "                                '\\n    (' +\n",
    "                                ', '.join(['{}: {:.4f}'.format(it[0], it[1]) \\\n",
    "                                           for it in train_loss_dict.items() \\\n",
    "                                           if it[0]!=loss_opt]) +\n",
    "                                ', ET: {}).'.format(dur_time))\n",
    "                    # reset timer to measure the displayed training steps:\n",
    "                    start_time = time.time()\n",
    "\n",
    "                ################################################################\n",
    "\n",
    "                # Validation:\n",
    "                # display validation set accuracy and loss after each completed \n",
    "                # epoch (1 epoch ^= data_train.df.shape[0]/data_train.batch_size\n",
    "                # training steps => steps per epoch)\n",
    "                val_epoch = int(data_train.df.shape[0] / data_train.batch_size)\n",
    "                \n",
    "                if data_train.num_iter % val_epoch == 0:\n",
    "                    # After each training epoch we will use the complete \n",
    "                    # validation data set to calculate the error/accuracy on the\n",
    "                    # validation set:\n",
    "                    # loop through one validation data set epoch:\n",
    "                    # initialize dictionary which contains all validation \n",
    "                    # losses:\n",
    "                    valid_loss_dict = dict.fromkeys(loss_dict, 0)\n",
    "                    valid_steps_per_epoch = int(data_valid.df.shape[0] / \n",
    "                                                data_valid.batch_size)\n",
    "                    for j in range(valid_steps_per_epoch):\n",
    "                        # DISCLAIMER: we do not run the opt_step here (on \n",
    "                        # the validation data set) because we do not want to \n",
    "                        # train our network on the validation set. Important \n",
    "                        # for batch normalization and dropout \n",
    "                        # (training -> False).\n",
    "                        # get validation data set (mini) batch:\n",
    "                        lst = iq.next_batch_sintel(deq=data_valid.dequeue(), \n",
    "                                                   output_shape=image_shape,\n",
    "                                                   is_scale=False,\n",
    "                                                   is_flip=False,\n",
    "                                                   is_rotated=False,\n",
    "                                                   norm=norm)\n",
    "                        img_b_val, alb_b_val, shad_b_val, inv_b_val = lst\n",
    "\n",
    "                        # calculate the mean loss of this validation batch and \n",
    "                        # sum it with the previous mean batch losses:\n",
    "                        fd_val = {x: img_b_val,\n",
    "                                  y_albedo_label: alb_b_val,\n",
    "                                  y_shading_label: shad_b_val,\n",
    "                                  invalid_px_mask: inv_b_val,\n",
    "                                  training: False}\n",
    "\n",
    "                        for key, val in loss_dict.items():\n",
    "                            # divide each loss loss by the iteration steps \n",
    "                            # (steps_per_epoch) to get the mean val loss:\n",
    "                            mean_val = val / valid_steps_per_epoch\n",
    "                            valid_loss_dict[key] += sess.run(mean_val, \n",
    "                                                             feed_dict=fd_val)\n",
    "                    # adding a mean loss summary op (for tensorboard):\n",
    "                    feed_dict_vl = {valid_dict[key]: valid_loss_dict[key] for \\\n",
    "                                    key in valid_dict.keys()}\n",
    "                    val_loss_sums = sess.run(valid_sums_merged,\n",
    "                                             feed_dict=feed_dict_vl)\n",
    "                    summary_writer.add_summary(summary=val_loss_sums, \n",
    "                                               global_step=global_step.eval())\n",
    "\n",
    "                    dur_time = time.time() - start_time\n",
    "                    dur_time = ghelp.get_time_format(time_in_sec=dur_time)\n",
    "                    dur_time = ghelp.time_tuple_to_str(time_tuple=dur_time)\n",
    "                    logger.info('step {} '.format(data_train.num_iter) +\n",
    "                                '(epoch ' + \n",
    "                                '{}):'.format(data_train.completed_epochs + 1) +\n",
    "                                ' mean validation losses:\\n    ' +\n",
    "                                ', '.join(['{}: {:.4f}'.format(it[0], it[1]) \\\n",
    "                                           for it in valid_loss_dict.items()]) +\n",
    "                                ' (ET: {}).'.format(dur_time))\n",
    "                    # reset timer to measure the displayed training steps:\n",
    "                    start_time = time.time()\n",
    "\n",
    "                ################################################################\n",
    "\n",
    "                if data_train.num_iter % save_step == 0:\n",
    "                    # save checkpoint files to disk:\n",
    "                    save_path = saver.save(sess, log_dir + 'tfmodel',\n",
    "                                           global_step=global_step.eval())\n",
    "                    s = sess.run(merge_train_summaries, feed_dict=feed_dict_tr)\n",
    "                    # adds a Summary protocol buffer to the event file \n",
    "                    # (global_step: Number. Optional global step value to record\n",
    "                    # with the summary. Each stepp i is assigned to the \n",
    "                    # corresponding summary parameter.)\n",
    "                    summary_writer.add_summary(summary=s, \n",
    "                                               global_step=global_step.eval())\n",
    "                    dur_time = time.time() - start_time\n",
    "                    dur_time = ghelp.get_time_format(time_in_sec=dur_time)\n",
    "                    dur_time = ghelp.time_tuple_to_str(time_tuple=dur_time)\n",
    "                    logger.info('Saved data (step ' + \n",
    "                                '{}):\\n'.format(data_train.num_iter) +\n",
    "                                '    Checkpoint file written to: ' + \n",
    "                                '{} '.format(save_path) +\n",
    "                                '(ET: {}).'.format(dur_time))\n",
    "                    # reset timer to measure the displayed training steps:\n",
    "                    start_time = time.time()\n",
    "\n",
    "            # end while loop when there are no elements left to dequeue:\n",
    "            except IndexError:\n",
    "                end_total_time = time.time() - start_total_time\n",
    "                end_total_time = ghelp.get_time_format(end_total_time)\n",
    "                end_total_time = ghelp.time_tuple_to_str(time_tuple=end_total_time)\n",
    "                logger.info('Training done... total training time: ' + \n",
    "                            '{}.'.format(end_total_time))\n",
    "                break\n",
    "\n",
    "    logger.info('Finished training.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-31T14:05:02.277830Z",
     "start_time": "2017-08-31T14:05:00.568554Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_network_iiw(log_dir, data_dir, path_inference_graph, checkpoint_path,\n",
    "                      restore_scope, image_shape, initial_learning_rate, \n",
    "                      loss_opt, lambda_loss, batch_size, num_epochs, \n",
    "                      display_step, save_step, nodes_name_dict, logger, \n",
    "                      dataset='iiw', is_sample=False, norm=True, \n",
    "                      plot_inference_graph=False):\n",
    "    \"\"\"\n",
    "    :param log_dir: path to directory for saving summary/log files\n",
    "    :type log_dir: str\n",
    "    :param data_dir: path to directory with training and validation data (in \n",
    "        this directory the training and validation csv files have to be\n",
    "        located.)\n",
    "    :type data_dir: str\n",
    "    :param path_inference_graph: path to inference graph (graph without \n",
    "        'training' ops)\n",
    "    :type path_inference_graph: str\n",
    "    :param checkpoint_path: path to model parameters (checkpoint files\n",
    "        e.g. 'logs/2/tfmodel-5' or None)\n",
    "    :type checkpoint_path: str\n",
    "    :param restore_scope: a scope name which has to be defined to find \n",
    "        parameters to restore (e.g. 'vgg_16'). Usually this is needed to\n",
    "        find a pre-trained (tf slim) model within another model.\n",
    "    :type restore_scope: str\n",
    "    :param image_shape: Shape of images that should be used for training \n",
    "        (shape of cnn input tensor)\n",
    "    :type image_shape: list with len(image_shape)==3\n",
    "    :param initial_learning_rate: hyper-parameters for initial learning rate\n",
    "    :type initial_learning_rate: float\n",
    "    :param loss_opt: loss function used for optimization \n",
    "        ('berhu', 'l2', 'l2_inv', 'l2_avg')\n",
    "    :type loss_opt: str\n",
    "    :param batch_size: nr of data which is put through the network before \n",
    "        updating it, as default use: 16, 32 or 64. \n",
    "        batch_size determines how many data samples are loaded in the memory \n",
    "        (be careful with memory space)\n",
    "    :type batch_size: int\n",
    "    :param num_epochs: nr of times the training process loops through the \n",
    "        complete training data set (how often is the tr set 'seen')\n",
    "        if you have 1000 training examples, and your batch size is 500, then it\n",
    "        will take 2 iterations to complete 1 epoch.\n",
    "    :type num_epochs: int\n",
    "    :param display_step: every display_step'th training iteration information is\n",
    "        printed to stdout and file training.log in log_dir (default: 100)\n",
    "    :type display_step: int\n",
    "    :param save_step: every save_step'th training iteration a summary file is \n",
    "        written to log_dir and checkpoint files are saved\n",
    "    :type save_step: int\n",
    "    :param nodes_name_dict: dictionary that contains name of input, albedo and \n",
    "        shading output in form {'input': '', \n",
    "                                'output_albedo': '',\n",
    "                                'output_shading': ''}\n",
    "    :type nodes_name_dict: dict\n",
    "    :param dataset: which dataset to use for training (default: 'sintel')\n",
    "    :type dataset: str, must be \\elem {'iiw', 'sintel'}\n",
    "    :param is_sample: flag, if True only a smaller sample size is used for \n",
    "        training and validation (default: False).\n",
    "    :type is_sample: boolean\n",
    "    :param norm: flag, if True image pixels are scaled to range [0, 1]\n",
    "        (default: True)\n",
    "    :type norm: boolean\n",
    "    :param plot_inference_graph: flag, True if inference graph should be \n",
    "        plotted (default: False).\n",
    "    :type plot_inference_graph: boolean\n",
    "    \"\"\"\n",
    "    ############################################################################\n",
    "    logger.info('Training on images of shape: {}'.format(image_shape))\n",
    "    logger.info('Training on [0, 1] normalized pixel values: {}'.format(norm))\n",
    "    logger.info('Initial learning rate: {}'.format(initial_learning_rate))\n",
    "    logger.info('Loss function used for optimization: {}'.format(loss_opt))\n",
    "    logger.info('Batch size: {}'.format(batch_size))\n",
    "    logger.info('# epochs: {}'.format(num_epochs))\n",
    "    logger.info(\"Loss 'regularizer' (L_1/2 + lambda * M(W)HDL), \" + \n",
    "                \"lambda = {}\".format(lambda_loss))\n",
    "    logger.info('Write summary and checkpoints to file (in directory ' +\n",
    "                '{}) every '.format(log_dir) +\n",
    "                '{} iterations.'.format(save_step))\n",
    "\n",
    "    # load meta graph (inference graph)\n",
    "    # how to work with restored models:\n",
    "    # https://www.tensorflow.org/programmers_guide/meta_graph\n",
    "    # http://cv-tricks.com/tensorflow-tutorial/save-restore-tensorflow-models-quick-complete-tutorial/\n",
    "    saver_restore = tf.train.import_meta_graph(path_inference_graph, \n",
    "                                               clear_devices=True)\n",
    "\n",
    "    logger.debug('Restored inference graph from\\n' +\n",
    "                 '    {}'.format(path_inference_graph))\n",
    "    variables_to_restore = slim.get_variables_to_restore(include=[restore_scope],\n",
    "                                                         exclude=None)\n",
    "    print(variables_to_restore)\n",
    "    logger.info('# of parameters that can be restored: ' + \n",
    "                '{}.'.format(len(variables_to_restore)))\n",
    "\n",
    "    ############################################################################\n",
    "\n",
    "    # save default graph in variable:\n",
    "    graph = tf.get_default_graph()\n",
    "    if plot_inference_graph:\n",
    "        # plot imported inference graph:\n",
    "        plt_help.show_graph(graph.as_graph_def())\n",
    "\n",
    "    # lets get the input\n",
    "    x = graph.get_tensor_by_name(name=nodes_name_dict['input'])\n",
    "\n",
    "    # bool variable that indicates if we are in training mode (training=True) or\n",
    "    # valid/test mode (training=False) this indicator is important if dropout \n",
    "    # or/and batch normalization is used.\n",
    "    try:\n",
    "        # try importing training node (is needed for models that use batch \n",
    "        # normalization etc.)\n",
    "        try:\n",
    "            training = graph.get_tensor_by_name(name='is_training_1:0')\n",
    "        except KeyError:\n",
    "            training = graph.get_tensor_by_name(name='is_training:0')\n",
    "        logger.debug('Was able to catch is_training node!')\n",
    "    except KeyError:\n",
    "        # elsewise just define a placeholder wich is used as dummy variable\n",
    "        # and won't be used later:\n",
    "        training = tf.placeholder(dtype=tf.bool, name='is_training_1')\n",
    "        logger.debug('Initialized is_training node.')\n",
    "\n",
    "    # define new global step variable:\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "    # get the last global training step if we continue training:\n",
    "    try:\n",
    "        last_global_step = int(checkpoint_path.split('/')[-1].split('-')[-1])\n",
    "    except (ValueError, AttributeError):\n",
    "        last_global_step = 0\n",
    "    \n",
    "    # get graph output nodes:\n",
    "    y_albedo_pred = graph.get_tensor_by_name(name=nodes_name_dict['output_albedo'])\n",
    "    y_shading_pred = graph.get_tensor_by_name(name=nodes_name_dict['output_shading'])\n",
    "    # y_albedo_pred = tf.clip_by_value(t=y_albedo, clip_value_min=0, \n",
    "    #                                  clip_value_max=1, \n",
    "    #                                  name='0_1_clipping_albedo')\n",
    "    # y_shading_pred = tf.clip_by_value(t=y_shading, clip_value_min=0,\n",
    "    #                                   clip_value_max=1, \n",
    "    #                                   name='0_1_clipping_shading')\n",
    "\n",
    "    ############################################################################\n",
    "    ############################################################################\n",
    "\n",
    "    with tf.name_scope('loss'):\n",
    "        if loss_opt not in ('l1_mhdl', 'l1_mwhdl', 'l2_mhdl', \n",
    "                            'l2_mwhdl'):\n",
    "            raise ValueError('{} is not a valid loss '.format(loss_opt) + \n",
    "                             'function. Set parameter loss_opt to one of the ' +\n",
    "                             \"following: ('l1_mhdl', 'l1_mwhdl', \" + \n",
    "                             \"l2_mhdl', 'l2_mwhdl')\")\n",
    "            \n",
    "    \n",
    "        point1 = tf.placeholder(dtype=tf.int32, name='point1')\n",
    "        point2 = tf.placeholder(dtype=tf.int32, name='point2')\n",
    "        # get the human darker labels:\n",
    "        human_labels = tf.placeholder(dtype=tf.int32, name='human_labels')\n",
    "        # list of weights/darker scores:\n",
    "        darker_weights = tf.placeholder(dtype=tf.float32, name='darker_weights')\n",
    "\n",
    "        losses = chlp.iiw_loss_fct(input_image=x,\n",
    "                                   prediction_albedo=y_albedo_pred,\n",
    "                                   prediction_shading=y_shading_pred,\n",
    "                                   albedo_comp_point1=point1,\n",
    "                                   albedo_comp_point2=point2,\n",
    "                                   albedo_comp_human_labels=human_labels,\n",
    "                                   albedo_comp_weights=darker_weights,\n",
    "                                   albedo_comp_delta=0.1,\n",
    "                                   lambda_=lambda_loss)\n",
    "\n",
    "        loss_dict = {'l1_mhdl': losses[0],\n",
    "                     'l1_mwhdl': losses[1],\n",
    "                     'l2_mhdl': losses[2],\n",
    "                     'l2_mwhdl': losses[3],\n",
    "                     'l1': losses[4],\n",
    "                     'l2': losses[5],\n",
    "                     'mhdl': losses[6],\n",
    "                     'mwhdl': losses[7]}\n",
    "        \n",
    "        loss = loss_dict[loss_opt]\n",
    "\n",
    "    logger.debug('Defined training losses.')\n",
    "\n",
    "    ############################################################################\n",
    "\n",
    "    # Use an AdamOptimizer to train the network:\n",
    "    with tf.name_scope('optimization'):\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=initial_learning_rate)\n",
    "        \n",
    "        # use slim optimizaton op only if a batch normalization is in the \n",
    "        # tf graph.\n",
    "        use_slim_train_opt = False\n",
    "        for v in graph.as_graph_def().node:\n",
    "            if 'batchnorm' in v.name.lower():\n",
    "                use_slim_train_opt = True\n",
    "                break\n",
    "        # Many networks utilize modules, like BatchNorm, that require \n",
    "        # performing a series of non-gradient updates during training. \n",
    "        # slim.learning.create_train_op allows a user to pass in a list of \n",
    "        # update_ops to call along with the gradient updates.\n",
    "        #   train_op = slim.learning.create_train_op(total_loss, optimizer,\n",
    "        #                                            update_ops)\n",
    "        # By default, slim.learning.create_train_op includes all update ops \n",
    "        # that are part of the `tf.GraphKeys.UPDATE_OPS` collection.\n",
    "        # Additionally, TF-Slim's slim.batch_norm function adds the moving mean\n",
    "        # and moving variance updates to this collection. Consequently, users \n",
    "        # who want to use slim.batch_norm will not need to take any additional\n",
    "        # steps in order to have the moving mean and moving variance updates be\n",
    "        # computed.\n",
    "        # (see https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/slim/python/slim/learning.py)\n",
    "        if use_slim_train_opt:\n",
    "            opt_step = slim.learning.create_train_op(total_loss=loss, \n",
    "                                                     optimizer=optimizer,\n",
    "                                                     global_step=global_step) \n",
    "            logger.debug('Using slim optimizaton op because a batch ' +\n",
    "                         'normalization is apparently in the tf graph.')\n",
    "        else:\n",
    "            opt_step = optimizer.minimize(loss, global_step=global_step)\n",
    "            logger.debug('Using default tf optimizaton op because there is ' +\n",
    "                         'apparently no batch normalization in the tf ' +\n",
    "                         'graph.')\n",
    "\n",
    "    logger.debug('Defined optimization method.')\n",
    "    \n",
    "    \n",
    "    ############################################################################\n",
    "\n",
    "    # to get every summary defined above we merge them to get one target:\n",
    "    merge_train_summaries = tf.summary.merge_all()\n",
    "    # define a FileWriter op which writes summaries defined above to disk:\n",
    "    summary_writer = tf.summary.FileWriter(log_dir)\n",
    "    # Create a saver for writing training checkpoints.\n",
    "    saver = tf.train.Saver(max_to_keep=None)\n",
    "\n",
    "    ############################################################################\n",
    "\n",
    "    # introduce some validation set specific summaries\n",
    "    # These summaries need to be defined blow the function \n",
    "    # merge_train_summaries = tf.summary.merge_all()\n",
    "    # because the validation set summaries are added to the summary writer at \n",
    "    # different times. If they had been summarized with the training summaries \n",
    "    # they would have to be defined at times where merge_train_summaries are \n",
    "    # added to the summary writer\n",
    "    with tf.name_scope('loss/valid/'):\n",
    "        valid_dict = {key: tf.placeholder(dtype=tf.float32, name=key) for \\\n",
    "                      key in loss_dict.keys()}\n",
    "        valid_summaries = [tf.summary.scalar(name=key, tensor=val) for \\\n",
    "                           key, val in valid_dict.items()]\n",
    "        valid_sums_merged = tf.summary.merge(inputs=valid_summaries,\n",
    "                                             collections=None, name=None)\n",
    "    logger.debug('Defined validation losses.')\n",
    "    logger.debug('Finished building training graph.')\n",
    "    logger.info('Total parameters of network: ' +\n",
    "                '{}'.format(nnhelp.network_params()))\n",
    "\n",
    "    ############################################################################\n",
    "    ############################################################################\n",
    "\n",
    "    # import data:\n",
    "    if is_sample:\n",
    "        sample = 'sample_'\n",
    "    else:\n",
    "        sample = ''\n",
    "    \n",
    "    file = sample + 'data_iiw_train.csv'\n",
    "    df_train = pd.read_csv(filepath_or_buffer=data_dir + file, \n",
    "                           sep=',', header=None,\n",
    "                           names=['img', 'json_label'])\n",
    "    # complete image paths:\n",
    "    df_train = data_dir + df_train\n",
    "\n",
    "    # instantiate a data queue for feeding data in (mini) batches to cnn:\n",
    "    data_train = iq.DataQueue(df=df_train, batch_size=batch_size,\n",
    "                              num_epochs=num_epochs)\n",
    "\n",
    "    logger.debug('Imported {} training data '.format(dataset) + \n",
    "                 '(#: {}) '.format(data_train.df.shape[0]) + \n",
    "                 'from\\n    {}'.format(data_dir + file))\n",
    "    \n",
    "    \n",
    "    file = sample + 'data_iiw_valid.csv'\n",
    "    df_valid = pd.read_csv(filepath_or_buffer=data_dir + file, \n",
    "                           sep=',', header=None,\n",
    "                           names=['img', 'json_label'])\n",
    "    # complete image paths:\n",
    "    df_valid = data_dir + df_valid\n",
    "    # instantiate a data queue for feeding data in (mini) batches to cnn:\n",
    "    data_valid = iq.DataQueue(df=df_valid, batch_size=batch_size,\n",
    "                              num_epochs=num_epochs)\n",
    "    logger.debug('Imported {} validation data '.format(dataset) +\n",
    "                 '(#: {}) '.format(data_valid.df.shape[0]) + \n",
    "                 'from\\n    {}'.format(data_dir + file))\n",
    "\n",
    "    ############################################################################\n",
    "    ############################################################################\n",
    "\n",
    "    logger.info('Start training:')\n",
    "    # Initialization:\n",
    "    # Op that initializes global variables in the graph:\n",
    "    init_global = tf.global_variables_initializer()\n",
    "    # Op that initializes local variables in the graph:\n",
    "    init_local = tf.local_variables_initializer()\n",
    "\n",
    "    config = tf.ConfigProto(device_count = {'GPU': 1},\n",
    "                            intra_op_parallelism_threads=3\n",
    "    #                        allow_soft_placement = True,\n",
    "    #                        log_device_placement=False\n",
    "                           )\n",
    "    with tf.Session(config=config) as sess: \n",
    "        \n",
    "        ########################################################################\n",
    "        \n",
    "        # initialize all variables:\n",
    "        sess.run([init_global, init_local])\n",
    "        \n",
    "        # assign the last true global step to to global step:\n",
    "        sess.run(global_step.assign(last_global_step))\n",
    "        logger.info('Assigned last global training step: ' +\n",
    "                    '{}'.format(global_step.eval()))\n",
    "\n",
    "        if checkpoint_path:\n",
    "            try:\n",
    "                # restore saved model parameters (weights, biases, etc):\n",
    "                saver_restore.restore(sess, checkpoint_path)\n",
    "                logger.info('Restoring parameters from ' +\n",
    "                            '{}'.format(checkpoint_path))\n",
    "            except (tf.errors.InvalidArgumentError, tf.errors.NotFoundError):\n",
    "                # in the worst case parameters are loaded twice.\n",
    "                # restore the parameters:\n",
    "                init_fn = slim.assign_from_checkpoint_fn(checkpoint_path,\n",
    "                                                         variables_to_restore)\n",
    "                init_fn(sess)\n",
    "                logger.info('Restoring parameters from ' +\n",
    "                            '{}'.format(checkpoint_path))\n",
    "        # Adds a Graph to the event file.\n",
    "        # create summary that give output (TensorFlow op that output protocol \n",
    "        # buffers containing 'summarized' data) of the built Tensorflow graph:\n",
    "        summary_writer.add_graph(sess.graph)\n",
    "\n",
    "        # start timer for total training time:\n",
    "        start_total_time = time.time()\n",
    "        # set timer to measure the displayed training steps:\n",
    "        start_time = start_total_time\n",
    "\n",
    "        ########################################################################\n",
    "        \n",
    "        # Training:\n",
    "        # train loop\n",
    "        # train until all data is processed (queue empty),\n",
    "        # number of iterations depends on number of data, number of epochs and \n",
    "        # batch size:\n",
    "        iter_start = data_train.iter_left\n",
    "        logger.info('For training it takes {}\\n'.format(iter_start) +\n",
    "                    '    (= # data / batch_size * epochs) iterations to loop ' +\n",
    "                    'through {} samples of\\n  '.format(data_train.df.shape[0]) +\n",
    "                    '  training data over {} '.format(data_train.num_epochs) +\n",
    "                    'epochs summarized in batches of size ' + \n",
    "                    '{}.\\n'.format(data_train.batch_size) +\n",
    "                    '    So, there are # data / batch_size = ' +\n",
    "                    '{}'.format(int(data_train.df.shape[0]/data_train.batch_size))+\n",
    "                    ' iterations per epoch.')\n",
    "\n",
    "        while data_train.iter_left >= 0:\n",
    "            try:\n",
    "                # take a (mini) batch of the training data:\n",
    "                deq_train = data_train.dequeue()\n",
    "                lst = iq.next_batch_iiw(deq=deq_train,\n",
    "                                        output_shape=image_shape,\n",
    "                                        norm=norm)\n",
    "                (df_whdr, imgs_b, imgs_original_b, \n",
    "                 js_label_b, js_originial_label_b) = lst\n",
    "                fd_tr = {x: imgs_b,\n",
    "                         point1: df_whdr[['batch_nr', 'y1', 'x1']].values,\n",
    "                         point2: df_whdr[['batch_nr', 'y2', 'x2']].values,\n",
    "                         human_labels: df_whdr['darker'].values,\n",
    "                         darker_weights: df_whdr['darker_score'].values,\n",
    "                         training: True}\n",
    "                # run optimization step:\n",
    "                sess.run(opt_step, feed_dict=fd_tr)\n",
    "\n",
    "                ################################################################\n",
    "\n",
    "                # report training set accuracy every display_step-th step:\n",
    "                if (data_train.num_iter) % display_step == 0:\n",
    "                    # console output:\n",
    "                    train_loss_dict = {}\n",
    "                    for key, val in loss_dict.items():\n",
    "                        train_loss_dict[key] = sess.run(val, \n",
    "                                                        feed_dict=fd_tr)\n",
    "\n",
    "                    dur_time = time.time() - start_time\n",
    "                    dur_time = ghelp.get_time_format(time_in_sec=dur_time)\n",
    "                    dur_time = ghelp.time_tuple_to_str(time_tuple=dur_time)\n",
    "                    logger.info('step {}: '.format(data_train.num_iter) +\n",
    "                                'training ({}) loss '.format(loss_opt) +\n",
    "                                '{:.4f}'.format(train_loss_dict[loss_opt]) + \n",
    "                                '\\n    (' +\n",
    "                                ', '.join(['{}: {:.4f}'.format(it[0], it[1]) \\\n",
    "                                           for it in train_loss_dict.items() \\\n",
    "                                           if it[0]!=loss_opt]) +\n",
    "                                ', ET: {}).'.format(dur_time))\n",
    "                    # reset timer to measure the displayed training steps:\n",
    "                    start_time = time.time()\n",
    "\n",
    "                ################################################################\n",
    "\n",
    "                # Validation:\n",
    "                # display validation set accuracy and loss after each completed \n",
    "                # epoch (1 epoch ^= data_train.df.shape[0]/data_train.batch_size\n",
    "                # training steps => steps per epoch)\n",
    "                val_epoch = int(data_train.df.shape[0] / data_train.batch_size)\n",
    "                \n",
    "                if data_train.num_iter % val_epoch == 0:\n",
    "                    # After each training epoch we will use the complete \n",
    "                    # validation data set to calculate the error/accuracy on the\n",
    "                    # validation set:\n",
    "                    # loop through one validation data set epoch:\n",
    "                    # initialize dictionary which contains all validation \n",
    "                    # losses:\n",
    "                    valid_loss_dict = dict.fromkeys(loss_dict, 0)\n",
    "                    valid_steps_per_epoch = int(data_valid.df.shape[0] / \n",
    "                                                data_valid.batch_size)\n",
    "                    for j in range(valid_steps_per_epoch):\n",
    "                        # DISCLAIMER: we do not run the opt_step here (on \n",
    "                        # the validation data set) because we do not want to \n",
    "                        # train our network on the validation set. Important \n",
    "                        # for batch normalization and dropout \n",
    "                        # (training -> False).\n",
    "                        # get validation data set (mini) batch:\n",
    "                        lst = iq.next_batch_iiw(deq=data_valid.dequeue(),\n",
    "                                                output_shape=image_shape,\n",
    "                                                norm=norm)\n",
    "                        (df_whdr_val, imgs_b_val, imgs_original_b_val, \n",
    "                         js_label_b_val, js_originial_label_b_val) = lst\n",
    "\n",
    "                        # calculate the mean loss of this validation batch and \n",
    "                        # sum it with the previous mean batch losses:\n",
    "                        fd_val = {x: imgs_b_val,\n",
    "                                  point1: df_whdr_val[['batch_nr', \n",
    "                                                       'y1', 'x1']].values,\n",
    "                                  point2: df_whdr_val[['batch_nr',\n",
    "                                                       'y2', 'x2']].values,\n",
    "                                  human_labels: df_whdr_val['darker'].values,\n",
    "                                  darker_weights: df_whdr_val['darker_score'].values,\n",
    "                                  training: False}\n",
    "\n",
    "                        for key, val in loss_dict.items():\n",
    "                            # divide each loss loss by the iteration steps \n",
    "                            # (steps_per_epoch) to get the mean val loss:\n",
    "                            mean_val = val / valid_steps_per_epoch\n",
    "                            valid_loss_dict[key] += sess.run(mean_val, \n",
    "                                                             feed_dict=fd_val)\n",
    "                    # adding a mean loss summary op (for tensorboard):\n",
    "                    feed_dict_vl = {valid_dict[key]: valid_loss_dict[key] for \\\n",
    "                                    key in valid_dict.keys()}\n",
    "                    val_loss_sums = sess.run(valid_sums_merged,\n",
    "                                             feed_dict=feed_dict_vl)\n",
    "                    summary_writer.add_summary(summary=val_loss_sums, \n",
    "                                               global_step=global_step.eval())\n",
    "                    dur_time = time.time() - start_time\n",
    "                    dur_time = ghelp.get_time_format(time_in_sec=dur_time)\n",
    "                    dur_time = ghelp.time_tuple_to_str(time_tuple=dur_time)\n",
    "                    logger.info('step {} '.format(data_train.num_iter) +\n",
    "                                '(epoch ' + \n",
    "                                '{}):'.format(data_train.completed_epochs + 1) +\n",
    "                                ' mean validation losses:\\n    ' +\n",
    "                                ', '.join(['{}: {:.4f}'.format(it[0], it[1]) \\\n",
    "                                           for it in valid_loss_dict.items()]) +\n",
    "                                ' (ET: {}).'.format(dur_time))\n",
    "                    # reset timer to measure the displayed training steps:\n",
    "                    start_time = time.time()\n",
    "\n",
    "                ################################################################\n",
    "\n",
    "                if data_train.num_iter % save_step == 0:\n",
    "                    # save checkpoint files to disk:\n",
    "                    save_path = saver.save(sess, log_dir + 'tfmodel',\n",
    "                                           global_step=global_step.eval())\n",
    "                    \n",
    "                    s = sess.run(merge_train_summaries, feed_dict=fd_tr)\n",
    "                    # adds a Summary protocol buffer to the event file \n",
    "                    # (global_step: Number. Optional global step value to record\n",
    "                    # with the summary. Each stepp i is assigned to the \n",
    "                    # corresponding summary parameter.)\n",
    "                    summary_writer.add_summary(summary=s, \n",
    "                                               global_step=global_step.eval())\n",
    "                    dur_time = time.time() - start_time\n",
    "                    dur_time = ghelp.get_time_format(time_in_sec=dur_time)\n",
    "                    dur_time = ghelp.time_tuple_to_str(time_tuple=dur_time)\n",
    "                    logger.info('Saved data (step ' + \n",
    "                                '{}):\\n'.format(data_train.num_iter) +\n",
    "                                '    Checkpoint file written to: ' + \n",
    "                                '{} '.format(save_path) +\n",
    "                                '(ET: {}).'.format(dur_time))\n",
    "                    # reset timer to measure the displayed training steps:\n",
    "                    start_time = time.time()\n",
    "\n",
    "            # end while loop when there are no elements left to dequeue:\n",
    "            except IndexError:\n",
    "                end_total_time = time.time() - start_total_time\n",
    "                end_total_time = ghelp.get_time_format(end_total_time)\n",
    "                end_total_time = ghelp.time_tuple_to_str(time_tuple=end_total_time)\n",
    "                logger.info('Training done... total training time: ' + \n",
    "                            '{}.'.format(end_total_time))\n",
    "\n",
    "                break\n",
    "\n",
    "    logger.info('Finished training.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_network(dataset, **kwargs):\n",
    "    # create logger (write to file and stdout):\n",
    "    logger = ghelp.create_logger(filename=kwargs['log_dir'] + 'training.log')\n",
    "    logger.debug('Python version: \\n    ' + sys.version + \n",
    "                 '\\n    Tensorflow version: ' + tf.__version__)\n",
    "    logger.info('Parameter summary: \\n' + \n",
    "                \"\\n\".join(\"{}: {}\".format(k, v) for k, v in params.items()))\n",
    "    \n",
    "    if dataset=='iiw':\n",
    "        return train_network_iiw(logger=logger, **kwargs)\n",
    "    elif dataset=='sintel':\n",
    "        return train_network_sintel(logger=logger, **kwargs)\n",
    "    else:\n",
    "        raise ValueError(\"Invalid dataset! Enter one of the \" +\n",
    "                         \"following: ('iiw', 'sintel')\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "If not available download resnet_v1_50 ckpt files to ./models/slim/checkpoints\n",
      "Data has apparently already been downloaded and unpacked.\n"
     ]
    }
   ],
   "source": [
    "################################################################################\n",
    "################################################################################\n",
    "# sintel dataset parameters:\n",
    "\n",
    "# # sintel_narihira2015:\n",
    "# # this model is the implementation of the narihira2015 network.\n",
    "# # 2 scales, encoder is basically AlexNet, decoder consists mainly of\n",
    "# # deconvolution functions as up-scaling functions. This model implementation\n",
    "# # does not have any pre-trained weights.\n",
    "# # Narihira et al recommend using multiples of 13 for spatial image input sizes\n",
    "# # (eg: [32 * 13, 32 * 13, 3])\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'deconv_s2out_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'deconv_s2out_shading/BiasAdd:0'}\n",
    "\n",
    "# params = {'log_dir': 'logs/sintel/narihira2015/test/',  # path to summary files\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/narihira2015/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': None, # e.g. 'logs/2/tfmodel-5' or None\n",
    "#           'restore_scope': None,\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 2,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # sintel_slim_vgg16_narihira2015:\n",
    "# # this model uses a pre-trained vgg16 as encoder and a decoder which\n",
    "# # constists of 2 scales (main upscaling function: deconvolution). \n",
    "# # This decoder has basically the proposed structure of narihira2015:\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'scale2/deconv6_s2_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'scale2/deconv6_s2_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download vgg16 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_vgg16_narihira2015/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/vgg16_narihira2015/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/vgg_16.ckpt',\n",
    "#           'restore_scope': 'vgg_16',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # sintel_slim_vgg16_deconv_decoder:\n",
    "# # this model uses a pre-trained vgg16 as encoder and a decoder\n",
    "# # which basically consists of deconvolution functions as upscaling functions\n",
    "# # (it uses only 1 scale):\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'scale2/deconv6_s2_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'scale2/deconv6_s2_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download vgg16 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_vgg16_deconv_decoder/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/vgg16/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/vgg_16.ckpt',\n",
    "#           'restore_scope': 'vgg_16',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 2,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # sintel_slim_resnet_v1_50_deconv_decoder:\n",
    "# # this model uses a pre-trained resnet_50 as encoder and a decoder \n",
    "# # which basically consists of deconvolution functions as upscaling functions\n",
    "# # (it uses only 1 scale):\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'decoder/deconv7_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'decoder/deconv7_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download resnet_v1_50 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_resnet_v1_50_deconv_decoder/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/resnet_v1_50/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/resnet_v1_50.ckpt',\n",
    "#           'restore_scope': 'resnet_v1_50',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # sintel_slim_resnet_v1_50_narihira2015:\n",
    "# # this model uses a pre-trained resnet_50 as encoder and a decoder which\n",
    "# # constists of 2 scales (main upscaling function: deconvolution). \n",
    "# # This decoder has basically the proposed structure of narihira2015:\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'decoder/deconv7_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'decoder/deconv7_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download resnet_v1_50 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_resnet_v1_50_narihira2015/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/resnet_v1_50_narihira2015/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/resnet_v1_50.ckpt',\n",
    "#           'restore_scope': 'resnet_v1_50',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 2,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "# # sintel_slim_resnet_v1_50_narihira2015_reduced:\n",
    "# # this model uses a pre-trained resnet_50 as encoder and a decoder which\n",
    "# # constists of 2 scales (main upscaling function: deconvolution). \n",
    "# # This decoder has basically the proposed structure of narihira2015:\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'decoder/deconv7_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'decoder/deconv7_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download resnet_v1_50 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_resnet_v1_50_narihira2015_reduced/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/resnet_v1_50_narihira2015_reduced/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/resnet_v1_50.ckpt',\n",
    "#           'restore_scope': 'resnet_v1_50',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # sintel_slim_resnet_v1_50_deconv_decoder_reduced:\n",
    "# # this model uses a pre-trained resnet_50 as encoder and a decoder \n",
    "# # which basically consists of deconvolution functions as upscaling functions\n",
    "# # (it uses only 1 scale):\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'decoder/deconv7_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'decoder/deconv7_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download resnet_v1_50 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_resnet_v1_50_deconv_decoder_reduced/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/resnet_v1_50_reduced/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/resnet_v1_50.ckpt',\n",
    "#           'restore_scope': 'resnet_v1_50',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "# # sintel_slim_vgg16_narihira2015_reduced_bn:\n",
    "# # this model uses a pre-trained vgg16 as encoder and a decoder which\n",
    "# # constists of 2 scales (main upscaling function: deconvolution). \n",
    "# # This decoder has basically the proposed structure of narihira2015:\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'scale2/deconv6_s2_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'scale2/deconv6_s2_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download vgg16 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_vgg16_narihira2015_reduced_bn/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/vgg16_narihira2015_reduced_bn/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/vgg_16.ckpt',\n",
    "#           'restore_scope': 'vgg_16',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # sintel_slim_vgg16_deconv_decoder_reduced_bn:\n",
    "# # this model uses a pre-trained vgg16 as encoder and a decoder\n",
    "# # which basically consists of deconvolution functions as upscaling functions\n",
    "# # (it uses only 1 scale):\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'scale2/deconv6_s2_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'scale2/deconv6_s2_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download vgg16 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/sintel/slim_vgg16_deconv_decoder_reduced_bn/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/vgg16_reduced_bn/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/vgg_16.ckpt',\n",
    "#           'restore_scope': 'vgg_16',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1',\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 100,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'sintel',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# sintel_slim_resnet_v1_50_up_projection:\n",
    "# this model uses a pre-trained resnet_50 as encoder and a decoder which\n",
    "# constists of 2 scales (main upscaling function: deconvolution). \n",
    "# This decoder has basically the proposed structure of narihira2015:\n",
    "nodes_name_dict = {'input': 'input:0',\n",
    "                   'output_albedo': 'decoder/conv2_albedo/BatchNorm/cond/Merge:0',\n",
    "                   'output_shading': 'decoder/conv2_shading/BatchNorm/cond/Merge:0'}\n",
    "# download checkpoint files:\n",
    "url = \"http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz\"\n",
    "checkpoints_dir = './models/slim/checkpoints'\n",
    "print('If not available download resnet_v1_50 ckpt files to ' + checkpoints_dir)\n",
    "download.maybe_download_and_extract(url=url, \n",
    "                                    download_dir=checkpoints_dir,\n",
    "                                    print_download_progress=True)\n",
    "params = {'log_dir': 'logs/sintel/sintel_slim_resnet_v1_50_up_projection/test/',\n",
    "          'data_dir': '/media/sdb/udo/data/',\n",
    "          'path_inference_graph': 'models/slim/graphs/resnet_v1_50_up_project_dim_Nonex320x320x3/tfmodel_inference.meta',\n",
    "          'checkpoint_path': 'models/slim/checkpoints/resnet_v1_50.ckpt',\n",
    "          'restore_scope': 'resnet_v1_50',\n",
    "          'image_shape': [320, 320, 3],\n",
    "          'initial_learning_rate': 5e-4,  # hyper param\n",
    "          'loss_opt': 'l1',\n",
    "          'batch_size': 16,  # hyper param\n",
    "          'num_epochs': 100,  # hyper param \n",
    "          'display_step': 20,\n",
    "          'save_step': 100,\n",
    "          'nodes_name_dict': nodes_name_dict,\n",
    "          'dataset': 'sintel',\n",
    "          'is_sample': False,\n",
    "          'norm': True,\n",
    "          'plot_inference_graph': False\n",
    "         }\n",
    "\n",
    "################################################################################\n",
    "################################################################################\n",
    "# iiw dataset parameters:\n",
    "\n",
    "# # iiw_slim_resnet_v1_50_deconv_decoder:\n",
    "# # this model uses a pre-trained resnet_50 as encoder and  a decoder \n",
    "# # which basically consists of deconvolution functions as upscaling functions\n",
    "# # (it uses only 1 scale)\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'decoder/deconv7_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'decoder/deconv7_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download resnet_v1_50 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/iiw/slim_resnet_v1_50_deconv_decoder/norm_l1_10mhdl/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/resnet_v1_50/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/resnet_v1_50.ckpt',\n",
    "#           'restore_scope': 'resnet_v1_50',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1_mhdl',\n",
    "#           'lambda_loss': 1.0,\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 10,  # hyper param \n",
    "#           'display_step': 2,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'iiw',  # in iiw dataset min image height/width = 340 px\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # iiw_slim_vgg16_deconv_decoder:\n",
    "# # this model uses a pre-trained vgg16 as encoder and a decoder\n",
    "# # which basically consists of deconvolution functions as upscaling functions\n",
    "# # (it uses only 1 scale):\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'scale2/deconv6_s2_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'scale2/deconv6_s2_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download vgg16 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/iiw/slim_vgg16_deconv_decoder/norm_l1_10mhdl/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/vgg16/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/vgg_16.ckpt',\n",
    "#           'restore_scope': 'vgg_16',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1_mhdl',\n",
    "#           'lambda_loss': 1.0,\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 20,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'iiw',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # iiw_slim_vgg16_narihira2015:\n",
    "# # this model uses a pre-trained vgg16 as encoder and a decoder which\n",
    "# # constists of 2 scales (main upscaling function: deconvolution). \n",
    "# # This decoder has basically the proposed structure of narihira2015:\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'scale2/deconv6_s2_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'scale2/deconv6_s2_shading/BiasAdd:0'}\n",
    "# # download checkpoint files:\n",
    "# url = \"http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz\"\n",
    "# checkpoints_dir = './models/slim/checkpoints'\n",
    "# print('If not available download vgg16 ckpt files to ' + checkpoints_dir)\n",
    "# download.maybe_download_and_extract(url=url, \n",
    "#                                     download_dir=checkpoints_dir,\n",
    "#                                     print_download_progress=True)\n",
    "# params = {'log_dir': 'logs/iiw/slim_vgg16_narihira2015/norm_l1_10mhdl/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/vgg16_narihira2015/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'models/slim/checkpoints/vgg_16.ckpt',\n",
    "#           'restore_scope': 'vgg_16',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1_mhdl',\n",
    "#           'lambda_loss': 1.0,\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 20,  # hyper param \n",
    "#           'display_step': 20,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'iiw',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "# # iiw_slim_resnet_v1_50_narihira2015_reduced:\n",
    "# # this model uses a pre-trained resnet_50 as encoder and a decoder which\n",
    "# # constists of 2 scales (main upscaling function: deconvolution). \n",
    "# # This decoder has basically the proposed structure of narihira2015:\n",
    "# nodes_name_dict = {'input': 'input:0',\n",
    "#                    'output_albedo': 'decoder/deconv7_albedo/BiasAdd:0',\n",
    "#                    'output_shading': 'decoder/deconv7_shading/BiasAdd:0'}\n",
    "# params = {'log_dir': 'logs/iiw/slim_resnet_v1_50_narihira2015_reduced/norm_l1_10mhdl/test/',\n",
    "#           'data_dir': '/media/sdb/udo/data/',\n",
    "#           'path_inference_graph': 'models/slim/graphs/resnet_v1_50_narihira2015_reduced/tfmodel_inference.meta',\n",
    "#           'checkpoint_path': 'logs/sintel/slim_resnet_v1_50_narihira2015_reduced/2/tfmodel-4500',\n",
    "#           'restore_scope': '',\n",
    "#           'image_shape': [320, 320, 3],\n",
    "#           'initial_learning_rate': 5e-4,  # hyper param\n",
    "#           'loss_opt': 'l1_mhdl',\n",
    "#           'lambda_loss': 1.0,\n",
    "#           'batch_size': 16,  # hyper param\n",
    "#           'num_epochs': 20,  # hyper param \n",
    "#           'display_step': 2,\n",
    "#           'save_step': 100,\n",
    "#           'nodes_name_dict': nodes_name_dict,\n",
    "#           'dataset': 'iiw',\n",
    "#           'is_sample': False,\n",
    "#           'norm': True,\n",
    "#           'plot_inference_graph': False\n",
    "#          }\n",
    "\n",
    "\n",
    "################################################################################\n",
    "################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2018-01-05 01:51:11 DEBUG]: Python version: \n",
      "    3.5.2 (default, Nov 23 2017, 16:37:01) \n",
      "[GCC 5.4.0 20160609]\n",
      "    Tensorflow version: 1.4.0\n",
      "[2018-01-05 01:51:11 INFO]: Parameter summary: \n",
      "display_step: 20\n",
      "num_epochs: 100\n",
      "dataset: sintel\n",
      "batch_size: 16\n",
      "restore_scope: resnet_v1_50\n",
      "loss_opt: l1\n",
      "path_inference_graph: models/slim/graphs/resnet_v1_50_up_project_dim_Nonex320x320x3/tfmodel_inference.meta\n",
      "is_sample: False\n",
      "nodes_name_dict: {'output_shading': 'decoder/conv2_shading/BatchNorm/cond/Merge:0', 'input': 'input:0', 'output_albedo': 'decoder/conv2_albedo/BatchNorm/cond/Merge:0'}\n",
      "data_dir: /media/sdb/udo/data/\n",
      "initial_learning_rate: 0.0005\n",
      "plot_inference_graph: False\n",
      "log_dir: logs/sintel/sintel_slim_resnet_v1_50_up_projection/test/\n",
      "save_step: 100\n",
      "image_shape: [320, 320, 3]\n",
      "checkpoint_path: models/slim/checkpoints/resnet_v1_50.ckpt\n",
      "norm: True\n",
      "[2018-01-05 01:51:11 INFO]: Training on images of shape: [320, 320, 3]\n",
      "[2018-01-05 01:51:11 INFO]: Training on [0, 1] normalized pixel values: True\n",
      "[2018-01-05 01:51:11 INFO]: Initial learning rate: 0.0005\n",
      "[2018-01-05 01:51:11 INFO]: Loss function used for optimization: l1\n",
      "[2018-01-05 01:51:11 INFO]: Batch size: 16\n",
      "[2018-01-05 01:51:11 INFO]: # epochs: 100\n",
      "[2018-01-05 01:51:11 INFO]: Write summary and checkpoints to file (in directory logs/sintel/sintel_slim_resnet_v1_50_up_projection/test/) every 100 iterations.\n",
      "[2018-01-05 01:51:16 DEBUG]: Restored inference graph from\n",
      "    models/slim/graphs/resnet_v1_50_up_project_dim_Nonex320x320x3/tfmodel_inference.meta\n",
      "[2018-01-05 01:51:16 INFO]: # of parameters that can be restored: 265.\n",
      "[2018-01-05 01:51:17 DEBUG]: Was able to catch is_training node!\n",
      "[2018-01-05 01:51:17 DEBUG]: Defined training losses.\n",
      "[2018-01-05 01:51:21 DEBUG]: Using slim optimizaton op because a batch normalization is apparently in the tf graph.\n",
      "[2018-01-05 01:51:21 DEBUG]: Defined optimization method.\n",
      "[2018-01-05 01:51:23 DEBUG]: Defined validation losses.\n",
      "[2018-01-05 01:51:23 DEBUG]: Finished building training graph.\n",
      "[2018-01-05 01:51:23 INFO]: Total parameters of network: 49015284\n",
      "[2018-01-05 01:51:23 DEBUG]: Imported sintel training data (#: 690) from\n",
      "    /media/sdb/udo/data/data_sintel_shading_train.csv\n",
      "[2018-01-05 01:51:23 DEBUG]: Imported sintel validation data (#: 102) from\n",
      "    /media/sdb/udo/data/data_sintel_shading_valid.csv\n",
      "[2018-01-05 01:51:23 INFO]: Start training:\n",
      "[2018-01-05 01:51:29 INFO]: Assigned last global training step: 0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from models/slim/checkpoints/resnet_v1_50.ckpt\n",
      "INFO:tensorflow:Restoring parameters from models/slim/checkpoints/resnet_v1_50.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2018-01-05 01:51:30 INFO]: Restoring parameters from models/slim/checkpoints/resnet_v1_50.ckpt\n",
      "[2018-01-05 01:51:34 INFO]: For training it takes 4300\n",
      "    (= # data / batch_size * epochs) iterations to loop through 690 samples of\n",
      "    training data over 100 epochs summarized in batches of size 16.\n",
      "    So, there are # data / batch_size = 43 iterations per epoch.\n",
      "[2018-01-05 01:56:31 INFO]: step 20: training (l1) loss 0.7445\n",
      "    (l2_inv: 2.0720, l2_avg: 2.2111, berhu: 0.7561, l2: 2.3501, ET: 04:56 min).\n",
      "[2018-01-05 02:01:21 INFO]: step 40: training (l1) loss 0.7925\n",
      "    (l2_inv: 1.0971, l2_avg: 1.2541, berhu: 0.9695, l2: 1.4112, ET: 04:50 min).\n",
      "[2018-01-05 02:02:41 INFO]: step 43 (epoch 1): mean validation losses:\n",
      "    l2_inv: 248675.5469, l2_avg: 384162.2539, berhu: 625.4374, l2: 519648.9688, l1: 520.9804 (ET: 01:19 min).\n",
      "[2018-01-05 02:06:49 INFO]: step 60: training (l1) loss 0.7205\n",
      "    (l2_inv: 1.0665, l2_avg: 1.1963, berhu: 0.9575, l2: 1.3262, ET: 04:08 min).\n"
     ]
    }
   ],
   "source": [
    "train_network(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-25T14:42:31.406731Z",
     "start_time": "2017-08-25T14:42:31.382090Z"
    },
    "collapsed": true,
    "run_control": {
     "frozen": false,
     "read_only": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# !tensorboard --logdir /logs/3"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "notify_time": "5",
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "48px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}