Reading_Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.date(2017, 3, 2)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import date\n",
    "date.today()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "author = \"kyubyong. https://github.com/Kyubyong/tensorflow-exercises\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.0'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.12.0'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "NOTE on notation\n",
    "\n",
    "    _x, _y, _z, _X, _Y, _Z, ...: NumPy arrays\n",
    "    x, y, z, X, Y, Z, ...: Tensors\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Placeholder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Make data and save to npz.\n",
    "_x = np.zeros((100, 10), np.int32)\n",
    "for i in range(100):\n",
    "    _x[i] = np.random.permutation(10)\n",
    "_x, _y = _x[:, :-1], _x[:, -1]\n",
    "\n",
    "import os\n",
    "if not os.path.exists('example'): os.mkdir('example')\n",
    "np.savez('example/example.npz', _x=_x, _y=_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_hat = [8 9 3 9 1 3 5 9 2 2 2 3 1 1 9 6 8 4 9 0 4 8 2 3 7 2 2 5 8 8]\n",
      "true y = [8 9 3 9 1 3 5 9 2 2 2 3 1 1 9 6 8 4 9 0 4 8 2 3 7 2 2 5 8 8]\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "data = np.load('example/example.npz')\n",
    "_x, _y = data[\"_x\"], data[\"_y\"]\n",
    "\n",
    "#Q1. Make a placeholder for x such that it should be of dtype=int32, shape=(None, 9).\n",
    "# Inputs and targets\n",
    "x_pl = ...\n",
    "y_hat = 45 - tf.reduce_sum(x_pl, axis=1) # We find a digit x_pl doesn't contain.\n",
    "\n",
    "# Session\n",
    "with tf.Session() as sess:\n",
    "    _y_hat = sess.run(y_hat, {x_pl: _x})\n",
    "    print(\"y_hat =\", _y_hat[:30])\n",
    "    print(\"true y =\", _y[:30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## TFRecord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 == 8; 9 == 9; 3 == 3; 9 == 9; 1 == 1; 3 == 3; 5 == 5; 9 == 9; 2 == 2; 2 == 2; 2 == 2; 3 == 3; 1 == 1; 1 == 1; 9 == 9; 6 == 6; 8 == 8; 4 == 4; 9 == 9; 0 == 0; 4 == 4; 8 == 8; 2 == 2; 3 == 3; 7 == 7; 2 == 2; 2 == 2; 5 == 5; 8 == 8; 8 == 8; 0 == 0; 3 == 3; 3 == 3; 9 == 9; 1 == 1; 5 == 5; 1 == 1; 3 == 3; 3 == 3; 7 == 7; 9 == 9; 2 == 2; 4 == 4; 7 == 7; 5 == 5; 4 == 4; 2 == 2; 4 == 4; 1 == 1; 0 == 0; 3 == 3; 5 == 5; 4 == 4; 8 == 8; 7 == 7; 6 == 6; 2 == 2; 1 == 1; 2 == 2; 2 == 2; 4 == 4; 1 == 1; 1 == 1; 6 == 6; 0 == 0; 4 == 4; 5 == 5; 1 == 1; 2 == 2; 1 == 1; 0 == 0; 8 == 8; 0 == 0; 4 == 4; 5 == 5; 6 == 6; 6 == 6; 7 == 7; 5 == 5; 0 == 0; 0 == 0; 6 == 6; 9 == 9; 9 == 9; 9 == 9; 3 == 3; 9 == 9; 4 == 4; 8 == 8; 7 == 7; 9 == 9; 0 == 0; 3 == 3; 7 == 7; 4 == 4; 1 == 1; 7 == 7; 4 == 4; 9 == 9; 8 == 8; Done training -- epoch limit reached\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Load data\n",
    "data = np.load('example/example.npz')\n",
    "_x, _y = data[\"_x\"], data[\"_y\"]\n",
    "\n",
    "# Serialize\n",
    "with tf.python_io.TFRecordWriter(\"example/tfrecord\") as fout:\n",
    "    for _xx, _yy in zip(_x, _y):\n",
    "        ex = tf.train.Example()\n",
    "        \n",
    "        # Q2. Add each value to ex.\n",
    "        ex.features.feature['x']....\n",
    "        ex.features.feature['y']....\n",
    "        fout.write(ex.SerializeToString())\n",
    "\n",
    "def read_and_decode_single_example(fname):\n",
    "    # Create a string queue\n",
    "    fname_q = tf.train.string_input_producer([fname], num_epochs=1, shuffle=True)\n",
    "    \n",
    "    # Q3. Create a TFRecordReader\n",
    "    reader = ...\n",
    "    \n",
    "    # Read the string queue\n",
    "    _, serialized_example = reader.read(fname_q)\n",
    "    \n",
    "    # Q4. Describe parsing syntax\n",
    "    features = tf.parse_single_example(\n",
    "        serialized_example,\n",
    "        features={...\n",
    "                  ...}\n",
    "        )\n",
    "    # Output\n",
    "    x = features['x']\n",
    "    y = features['y']\n",
    "    \n",
    "    return x, y\n",
    "\n",
    "# Ops\n",
    "x, y = read_and_decode_single_example('example/tfrecord')\n",
    "y_hat = 45 - tf.reduce_sum(x)\n",
    "\n",
    "# Session\n",
    "with tf.Session() as sess:\n",
    "    #Q5. Initialize local variables\n",
    "    sess.run(...)\n",
    "    \n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(sess=sess, coord=coord)\n",
    "    \n",
    "    try:\n",
    "        while not coord.should_stop():\n",
    "            _y, _y_hat = sess.run([y, y_hat])\n",
    "            print(_y[0],\"==\", _y_hat, end=\"; \")\n",
    "    \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        print('Done training -- epoch limit reached')\n",
    "    finally:\n",
    "        # When done, ask the threads to stop.\n",
    "        coord.request_stop()\n",
    "    \n",
    "    # Wait for threads to finish.\n",
    "    coord.join(threads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Queue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5 9 3 1 2 9 7 7 8 6] == [5 9 3 1 2 9 7 7 8 6]\n",
      "[5 5 3 7 1 6 6 5 8 4] == [5 5 3 7 1 6 6 5 8 4]\n",
      "[3 1 5 6 6 4 4 9 3 8] == [3 1 5 6 6 4 4 9 3 8]\n",
      "[9 3 7 1 8 0 2 4 0 8] == [9 3 7 1 8 0 2 4 0 8]\n",
      "[0 8 8 0 1 4 0 2 3 2] == [0 8 8 0 1 4 0 2 3 2]\n",
      "[4 4 2 3 4 3 2 9 4 1] == [4 4 2 3 4 3 2 9 4 1]\n",
      "[9 3 5 9 4 7 1 2 8 2] == [9 3 5 9 4 7 1 2 8 2]\n",
      "[8 2 7 0 1 0 1 1 7 3] == [8 2 7 0 1 0 1 1 7 3]\n",
      "[6 4 0 0 9 9 2 1 2 5] == [6 4 0 0 9 9 2 1 2 5]\n",
      "[9 9 5 9 4 7 9 1 2 3] == [9 9 5 9 4 7 9 1 2 3]\n",
      "[3 4 1 3 2 9 9 7 9 0] == [3 4 1 3 2 9 9 7 9 0]\n",
      "[1 8 5 2 2 1 3 9 4 4] == [1 8 5 2 2 1 3 9 4 4]\n",
      "[0 4 8 4 4 7 6 2 0 8] == [0 4 8 4 4 7 6 2 0 8]\n",
      "[1 1 7 8 0 4 9 7 8 9] == [1 1 7 8 0 4 9 7 8 9]\n",
      "[0 4 1 5 7 1 6 9 2 2] == [0 4 1 5 7 1 6 9 2 2]\n",
      "[5 7 3 2 1 5 3 8 9 8] == [5 7 3 2 1 5 3 8 9 8]\n",
      "[9 5 4 5 2 6 6 0 2 1] == [9 5 4 5 2 6 6 0 2 1]\n",
      "[2 9 0 4 3 1 1 4 7 6] == [2 9 0 4 3 1 1 4 7 6]\n",
      "[4 2 2 5 1 9 3 7 6 0] == [4 2 2 5 1 9 3 7 6 0]\n",
      "[9 3 3 3 9 5 8 3 8 0] == [9 3 3 3 9 5 8 3 8 0]\n",
      "Done training -- epoch limit reached\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Load data\n",
    "data = np.load('example/example.npz')\n",
    "_x, _y = data[\"_x\"], data[\"_y\"]\n",
    "\n",
    "# Hyperparams\n",
    "batch_size = 10 # We will feed mini-batches of size 10.\n",
    "num_epochs = 2 # We will feed data for two epochs.\n",
    "\n",
    "# Convert to tensors\n",
    "x = tf.convert_to_tensor(_x)\n",
    "y = tf.convert_to_tensor(_y)\n",
    "\n",
    "# Q6. Make slice queues\n",
    "x_q, y_q = ...\n",
    "\n",
    "# Batching\n",
    "x_batch, y_batch = tf.train.batch([x_q, y_q], batch_size=batch_size)\n",
    "\n",
    "# Targets\n",
    "y_hat = 45 - tf.reduce_sum(x_batch, axis=1)\n",
    "\n",
    "# Session\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.local_variables_initializer())\n",
    "    \n",
    "    # Q7. Make a train.Coordinator and threads.\n",
    "    coord = ...\n",
    "    threads = ...\n",
    "    \n",
    "    try:\n",
    "        while not coord.should_stop():\n",
    "            _y_hat, _y_batch = sess.run([y_hat, y_batch])\n",
    "            print(_y_hat, \"==\", _y_batch)\n",
    "    \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        print('Done training -- epoch limit reached')\n",
    "    finally:\n",
    "        # When done, ask the threads to stop.\n",
    "        coord.request_stop()\n",
    "    \n",
    "    # Wait for threads to finish.\n",
    "    coord.join(threads)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Read csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 5 6 1 8 4 9 9 1 2] == [3 5 6 1 8 4 9 9 1 2]\n",
      "[9 7 0 9 2 5 1 3 8 0] == [9 7 0 9 2 5 1 3 8 0]\n",
      "[8 4 0 2 1 2 3 7 1 1] == [8 4 0 2 1 2 3 7 1 1]\n",
      "[4 2 5 9 3 9 1 1 2 8] == [4 2 5 9 3 9 1 1 2 8]\n",
      "[9 0 1 1 9 9 5 4 6 0] == [9 0 1 1 9 9 5 4 6 0]\n",
      "[6 5 8 5 3 8 8 4 7 2] == [6 5 8 5 3 8 8 4 7 2]\n",
      "[8 2 0 0 9 3 4 3 4 5] == [8 2 0 0 9 3 4 3 4 5]\n",
      "[4 5 2 6 6 5 7 2 1 6] == [4 5 2 6 6 5 7 2 1 6]\n",
      "[3 7 4 8 1 1 9 1 5 7] == [3 7 4 8 1 1 9 1 5 7]\n",
      "[2 0 2 4 2 1 9 2 7 8] == [2 0 2 4 2 1 9 2 7 8]\n",
      "[3 7 7 7 2 0 9 4 9 3] == [3 7 7 7 2 0 9 4 9 3]\n",
      "[4 1 5 8 3 1 4 9 8 2] == [4 1 5 8 3 1 4 9 8 2]\n",
      "[6 5 4 7 0 3 2 5 6 8] == [6 5 4 7 0 3 2 5 6 8]\n",
      "[9 8 8 0 4 3 2 3 2 6] == [9 8 8 0 4 3 2 3 2 6]\n",
      "[2 8 0 4 4 9 2 3 7 6] == [2 8 0 4 4 9 2 3 7 6]\n",
      "[4 1 9 4 3 1 7 4 9 0] == [4 1 9 4 3 1 7 4 9 0]\n",
      "[2 3 1 7 9 9 9 0 2 8] == [2 3 1 7 9 9 9 0 2 8]\n",
      "[4 1 8 2 2 1 5 2 1 2] == [4 1 8 2 2 1 5 2 1 2]\n",
      "[9 4 0 5 4 7 9 4 0 7] == [9 4 0 5 4 7 9 4 0 7]\n",
      "[6 0 6 5 0 3 3 5 9 7] == [6 0 6 5 0 3 3 5 9 7]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Load data\n",
    "data = np.load('example/example.npz')\n",
    "_x, _y = data[\"_x\"], data[\"_y\"]\n",
    "_x = np.concatenate((_x, np.expand_dims(_y, axis=1)), 1)\n",
    "\n",
    "# Write to a csv file\n",
    "_x_str = np.array_str(_x)\n",
    "_x_str = re.sub(\"[\\[\\]]\", \"\", _x_str)\n",
    "_x_str = re.sub(\"(?m)^ +\", \"\", _x_str)\n",
    "_x_str = re.sub(\"[ ]+\", \",\", _x_str)\n",
    "with open('example/example.csv', 'w') as fout:\n",
    "    fout.write(_x_str)\n",
    "    \n",
    "# Hyperparams\n",
    "batch_size = 10\n",
    "\n",
    "# Create a string queue\n",
    "fname_q = tf.train.string_input_producer([\"example/example.csv\"])\n",
    "\n",
    "# Q8. Create a TextLineReader\n",
    "reader = ...\n",
    "\n",
    "# Read the string queue\n",
    "_, value = reader.read(fname_q)\n",
    "\n",
    "# Q9. Decode value\n",
    "record_defaults = [[0]]*10\n",
    "col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = tf.decode_csv(\n",
    "    ...)\n",
    "x = tf.stack([col1, col2, col3, col4, col5, col6, col7, col8, col9])\n",
    "y = col10\n",
    "\n",
    "# Batching\n",
    "x_batch, y_batch = tf.train.shuffle_batch(\n",
    "      [x, y], batch_size=batch_size, capacity=200, min_after_dequeue=100)\n",
    "\n",
    "# Ops\n",
    "y_hat = 45 - tf.reduce_sum(x_batch, axis=1)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(coord=coord)\n",
    "\n",
    "    for i in range(num_epochs*10):\n",
    "        _y_hat, _y_batch = sess.run([y_hat, y_batch])\n",
    "        print(_y_hat, \"==\", _y_batch)\n",
    "\n",
    "    coord.request_stop()\n",
    "    coord.join(threads)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Read image files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 samples have been seen\n",
      "20 samples have been seen\n",
      "30 samples have been seen\n",
      "40 samples have been seen\n",
      "50 samples have been seen\n",
      "60 samples have been seen\n",
      "70 samples have been seen\n",
      "80 samples have been seen\n",
      "90 samples have been seen\n",
      "100 samples have been seen\n",
      "Done training -- epoch limit reached\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Hyperparams\n",
    "batch_size = 10\n",
    "num_epochs = 1\n",
    "\n",
    "# Make fake images and save\n",
    "for i in range(100):\n",
    "    _x = np.random.randint(0, 256, size=(10, 10, 4))\n",
    "    plt.imsave(\"example/image_{}.jpg\".format(i), _x)\n",
    "\n",
    "# Import jpg files\n",
    "images = tf.train.match_filenames_once('example/*.jpg')\n",
    "\n",
    "# Create a string queue\n",
    "fname_q = tf.train.string_input_producer(images, num_epochs=num_epochs, shuffle=True)\n",
    "\n",
    "# Q10. Create a WholeFileReader\n",
    "reader = ...\n",
    "\n",
    "# Read the string queue\n",
    "_, value = reader.read(fname_q)\n",
    "\n",
    "# Q11. Decode value\n",
    "img = ...\n",
    "\n",
    "# Batching\n",
    "img_batch = tf.train.batch([img], shapes=([10, 10, 4]), batch_size=batch_size)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()])\n",
    "\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(coord=coord)\n",
    "    \n",
    "    num_samples = 0\n",
    "    try:\n",
    "        while not coord.should_stop():\n",
    "            sess.run(img_batch)\n",
    "            num_samples += batch_size\n",
    "            print(num_samples, \"samples have been seen\")\n",
    "\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        print('Done training -- epoch limit reached')\n",
    "    finally:\n",
    "        coord.request_stop()\n",
    "\n",
    "    coord.join(threads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}