LSTM MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Пример рекуррентной нейронной сети lstm для классификации MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.ops import rnn, rnn_cell\n",
    "from tensorflow.contrib import rnn\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('/tmp/data/', one_hot = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2243bde5128>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADydJREFUeJzt3X+QVfV5x/HPw7osCQQUjEgQgz8gFWGKdYNtsAmVmmoSg2mKkXYcOmNdk9GOmcl0tExnxMm0ITbROKkxWQMVZ4whk8RKiYk6yJQmWmQxRjBrI3FQFghoSAIYiyz79I89ZDa453sv9557z4Xn/Zpx9t7z3LPnmYufe+7d7/ner7m7AMQzouwGAJSD8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCOqkZh5spHX4KI1u5iGBUP5Pr+tNP2jVPLau8JvZZZLuktQm6evuviz1+FEarYtsfj2HBJCwwddW/dia3/abWZukuyVdLmmGpEVmNqPW3weguer5zD9H0lZ3f8nd35T0TUkLimkLQKPVE/7JkrYPud+Xbfs9ZtZlZj1m1nNIB+s4HIAi1RP+4f6o8Jb5we7e7e6d7t7Zro46DgegSPWEv0/SlCH3z5C0s752ADRLPeHfKGmamZ1lZiMlXS1pdTFtAWi0mof63L3fzG6U9KgGh/pWuPvzhXUGoKHqGud390ckPVJQLwCaiMt7gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCKquVXrNbJuk/ZIOS+p3984imkLztM2Ynqy/8KlTkvUX//KeZH1AnlsbIUvu+5Vfn5Wsr7zjQ8n6hOVPJevR1RX+zJ+5+2sF/B4ATcTbfiCoesPvkh4zs01m1lVEQwCao963/XPdfaeZnSbpcTN7wd3XD31A9qLQJUmj9PY6DwegKHWd+d19Z/Zzj6SHJM0Z5jHd7t7p7p3t6qjncAAKVHP4zWy0mb3jyG1JH5S0pajGADRWPW/7J0p6yMyO/J5vuPsPCukKQMOZe/44bNHG2ni/yOY37XhRnDTljNzaT289Pbnvg5d8LVm/oGMgWR9R4c3jgPL3r2dfSVrz+oRkfcUlf5pb6+/bkdz3eLXB12qf701fQJFhqA8IivADQRF+ICjCDwRF+IGgCD8QVBGz+tBgL93+J8n6C39zd24tNaVWqjytdqDC+eF7vx2XrD994OxkPeXC0duS9Y+P2Zes73w0/5qzNeenpypHwJkfCIrwA0ERfiAowg8ERfiBoAg/EBThB4JinP84sPDSHyXrqbH8StNiK73+3/3rc5L1x//i/GS9nqmzP7ri6mT9o19Nf21418lbc2tr9N6aejqRcOYHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAY528Fc2Yly5+ckB7P/t5v87+eu9J8+i373pWsH/yHdybrP7+9LVmf/tn8JdoO976Y3HfUfz6drLd/LX3sQ4mvMthx8/uS+07+/JPJ+omAMz8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBFVxnN/MVkj6iKQ97j4z2zZe0ipJUyVtk3SVu/+qcW2e4J7enCx3ffxTyXrbrr25tcrz6X+RrO64OX2dQO8HvpysX37vdbm1tt7krvrlten1Cg75pmQ99V0G737g5eS+/cnqiaGaM/99ki47atstkta6+zRJa7P7AI4jFcPv7uslHX1qWSBpZXZ7paQrC+4LQIPV+pl/orvvkqTs52nFtQSgGRp+bb+ZdUnqkqRRyr/OG0Bz1Xrm321mkyQp+7kn74Hu3u3une7e2a6OGg8HoGi1hn+1pMXZ7cWSHi6mHQDNUjH8ZvagpKckvcfM+szsWknLJF1qZi9KujS7D+A4UvEzv7svyinNL7gX5PCN6esAGjkmPeq1xKR4Sd2/mZqsj9x9ILf20m3pOfX3XZO+hmCELFnfdDD/3FbPegInCq7wA4Ii/EBQhB8IivADQRF+ICjCDwTFV3efAN5YMCe3tvcP0v/ElYbyJmzOH6qTpK5x25L12Wvyp87O6Ugfu9Ly4hsTQ3mS9E/XJqYT65nkvhFw5geCIvxAUIQfCIrwA0ERfiAowg8ERfiBoBjnPwHs/MSbubXeD6SX9640LXZA6bH4SvunxvLrmZIrSdd8+8Zk/ex1TyXr0XHmB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOc/wVWaE1/p9b+R+3dtvyS57/Z/nJasM45fH878QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxXF+M1sh6SOS9rj7zGzbUknXSXo1e9gSd3+kUU0i7V2rRubWFk6+IrnvzLE7k/VPTngyWZ/c9vZkPXV++fnnzkvu+bZ1T1f43ahHNWf++yRdNsz2O919dvYfwQeOMxXD7+7rJe1tQi8Amqiez/w3mtlzZrbCzE4prCMATVFr+O+RdI6k2ZJ2Sfpi3gPNrMvMesys55AO1ng4AEWrKfzuvtvdD7v7gKR7JeWuFOnu3e7e6e6d7eqotU8ABasp/GY2acjdj0naUkw7AJqlmqG+ByXNk3SqmfVJulXSPDObLcklbZN0fQN7BNAA5p7+XvYijbXxfpHNb9rxUD9776xkff9nX0/Wn5i1Krd2254Lk/v+5IopyXp/345kPaINvlb7fG96QYQMV/gBQRF+ICjCDwRF+IGgCD8QFOEHguKru6t00pQzcmv92/ua2Elz+cbNyfqY4eZ7DrHwv/KnFD90bnoy6My/uzhZP3MpQ3314MwPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzp95Y0HulxFJki5e+j+5tTUvn5/cd9KVvTX1dCL4zRfOzK0NfDU9nfzQtDeKbgdDcOYHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaDCjPOn5uNL0ic+9/1kvWff1Nxa5HH8tpPHJet/tezR3NoIVfUN02gQzvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EFTFcX4zmyLpfkmnSxqQ1O3ud5nZeEmrJE2VtE3SVe7+q8a1Wp+X/zp/XrkkdY17OFm/88d/nls7Rz+uqafjwpz0Et2X//v6ZL3r5K25tYEK5572n70tWUd9qjnz90v6jLufJ+mPJd1gZjMk3SJprbtPk7Q2uw/gOFEx/O6+y92fyW7vl9QrabKkBZJWZg9bKenKRjUJoHjH9JnfzKZKukDSBkkT3X2XNPgCIem0opsD0DhVh9/Mxkj6jqRPu/u+Y9ivy8x6zKznkA7W0iOABqgq/GbWrsHgP+Du38027zazSVl9kqQ9w+3r7t3u3unune3qKKJnAAWoGH4zM0nLJfW6+x1DSqslLc5uL5aU/nM5gJZSzZTeuZKukbTZzJ7Nti2RtEzSt8zsWkmvSFrYmBaLMXnd/mS9/aa2ZP2m2U/k1pb//YeT+054Pv1x56QnNiXrlbTNmJ5b2zn/1OS+Yz78i2R93az7kvVK03JTw3nTv399ct/ptz2ZrKM+FcPv7j+Ucv+F5xfbDoBm4Qo/ICjCDwRF+IGgCD8QFOEHgiL8QFDmnl4muUhjbbxfZK05OnjgB2cn60/MWpVbG1HhNXRAA8n6bXsuTNYr+ei4/CnFF3Skj11v75X2f8+3b8itnfev25P79vftSNbxVht8rfb53qq+E50zPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExTh/ptIS3n+4+pXc2r9MfC657yE/nKxXnhOf/jdK7V9p392H30jWv/LL9yXrj/3b3GR9wvKnknUUi3F+ABURfiAowg8ERfiBoAg/EBThB4Ii/EBQ1Xxvfwj92/uS9Z9cMSW3du7n65uP3zvv68n6+5+7Kll/de/Ymo997pf6k3XfuDlZnyDG8Y9XnPmBoAg/EBThB4Ii/EBQhB8IivADQRF+IKiK8/nNbIqk+yWdLmlAUre732VmSyVdJ+nV7KFL3P2R1O9q5fn8wIngWObzV3ORT7+kz7j7M2b2DkmbzOzxrHanu3+h1kYBlKdi+N19l6Rd2e39ZtYraXKjGwPQWMf0md/Mpkq6QNKGbNONZvacma0ws1Ny9ukysx4z6zmkg3U1C6A4VYffzMZI+o6kT7v7Pkn3SDpH0mwNvjP44nD7uXu3u3e6e2e7OgpoGUARqgq/mbVrMPgPuPt3Jcndd7v7YXcfkHSvpDmNaxNA0SqG38xM0nJJve5+x5Dtk4Y87GOSthTfHoBGqeav/XMlXSNps5k9m21bImmRmc2W5JK2Sbq+IR0CaIhq/tr/Q2nYL4ZPjukDaG1c4QcERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiq4ld3F3ows1clvTxk06mSXmtaA8emVXtr1b4keqtVkb29293fWc0Dmxr+txzcrMfdO0trIKFVe2vVviR6q1VZvfG2HwiK8ANBlR3+7pKPn9KqvbVqXxK91aqU3kr9zA+gPGWf+QGUpJTwm9llZva/ZrbVzG4po4c8ZrbNzDab2bNm1lNyLyvMbI+ZbRmybbyZPW5mL2Y/h10mraTelprZjuy5e9bMPlRSb1PMbJ2Z9ZrZ82Z2U7a91Ocu0Vcpz1vT3/abWZukn0m6VFKfpI2SFrn7T5vaSA4z2yap091LHxM2s/dLOiDpfnefmW27XdJed1+WvXCe4u43t0hvSyUdKHvl5mxBmUlDV5aWdKWkv1WJz12ir6tUwvNWxpl/jqSt7v6Su78p6ZuSFpTQR8tz9/WS9h61eYGkldntlRr8n6fpcnprCe6+y92fyW7vl3RkZelSn7tEX6UoI/yTJW0fcr9PrbXkt0t6zMw2mVlX2c0MY2K2bPqR5dNPK7mfo1VcubmZjlpZumWeu1pWvC5aGeEfbvWfVhpymOvufyTpckk3ZG9vUZ2qVm5ulmFWlm4Jta54XbQywt8nacqQ+2dI2llCH8Ny953Zzz2SHlLrrT68+8giqdnPPSX38zuttHLzcCtLqwWeu1Za8bqM8G+UNM3MzjKzkZKulrS6hD7ewsxGZ3+IkZmNlvRBtd7qw6slLc5uL5b0cIm9/J5WWbk5b2VplfzctdqK16Vc5JMNZXxJUpukFe7+z01vYhhmdrYGz/bS4CKm3yizNzN7UNI8Dc762i3pVkn/Ielbks6U9Iqkhe7e9D+85fQ2T4NvXX+3cvORz9hN7u1iSf8tabOkgWzzEg1+vi7tuUv0tUglPG9c4QcExRV+QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeC+n9NtlByfRAtkgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2243baf79b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(mnist.train.images[1].reshape(28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_classes = 10\n",
    "batch_size = 128\n",
    "epochs = 10\n",
    "chunk_size = 28\n",
    "n_chunks = 28\n",
    "rnn_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# hight * width\n",
    "x = tf.placeholder('float',[None, n_chunks, chunk_size])\n",
    "y = tf.placeholder('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def NN_model(x):\n",
    "    # imput_data * weights + biases\n",
    "    layer = {'weights':tf.Variable(tf.random_normal([rnn_size, n_classes])),\n",
    "             'biases':tf.Variable(tf.random_normal([n_classes]))}\n",
    "    \n",
    "    # преобразование датасета к правильному размеру(28*batch_size*28)\n",
    "    # lstm от tensorflow ест сэмплы именно в таком виде\n",
    "    x = tf.transpose(x, [1,0,2])\n",
    "    x = tf.reshape(x, [-1, chunk_size])\n",
    "    x = tf.split(x, n_chunks, 0)\n",
    "    \n",
    "    #1-слойная lstm сеть с 128 скрытыми состояниями\n",
    "    lstm_cell = rnn_cell.BasicLSTMCell(rnn_size)\n",
    "    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)\n",
    "    \n",
    "    # output = layer_4 * weights_output + biases_output\n",
    "    output = tf.matmul(outputs[-1], layer['weights']) + layer['biases']\n",
    "    \n",
    "    return output\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow\\python\\util\\tf_should_use.py:107: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "epoch 0 complited out of 10 loss: 195.089896597\n",
      "epoch 1 complited out of 10 loss: 57.0894434415\n",
      "epoch 2 complited out of 10 loss: 39.6309488043\n",
      "epoch 3 complited out of 10 loss: 29.9862134126\n",
      "epoch 4 complited out of 10 loss: 25.3992832983\n",
      "epoch 5 complited out of 10 loss: 21.2556134267\n",
      "epoch 6 complited out of 10 loss: 18.3713042599\n",
      "epoch 7 complited out of 10 loss: 15.506275977\n",
      "epoch 8 complited out of 10 loss: 14.3855932602\n",
      "epoch 9 complited out of 10 loss: 12.4518984832\n",
      "accuracy: 0.9835\n"
     ]
    }
   ],
   "source": [
    "def train_NN(x):\n",
    "    predict = NN_model(x)\n",
    "    # softmax нормализует полученные веса output слоя(сумма всех значений = 1)\n",
    "    # cross_entropy_with_logits высчитывает ошибку для многоклассовой классификации\n",
    "    # ошибка вычисляется как distance(softmax(outputs), Labels)) = -summ(Labels * log(outputs))\n",
    "    # нaм нужно минимизировать эту ошибку\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = predict, labels = y))\n",
    "    \n",
    "    # оптимизатор(улучшенный стохастический градиентный спуск)\n",
    "    optimizer = tf.train.AdamOptimizer().minimize(cost)\n",
    "    \n",
    "    \n",
    "    with tf.Session() as session:\n",
    "        # инициализируем граф со всеми переменными\n",
    "        session.run(tf.initialize_all_variables())\n",
    "        \n",
    "        for epoch in range(epochs):\n",
    "            epoch_loss = 0\n",
    "            for _ in range(int(mnist.train.num_examples/batch_size)):\n",
    "                epoch_x, epoch_y = mnist.train.next_batch(batch_size)\n",
    "                epoch_x = epoch_x.reshape((batch_size, n_chunks, chunk_size))\n",
    "                # на каждой эпохе оптимизируем ошибку для каждой части выборки \n",
    "                _, c = session.run([optimizer, cost], feed_dict={x:epoch_x, y:epoch_y})\n",
    "                epoch_loss += c\n",
    "            print('epoch', epoch, 'complited out of', epochs, 'loss:', epoch_loss)\n",
    "       \n",
    "        # считаем метрику accuracy\n",
    "        correct = tf.equal(tf.argmax(predict, 1), tf.argmax(y, 1))\n",
    "        \n",
    "        accuracy = tf.reduce_mean(tf.cast(correct,'float'))\n",
    "        print( 'accuracy:', accuracy.eval({x:mnist.test.images.reshape((-1, n_chunks, chunk_size)), y:mnist.test.labels}))\n",
    "            \n",
    "train_NN(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}