notebooks/04_Multinomial_Logistic_Regression_solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multinomial Logistic Regression\n",
    "\n",
    "In this script we use multinomial logistic regression to predict the handwritten digits of the MNIST dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=6, micro=3, releaselevel='final', serial=0)\n",
      "1.1.0\n",
      "(4000, 1, 28, 28) (4000,) 28\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAADHCAYAAAA9KdaUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGzVJREFUeJzt3X2sXVWZx/HfQ2kpbbl9uX2hhTIi\nNkaZICVXNGEwDHVGNGOQqCiTmE5qBk3GRBMTJfKHZpJJDBmVP5gYMSJMgg5VfA86g6RY0cRYkVCY\njhRKpYW+cSm9fW9vWfNHj0kv93l6z7pr7/O2v5+EtH3Y7LPOOes5e3G6f3dZSkkAAACYnnO6PQAA\nAIB+xmIKAACgAIspAACAAiymAAAACrCYAgAAKMBiCgAAoACLKQAAgAIspgAAAAqwmAIAAChQtJgy\nsxvM7E9m9qyZ3VbVoIB+RU8AE9ETaAKb7nYyZjZD0jOS/k7STkm/l3RLSul/o/9meHg4rVy5clqP\nB1Rtx44dGh0dtarON52eGBoaSkuWLKlqCECRffv2aWxsrKs9wXUCvaTd68S5BY9xtaRnU0rbJMnM\n/kvSjZLCJlm5cqUeeeSRgocEqrNmzZqqT5ndE0uWLNEdd9xR9TiAafnc5z5X9SmndZ345S9/WfU4\ngGl597vf3dZxJX/Nd5GkHWf8eWerBjQVPQFMRE+gEUoWU97XXpP+ztDMbjWzTWa2aXR0tODhgJ6X\n3RNjY2MdGBbQNVwn0Agli6mdks78i+2LJb30+oNSSnenlEZSSiPDw8MFDwf0vOyeGBoa6tjggC7g\nOoFGKLln6veSVpnZpZJelPRRSf9YyaiA/tSonjCr7D7lYtFYphuwqUuvjacDGtUTaK5pL6ZSSuNm\n9ilJ/y1phqR7UkpPVzYyoM/QE8BE9ASaouSbKaWUHpL0UEVjAfoePQFMRE+gCfgJ6AAAAAVYTAEA\nABRgMQUAAFCg6J4pxKpKOvVSYio3idTA5FJfy51r0fHdqndDNMdfe+21rONz6vRV/xnE60GkqvnZ\nb/Ocb6YAAAAKsJgCAAAowGIKAACgAIspAACAAiymAAAACpDma1PdSaeqHrcKUYqi7v3P+i290a+q\nSs+dc47//2LR8TNmzMg6T875c8cSyU3nRfVTp05l1XPP76F/Oqfuz3GuE9WMpZP4ZgoAAKAAiykA\nAIACLKYAAAAKsJgCAAAowGIKAACgQGPTfHWn7apKRuXIPUdOUkjK31csd3+yKsaCibz5ljsHoxRe\nbv3cc/2Pm9z6zJkz237MrVu3uvUHH3zQrX//+99369dcc41b/9CHPuTWr7jiCrc+Pj5eXM9N/uX2\neRN1K21XZ73uhF8V+0xK9c7PTl4n+GYKAACgAIspAACAAiymAAAACrCYAgAAKFB0A7qZbZd0UNIp\nSeMppZEqBlWlum/8y93GIvd47wa63HO88sorbj266fXIkSNu/cILL3Tr27Ztc+vPPvusWx8dHXXr\nK1asmFS79tpr3WMj3b4xvVs9kTPfovkT3fAd3dzt3Qh+tvqsWbOy6tF5Zs+ePam2e/du99johvLD\nhw+79euuu86t79+/363fddddbv3+++9368ePH3frJ06ccOvee3Xy5En32EjuDcFV67XrRM6N2d26\nTuSex6tXFbLKvXE82jLp1VdfdetPPPGEW3/b297m1ufNm+fWc9Qx96tI8/1tSunlCs4DDAp6ApiI\nnsBA46/5AAAACpQuppKk/zGzP5jZrVUMCOhz9AQwET2BgVf613zXpJReMrOlkh42s/9LKW0884BW\n89wqSRdffHHhwwE9L6snFi9e3I0xAp3EdQIDr+ibqZTSS61f90r6oaSrnWPuTimNpJRGhoeHSx4O\n6Hm5PTE0NNTpIQIdxXUCTTDtb6bMbK6kc1JKB1u//3tJ/1rZyKY3prZq06nnpG0k6ejRo5Wc59Ch\nQ5NqUeoqV5TGiMayb98+t37s2DG3Hn0oLly40K2vXLlyUi0n5dht3eyJKtJ8UWovN2133nnnuXUv\nhTed+p49eybV1q9f7x57wQUXuPUoERS9jlGSdceOHW79z3/+s1tfvny5W49eY08vbes0lV68Tmzc\nuHFSLZoPIyN+8LCKtF3d9Sq2KJPy51X02kTp8agnoutcThoxGnsd15WSq/IyST9sDepcSd9JKf2i\n4HxAv6MngInoCTTCtBdTKaVtkvwfBAE0ED0BTERPoCn40QgAAAAFWEwBAAAUYDEFAABQoJpYWIfV\nubfSwYMH3Xq0z1yUXIrqUQIqSlJ5iYaqUhrRa7Bs2TK3HiUdonh/bmLCS2/lJkZ6MeVXpSr28opS\nMlG9qtTenDlzsh7XS+1J0ne+851JtWjPu2jsEW9/SCneI/LOO+9063fccYdbj/b+u+mmm9x6FXuu\n5e7FNiii5/3CCy9Mqi1YsCDrHHWn86LrQRV78+XK/UzNTf9FcvYz7GRqL8I3UwAAAAVYTAEAABRg\nMQUAAFCAxRQAAEABFlMAAAAF+jLNF6ki5Td37tysenSeKKGUm9LwRCmq6BxHjhxx69HY58+f79aj\nNMapU6cqqeemPQZZFam9qJ6bIIrquSm/qP7AAw+49SeffLLt8eTuVxe9jtu3b3fr73znO9365Zdf\n7tYPHDjg1nft2uXWc5JLmCj3NXruuecm1aKUZVUJydy+zX1cb553K9kcfb6PjY259ejaGqV/e/U6\nwTdTAAAABVhMAQAAFGAxBQAAUIDFFAAAQAEWUwAAAAUGKs1XhSihtHLlSrd+7Ngxtz5v3jy3vnfv\nXrceJRRmzZo1qRbtHxYlQ06cOOHW9+/f79b7YX+7fhhjHapId+Umi3JTflGS9aWXXnLrTz/9tFuP\n5u34+Pik2qWXXuoee8UVV7j1H/3oR2598eLFbj3quSgF9thjj7n1XDkprdx6U3kpsdw9HCO5e/BV\nkdqL6rnve1XJxd27d2cdH6V8+w3fTAEAABRgMQUAAFCAxRQAAEABFlMAAAAFplxMmdk9ZrbXzJ46\no7bIzB42s62tXxfWO0ygd9ATwET0BJqunTTfvZLukvSfZ9Ruk/RISunLZnZb68+fr354vWPBggVZ\nx0fpkCj9FyXrvH3yopRGlAiMxrJs2TK3Hqk7RdRL+0tN4V41qCeiNE+U5nvxxRfd+je+8Q23fvz4\ncbce9cpll102qbZu3Tr32K1bt7r19773vW79+uuvd+sXXHCBW1+40F8feIlDSfr1r3/t1q+99lq3\nvmrVqkm1Hk3t3ase64lnnnnGrUf7Juaoe8/E3M/4Kt77nP1hzyZ379Vob74qkoud7JUpX72U0kZJ\nr7yufKOk+1q/v0/SByoeF9Cz6AlgInoCTTfdpeiylNIuSWr9urS6IQF9iZ4AJqIn0Bi134BuZrea\n2SYz2zQ6Olr3wwE978yeGBsb6/ZwgK7jOoF+N93F1B4zWy5JrV/9H+stKaV0d0ppJKU0Mjw8PM2H\nA3retHpiaGioYwMEOozrBBpjuoupn0ha2/r9Wkk/rmY4QN+iJ4CJ6Ak0xpRpPjP7rqTrJC02s52S\nvijpy5LWm9nHJb0g6cN1DrIOVaUxokRTrigB4X3lHX2bkbu3Uu5rkJuMqDN50k3d7Inc1y56Dzy5\n82TPnj1u/dFHH3XrR44cceuzZ8/Oql999dWTalFiNdqbL9oPLDpP7tyPkoivvPL6e7RPu/POO936\nXXfd1fZjdrPfevE68dOf/tSte5+pOX1yNlV91ua+lznvcVXXviixGu2pmZs27zdTLqZSSrcE/2pN\nxWMB+gI9AUxET6Dp+AnoAAAABVhMAQAAFGAxBQAAUIDFFAAAQIF29uZrlGh/oqqSb0uX+j8EOEo6\nHT58eFLt6NGj7rHR/mG5ey5Vsade3fo9EVi1Kl6PaI5HSdMHHnjArUd7882bN8+tr1271q2vWLHC\nrXtpoVmzZrnHRvUozRf1Sm5CKerRqM+ff/55t46pRXN/w4YNbt3b7zTaY7HuPfiq4o0zN1mYe+17\n+eWX3frJkyfd+rnn+suNfnmNp8I3UwAAAAVYTAEAABRgMQUAAFCAxRQAAEABFlMAAAAF+jLNF6U3\nchIN3RKNJ0oubd++fVJt37597rFRguj888936wsWLHDrVez/VLfodeylMZaoM1GZ2xPPPfecW3/q\nqafc+qJFi9z6Jz/5Sbd+2WWXufWcvbyi1F4096NkUZRcjBJK0eseHR89p5wkVa99pvWqHTt2uPW5\nc+dOqi1ZsqSSx6z7vclJ6OWOJUqaHjhwwK0fPHjQrecm2aNerGJf0U5eD/hmCgAAoACLKQAAgAIs\npgAAAAqwmAIAACjQlzegV6FbN3FGjxvdKHvJJZdMqu3du9c9NrohMKpHN/hF2350Qy/cWDhocreT\n+Pa3v511nje96U1ufdWqVW49moc5N5VH28PMnj3brUfzJxpLVaGM6GZb76b6SO42IU0VbWt0+eWX\nT6qNj4/XOpbjx49nHe9tIybF45wxY0bb54hE8yfqrdz5Fl3jBgXfTAEAABRgMQUAAFCAxRQAAEAB\nFlMAAAAFWEwBAAAUmDLNZ2b3SPoHSXtTSn/dqn1J0j9L+su+Jl9IKT1U1yD7Uc6WN2erDw0NTapF\nqYg9e/a49WibgCgVGB0fbT/jJUkGWb/3RJTae/TRR936888/79aXLVvm1r20lJS/lUqUfPPmW3Rs\nJEpFRfVom5moz6N0XpSMuvLKK916vyT0erEnom2NvO1kNm/e7B4bbaUSJZ6jBGo0r3KvB9Fn7Zw5\ncybV5s+f7x4bJVyj60o0lq1bt7r1SDT3c7aN6WXtfDN1r6QbnPrXUkpXtv7pyYsGUJN7RU8AZ7pX\n9AQabMrFVEppo6RXOjAWoC/QE8BE9ASaruSeqU+Z2ZNmdo+ZLaxsRED/oieAiegJNMJ0F1Nfl3SZ\npCsl7ZL0lehAM7vVzDaZ2abR0dFpPhzQ86bVE2NjY50aH9BpXCfQGNNaTKWU9qSUTqWUXpP0TUlX\nn+XYu1NKIymlkeHh4emOE+hp0+0JL2AADAKuE2iSae3NZ2bLU0q7Wn+8SdJT1Q2prcev5dizyd2b\nK3dPuZx6lBRasWKFW4/25ov2rtq/f79bj1J+3v6BTdPtnoh46Z9obp44ccKtR/uKRemc1atXu/Wc\nfcWkOEHnnefkyZNZjxnVo/NEr8H3vvc9tx49p5GREbd+++23u/Wcvf96LfnX7Z545pln3PqGDRsm\n1RYu9P8GMkq4RenRqB6l/KL/mYp6y0vtSX4iNkrJRqK5Fl0Poh7KTdYOinZ+NMJ3JV0nabGZ7ZT0\nRUnXmdmVkpKk7ZI+UeMYgZ5CTwAT0RNouikXUymlW5zyt2oYC9AX6AlgInoCTcdPQAcAACjAYgoA\nAKAAiykAAIACA3XbfTfSLFWl/HL2J6pifz8pTvNFojTfoUOH3Hq0B1SOnDRTk+XMiejYKP0TpXai\nORvtWxadp4r3ODpH9JyisUepvZ/97Gdu/Re/+IVbj/aFu/nmm916lBrzEpa5r1dusnjQrVmzZlIt\nmidRPbeHcj+zc9+zOve3O3z4cNZYoj1cBx3fTAEAABRgMQUAAFCAxRQAAEABFlMAAAAFWEwBAAAU\nGKg0X52qStDknt+rR4mjKFV39OhRt56bAIn2i5o7d65bj/ZWw/RVkQqqKol01VVXufVoXuUmWaP5\n49Vz59rOnTvd+s9//nO3/pvf/MatR/sQrlu3zq1HvXjs2DG3XkVKq6mpvX5WZ8K1qtR7dJ4ozVun\nXpjjfDMFAABQgMUUAABAARZTAAAABVhMAQAAFGAxBQAAUIA0X5uq2lsp4u3BJUmvvvrqpFq0R15u\noilKb0XnmTFjhlvPTVLk7meIqdWZ5ps5c6Zbf+KJJ7LOk1uP5ptXj4791a9+5dYfeughtz42NubW\n3/72t7v1j3zkI2496tEonRfVvZ6I+pP+magbe7VGcsdSZ2q3qrHkzrecOT6derfxzRQAAEABFlMA\nAAAFWEwBAAAUYDEFAABQYMrFlJmtNLMNZrbFzJ42s0+36ovM7GEz29r6dWH9wwW6j54AJqIn0HTt\npPnGJX02pfS4mV0g6Q9m9rCkf5L0SErpy2Z2m6TbJH2+vqF2RlVJhyhxc/DgQbd+4MABtz4+Pt72\nY0apqChFEe21t3Ch/3k3Z84ct+6NUeq/NEaGvu6J3D27ojRftM/c+vXr3fqaNWvc+vz58916tH+e\nlyLcvXu3e6yXhpWkoaEht/7GN77Rrb/jHe9w6ydPnnTrUU9Ex0efF1Xszdchfd0TVakq9Z1b9z77\n69wfVornZvS5EH2ODMp1YspvplJKu1JKj7d+f1DSFkkXSbpR0n2tw+6T9IG6Bgn0EnoCmIieQNNl\n3TNlZm+QtFrS7yQtSyntkk43kqSlVQ8O6HX0BDARPYEmansxZWbzJD0o6TMpJf8n2/n/3a1mtsnM\nNo2Ojk5njEBPqqInoh8SCfQjrhNoqrYWU2Y2U6cb5P6U0g9a5T1mtrz175dL2uv9tymlu1NKIyml\nkeHh4SrGDHRdVT0R3bsD9BuuE2iydtJ8JulbkraklL56xr/6iaS1rd+vlfTj6ocH9B56ApiInkDT\ntZPmu0bSxyRtNrO/xGi+IOnLktab2cclvSDpw/UMsbOiBEGUtolSO/v27XPrUZond98yz+zZs936\nggUL3HqUzqsqWVTFnks9mvTom57wXqfofYz2h4zmeOS3v/2tW9+2bZtbj9J8URLv3HMnf2xFyaUo\nnXfJJZe49euvv96tHz582K1He/DlvpZR3etFeqI9uanVbshNZuce76kqtZc7l6vagy/nOtFJUy6m\nUkqPSYpmn591BgYYPQFMRE+g6fgJ6AAAAAVYTAEAABRgMQUAAFCAxRQAAECBdtJ8fSPnLv8obbNj\nx462z302XuJIkmbMmJF1nvPPP39SLdo7zztWyk8oRumNquo9mkbqC7mJG+89jtI2b37zm916NN8O\nHTrk1ufNm+fWo3RelHCN9vLy5vnq1avdY9///ve79Wj/sOg5RccfO3bMrR8/ftyt5+7N59XpqzJV\npIajVF23Xuuc55T7eR3N2agnoh+6GqV2c+dzjk6+H3wzBQAAUIDFFAAAQAEWUwAAAAVYTAEAABRg\nMQUAAFCgL9N80R36jz/++KTa5s2b3WOHhobc+qJFi9x67v52uSmCaKf0xYsXT6pFSZKcRNfZ6lXt\n3VRFuogkUntyEptROmfJkiVu/fbbb3frDz/8sFvfuHGjW1+6dKlbj8bznve8x62/613vmlSL+vbA\ngQNuPUoiRSm83Hr0nHL24JP83qoz/dQEVSTf6hzL2eo5+wpWleaLeiVK5+amxHPnba/uzcc3UwAA\nAAVYTAEAABRgMQUAAFCAxRQAAECBvrwBPbJu3bpJtWh7mA9+8INu/aqrrnLrb3nLW9x6dOPreeed\n59Yvuugitx5tP+PdsFrVzZJV3The1TYWvXATYa+r4r3Pfb+ibSBuueUWt7527Vq3Hm0PU8XWS/v2\n7XPr0c2w0Q3iufXcG8qr2MKJbWPak3MTd/T659zwfbbHjM5/zjm9831GNPaob1988UW37oWmznb+\nQZm3vfNOAgAA9CEWUwAAAAVYTAEAABRgMQUAAFBgysWUma00sw1mtsXMnjazT7fqXzKzF83sidY/\n76t/uED30RPARPQEmq6dNN+4pM+mlB43swsk/cHM/rKPxNdSSv9e3/Dy/PGPf2z72CilkVvPPX+U\nXIjSQjmqSs/VncIbgPRGz/VEFSm/KJmWu01RNJejdF6UaOrG1hlVbcmUm5TNOb5H+6fneiLivX65\nn9e56b/cLcByrzc5cufPhRde6NZvvvnmrPNXNW97dP5PvZhKKe2StKv1+4NmtkWSn+8HGoCeACai\nJ9B0WfdMmdkbJK2W9LtW6VNm9qSZ3WNmCyseG9Dz6AlgInoCTdT2YsrM5kl6UNJnUkpjkr4u6TJJ\nV+r0/5F8JfjvbjWzTWa2aXR0tIIhA72hip4YGxvr2HiBunGdQFO1tZgys5k63SD3p5R+IEkppT0p\npVMppdckfVPS1d5/m1K6O6U0klIaGR4ermrcQFdV1RNDQ0OdGzRQI64TaLJ20nwm6VuStqSUvnpG\nffkZh90k6anqhwf0HnoCmIieQNO1k+a7RtLHJG02sydatS9IusXMrpSUJG2X9IlaRliTqhJouSmQ\nXkppVHWeXk1X1KhvesJ7b3L3cMxN50SpwCjNV2dP5Kb2cs9T9z6WfdRbfdMTnqo+33PPHxnE60Td\n5+m2dtJ8j0ny3tmHqh8O0PvoCWAiegJNx09ABwAAKMBiCgAAoACLKQAAgAIspgAAAAq0k+ZrlH5I\nslWVMEEzRfMk2mcumm/R8ZGq9resE/tS4kzden+7MfcjzNn28M0UAABAARZTAAAABVhMAQAAFGAx\nBQAAUIDFFAAAQAHr5J36ZrZP0p9bf1ws6eWOPXj3NOV5Sv33XP8qpbSkmwOgJwZevz1XeqI7mvI8\npf57rm31REcXUxMe2GxTSmmkKw/eQU15nlKznmsdmvL6NeV5Ss16rnVoyuvXlOcpDe5z5a/5AAAA\nCrCYAgAAKNDNxdTdXXzsTmrK85Sa9Vzr0JTXrynPU2rWc61DU16/pjxPaUCfa9fumQIAABgE/DUf\nAABAgY4vpszsBjP7k5k9a2a3dfrx62Rm95jZXjN76ozaIjN72My2tn5d2M0xVsXMVprZBjPbYmZP\nm9mnW/WBfL51oif6f47QD9WiJ/p/njStJzq6mDKzGZL+Q9J7Jb1V0i1m9tZOjqFm90q64XW12yQ9\nklJaJemR1p8Hwbikz6aU3iLpnZL+pfVeDurzrQU9MTBzhH6oCD0xMPOkUT3R6W+mrpb0bEppW0rp\nhKT/knRjh8dQm5TSRkmvvK58o6T7Wr+/T9IHOjqomqSUdqWUHm/9/qCkLZIu0oA+3xrREwMwR+iH\nStETAzBPmtYTnV5MXSRpxxl/3tmqDbJlKaVd0unJJWlpl8dTOTN7g6TVkn6nBjzfitETAzZH6Idi\n9MSAzZMm9ESnF1Pm1IgT9jEzmyfpQUmfSSmNdXs8fYieGCD0QyXoiQHSlJ7o9GJqp6SVZ/z5Ykkv\ndXgMnbbHzJZLUuvXvV0eT2XMbKZON8n9KaUftMoD+3xrQk8MyByhHypDTwzIPGlST3R6MfV7SavM\n7FIzmyXpo5J+0uExdNpPJK1t/X6tpB93cSyVMTOT9C1JW1JKXz3jXw3k860RPTEAc4R+qBQ9MQDz\npGk90fEf2mlm75N0p6QZku5JKf1bRwdQIzP7rqTrdHpX7D2SvijpR5LWS7pE0guSPpxSev3Nh33H\nzP5G0q8lbZb0Wqv8BZ3+O/GBe751oif6f47QD9WiJ/p/njStJ/gJ6AAAAAX4CegAAAAFWEwBAAAU\nYDEFAABQgMUUAABAARZTAAAABVhMAQAAFGAxBQAAUIDFFAAAQIH/BwabYUlgMEhpAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x29a37ddde48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as imgplot\n",
    "import numpy as np\n",
    "\n",
    "# To be compatible with python3\n",
    "try:\n",
    "    import cPickle as pickle\n",
    "except ImportError:\n",
    "    import pickle\n",
    "\n",
    "import gzip\n",
    "import time\n",
    "import tensorflow as tf\n",
    "import sys\n",
    "\n",
    "print(sys.version_info)\n",
    "print(tf.__version__)\n",
    "\n",
    "with gzip.open('../data/mnist_4000.pkl.gz', 'rb') as f:\n",
    "    (X,y) = pickle.load(f, encoding='latin1')\n",
    "PIXELS = len(X[0,0,0,:])\n",
    "print(X.shape, y.shape, PIXELS) #As read\n",
    "fig = plt.figure(figsize=(10,30))\n",
    "for i in range(3):\n",
    "    a=fig.add_subplot(1,3,(i+1))\n",
    "    plt.imshow(X[i,0,:,:], interpolation='none',cmap=plt.get_cmap('gray'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4000, 784)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We need to reshape for the logistic regression\n",
    "X = X.reshape([4000, 784])\n",
    "np.shape(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class label\n",
      "[5 0 4 1 9]\n",
      "class label in OneHot encodig\n",
      "[[ 0.  0.  0.  0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]]\n"
     ]
    }
   ],
   "source": [
    "# Taken from http://stackoverflow.com/questions/29831489/numpy-1-hot-array\n",
    "def convertToOneHot(vector, num_classes=None):\n",
    "    result = np.zeros((len(vector), num_classes), dtype='float32')\n",
    "    result[np.arange(len(vector)), vector] = 1\n",
    "    return result\n",
    "print(\"class label\")\n",
    "print(y[0:5])\n",
    "print(\"class label in OneHot encodig\")\n",
    "print(convertToOneHot(y[0:5], 10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construction of the graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(1)\n",
    "#Note that we usually do not like to specify the batchsize. Choosing it `None` will leave it open\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784], name='x_data')\n",
    "y_true = tf.placeholder(tf.float32, shape=[None, 10], name='y_data')\n",
    "\n",
    "# We start with random weights a\n",
    "w = tf.Variable(tf.random_normal([784, 10], stddev=0.01))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "#<-------------------------- Your code here ---------------\n",
    "# Your code here, do a matrix multiplication between x,w and an addtion of b\n",
    "z = tf.matmul(x,w)+b\n",
    "# End of your code\n",
    "\n",
    "prob = tf.nn.softmax(z)\n",
    "init_op = tf.global_variables_initializer() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Store the graph and visualize it in tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.summary.writer.writer.FileWriter at 0x29a37ddd048>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.summary.FileWriter(\"tmp/Multinomial_Logistic_Regression/\", tf.get_default_graph())#<--- Where to store"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Doing a forward pass of the untrained network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true label =  5\n",
      "\n",
      "probability for each class =  [ 0.11525529  0.13671385  0.08958027  0.07389172  0.11342652  0.10523579\n",
      "  0.10331967  0.07971239  0.07903792  0.1038266 ]\n",
      "\n",
      "pred label =  1\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init_op)\n",
    "    res_val = sess.run(prob, feed_dict={x:X[0:10]})\n",
    "print(\"true label = \",y[0])\n",
    "print()\n",
    "print(\"probability for each class = \",res_val[0])\n",
    "print()\n",
    "print(\"pred label = \",np.argmax(res_val[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.29376\n",
      "0.115159\n",
      "0.102442\n",
      "0.0914249\n",
      "0.0405277\n",
      "0.0604342\n",
      "0.0259042\n",
      "0.0302623\n",
      "0.0484401\n",
      "0.0212244\n",
      "Loss for the validation set\n",
      "0.525234\n"
     ]
    }
   ],
   "source": [
    "loss = tf.reduce_mean(-tf.reduce_sum(y_true * tf.log(prob), reduction_indices=[1]))\n",
    "\n",
    "#train_op = tf.train.GradientDescentOptimizer(0.01).minimize(loss)\n",
    "train_op = tf.train.AdagradOptimizer(0.1).minimize(loss)\n",
    "init_op = tf.global_variables_initializer() \n",
    "with tf.Session() as sess:\n",
    "    sess.run(init_op)\n",
    "    for i in range(1000):\n",
    "        idx = np.random.permutation(2400)[0:128] #Easy minibatch of size 128\n",
    "        #res, out_val, _ = sess.run((loss, prob, train_op),feed_dict={x:X[idx], y_true:convertToOneHot(y[idx], 10)})\n",
    "        loss_, out_val, _ = sess.run((loss, prob, train_op),feed_dict={x:X[idx], y_true:convertToOneHot(y[idx], 10)})\n",
    "        if (i % 100 == 0):\n",
    "            print(loss_)\n",
    "    \n",
    "    # Get the loss for the validation results (from 2400:3000)\n",
    "    print('Loss for the validation set')\n",
    "    #<-------------------------- Your code here ---------------\n",
    "    loss_val = sess.run((loss), feed_dict={x:X[2400:3000], y_true:convertToOneHot(y[2400:3000], 10)})\n",
    "    print(loss_val)\n",
    "    # Get the results for the validation set\n",
    "    res_val = sess.run((prob), feed_dict={x:X[2400:3000]})\n",
    "    #<-------------------------  End of your code here --------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.885\n",
      "probs = [ 0.54000002  0.          0.33000001  0.          0.          0.05        0.07\n",
      "  0.01        0.          0.        ]\n",
      "predicted label = 0\n",
      "true label = 0\n"
     ]
    }
   ],
   "source": [
    "# estimate the preformance on the validation set\n",
    "# Your code here\n",
    "print(\"Accuracy =\",np.mean(np.argmax(res_val, axis = 1) == y[2400:3000]))\n",
    "import random \n",
    "rmd=random.randint(2400,3000)\n",
    "print(\"probs =\",np.round(res_val[rmd-2400],2))\n",
    "print(\"predicted label =\",np.argmax(res_val[rmd-2400]))\n",
    "print(\"true label =\",np.round(y[rmd],2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}