LOGISTIC_MULTICLASS.py.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ASSIGNMENT 1(B)\n",
    "SAURAV RAI\n",
    "In this code, we show how 10 logistic regression learners can be used for \n",
    "    digit classification.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.io import loadmat\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "''' Load the data'''\n",
    "X = loadmat('TrainImages.mat')\n",
    "X = X['trainData']\n",
    "y = loadmat('TrainLabels.mat')\n",
    "y = y['trainLabels']\n",
    "X_test = loadmat('TestImages.mat')\n",
    "X_test = X_test['testData']\n",
    "y_test = loadmat('TestLabels.mat')\n",
    "y_test = y_test['testLabels']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Handling bias term:\n",
    "X=np.insert(X,0,1,axis=1)\n",
    "X_test=np.insert(X_test,0,1,axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#theta is randomly initialized to a random value of size 785 * 1\n",
    "theta = np.random.rand(785,1)\n",
    "\n",
    "#old theta value is initialized to a very large value \n",
    "old_theta = theta * 99999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#this values will be used in the program below\n",
    "temp_theta = theta\n",
    "temp_theta_old=old_theta\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This is the hypothesis function( Sigmoid function ) and returns the probabilty of one traning example belonging to one class\n",
    "def hypothesis_class(X,theta):\n",
    "    return 1.0/(1+np.exp(-np.matmul(X,theta)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#The is the derivative function \n",
    "def derivative(X,theta,y):\n",
    "    hypo=hypothesis_class(X,theta)\n",
    "    #hypo = [i[0] for i in hypo] it is used if we want to make it a one dimensional array\n",
    "    # hypo = np.array(hypo) \n",
    "    sa = np.matmul(X.T,(hypo-y))\n",
    "    return sa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Gradient descent method is used as our optimization algorithmn\n",
    "def gdm(X,y,theta,old_theta):\n",
    "    while (np.linalg.norm( old_theta - theta,ord=2)) >= 30:\n",
    "        # the norm is printed as to see the convergence of our optimization algorithmn\n",
    "        print 'norm',np.linalg.norm(theta-old_theta,ord=2)\n",
    "        old_theta=theta\n",
    "        deri=derivative(X,theta,y)\n",
    "        theta = old_theta - 0.01 * deri\n",
    "    return theta    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This function will return all the probabilites that how much a traning example belongs to a class\n",
    "def predictions(X_test,theta):\n",
    "    return hypothesis_class(X_test,theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#In this step i have defined all the y label classes\n",
    "#from 0 class to 9 class\n",
    "ylabel0=np.zeros((60000, 1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 0:\n",
    "        ylabel0[i]=1\n",
    "    else:\n",
    "        ylabel0[i]=0\n",
    "\n",
    "ylabel1=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 1:\n",
    "        ylabel1[i]=1\n",
    "    else:\n",
    "        ylabel1[i]=0\n",
    "        \n",
    "ylabel2=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 2:\n",
    "        ylabel2[i]=1\n",
    "    else:\n",
    "        ylabel2[i]=0\n",
    "\n",
    "ylabel3=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 3:\n",
    "        ylabel3[i]=1\n",
    "    else:\n",
    "        ylabel3[i]=0\n",
    "\n",
    "\n",
    "ylabel4=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 4:\n",
    "        ylabel4[i]=1\n",
    "    else:\n",
    "        ylabel4[i]=0\n",
    "        \n",
    "ylabel5=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 5:\n",
    "        ylabel5[i]=1\n",
    "    else:\n",
    "        ylabel5[i]=0\n",
    "        \n",
    "        \n",
    "ylabel6=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 6:\n",
    "        ylabel6[i]=1\n",
    "    else:\n",
    "        ylabel6[i]=0\n",
    "\n",
    "ylabel7=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 7:\n",
    "        ylabel7[i]=1\n",
    "    else:\n",
    "        ylabel7[i]=0\n",
    "\n",
    "ylabel8=np.zeros((60000,1))\n",
    "for i in range(60000):\n",
    "    if y[i] == 8:\n",
    "        ylabel8[i]=1\n",
    "    else:\n",
    "        ylabel8[i]=0\n",
    "\n",
    "ylabel9=np.zeros((60000,1))                \n",
    "for i in range(60000):\n",
    "    if y[i] == 9:\n",
    "        ylabel9[i]=1\n",
    "    else:\n",
    "        ylabel9[i]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "#This list will contain all the ylabel classes\n",
    "ylabel=list()\n",
    "#appending all to ylabel\n",
    "ylabel.append(ylabel0)\n",
    "ylabel.append(ylabel1)\n",
    "ylabel.append(ylabel2)\n",
    "ylabel.append(ylabel3)\n",
    "ylabel.append(ylabel4)\n",
    "ylabel.append(ylabel5)\n",
    "ylabel.append(ylabel6)\n",
    "ylabel.append(ylabel7)\n",
    "ylabel.append(ylabel8)\n",
    "ylabel.append(ylabel9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#In this we are finding the optimum thetas return from the hypothsis class\n",
    "#for each of the ten y labels \n",
    "opt0=gdm(X,ylabel[0],temp_theta,temp_theta_old)\n",
    "print\"1 over\"\n",
    "opt1=gdm(X,ylabel[1],temp_theta,temp_theta_old)\n",
    "print \"2 over\"\n",
    "opt2=gdm(X,ylabel[2],temp_theta,temp_theta_old)\n",
    "print \"3 over\"\n",
    "opt3=gdm(X,ylabel[3],temp_theta,temp_theta_old)\n",
    "print \"4 over\"\n",
    "opt4=gdm(X,ylabel[4],temp_theta,temp_theta_old)\n",
    "print \"5 over\"\n",
    "opt5=gdm(X,ylabel[5],temp_theta,temp_theta_old)\n",
    "print \"6 over\"\n",
    "opt6=gdm(X,ylabel[6],temp_theta,temp_theta_old)\n",
    "print \"7 over\"\n",
    "opt7=gdm(X,ylabel[7],temp_theta,temp_theta_old)\n",
    "print \"8 over\"\n",
    "opt8=gdm(X,ylabel[8],temp_theta,temp_theta_old)\n",
    "print \"9 over\"\n",
    "opt9=gdm(X,ylabel[9],temp_theta,temp_theta_old)\n",
    "print \"10 over\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#all the ten optimum are appended to the optimum_thetas\n",
    "optimum_thetas=list()\n",
    "optimum_thetas.append(opt0)\n",
    "optimum_thetas.append(opt1)\n",
    "optimum_thetas.append(opt2)\n",
    "optimum_thetas.append(opt3)\n",
    "optimum_thetas.append(opt4)\n",
    "optimum_thetas.append(opt5)\n",
    "optimum_thetas.append(opt6)\n",
    "optimum_thetas.append(opt7)\n",
    "optimum_thetas.append(opt8)\n",
    "optimum_thetas.append(opt9)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Here we will find all the predictions or probabilities\n",
    "pred0=predictions(X_test,optimum_thetas[0])\n",
    "pred1=predictions(X_test,optimum_thetas[1])\n",
    "pred2=predictions(X_test,optimum_thetas[2])        \n",
    "pred3=predictions(X_test,optimum_thetas[3])\n",
    "pred4=predictions(X_test,optimum_thetas[4])\n",
    "pred5=predictions(X_test,optimum_thetas[5])\n",
    "pred6=predictions(X_test,optimum_thetas[6])\n",
    "pred7=predictions(X_test,optimum_thetas[7])\n",
    "pred8=predictions(X_test,optimum_thetas[8])\n",
    "pred9=predictions(X_test,optimum_thetas[9])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#all the predictions are appended to pred\n",
    "pred=list()\n",
    "pred.append(pred0)\n",
    "pred.append(pred1)\n",
    "pred.append(pred2)\n",
    "pred.append(pred3)\n",
    "pred.append(pred4)\n",
    "pred.append(pred5)\n",
    "pred.append(pred6)\n",
    "pred.append(pred7)\n",
    "pred.append(pred8)\n",
    "pred.append(pred9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the index value is  [ 7.  2.  1. ...,  4.  8.  6.]\n"
     ]
    }
   ],
   "source": [
    "#This index value will give me the index of the class whose prediction value is maximum\n",
    "index=np.zeros(10000)\n",
    "\n",
    "for i in range (10000):\n",
    "    maximum=0\n",
    "    maxind=0\n",
    "    for j in range(10):\n",
    "        if pred[j][i] > maximum:\n",
    "            maximum=pred[j][i]\n",
    "            maxind=j\n",
    "            \n",
    "        else:\n",
    "            continue\n",
    "        index[i]=maxind\n",
    "\n",
    "print 'the index value is ',index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy for multiclass is  0.832\n"
     ]
    }
   ],
   "source": [
    "#Here we will find the accuracy to which our multiclass logistic regression is accurate\n",
    "count = 0\n",
    "for i in range(10000):\n",
    "    if(index[i]== y_test[i]):\n",
    "        count = count +1\n",
    "    else:\n",
    "        count = count\n",
    "\n",
    "accuracy=float(count) / y_test.size\n",
    "print 'The accuracy for multiclass is ' ,accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}