linear-model/logistic-regression/logistic-regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 逻辑回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "import pandas as pd\n",
    "import scipy.optimize as opt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getData(path, names=None):\n",
    "    return pd.read_csv(path, header=None, names=names)\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = getData(os.getcwd() + \"/../data/ex2data1.txt\",[\"A\", \"B\", \"Y\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>Y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34.623660</td>\n",
       "      <td>78.024693</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30.286711</td>\n",
       "      <td>43.894998</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35.847409</td>\n",
       "      <td>72.902198</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>60.182599</td>\n",
       "      <td>86.308552</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>79.032736</td>\n",
       "      <td>75.344376</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           A          B  Y\n",
       "0  34.623660  78.024693  0\n",
       "1  30.286711  43.894998  0\n",
       "2  35.847409  72.902198  0\n",
       "3  60.182599  86.308552  1\n",
       "4  79.032736  75.344376  1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>Y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>65.644274</td>\n",
       "      <td>66.221998</td>\n",
       "      <td>0.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>19.458222</td>\n",
       "      <td>18.582783</td>\n",
       "      <td>0.492366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>30.058822</td>\n",
       "      <td>30.603263</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>50.919511</td>\n",
       "      <td>48.179205</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>67.032988</td>\n",
       "      <td>67.682381</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>80.212529</td>\n",
       "      <td>79.360605</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>99.827858</td>\n",
       "      <td>98.869436</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                A           B           Y\n",
       "count  100.000000  100.000000  100.000000\n",
       "mean    65.644274   66.221998    0.600000\n",
       "std     19.458222   18.582783    0.492366\n",
       "min     30.058822   30.603263    0.000000\n",
       "25%     50.919511   48.179205    0.000000\n",
       "50%     67.032988   67.682381    1.000000\n",
       "75%     80.212529   79.360605    1.000000\n",
       "max     99.827858   98.869436    1.000000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.insert(0,'ones',1) # 加入bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ones</th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>Y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.594216</td>\n",
       "      <td>0.635141</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.817101</td>\n",
       "      <td>-1.201489</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.531325</td>\n",
       "      <td>0.359483</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.280687</td>\n",
       "      <td>1.080923</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.688062</td>\n",
       "      <td>0.490905</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ones         A         B  Y\n",
       "0     1 -1.594216  0.635141  0\n",
       "1     1 -1.817101 -1.201489  0\n",
       "2     1 -1.531325  0.359483  0\n",
       "3     1 -0.280687  1.080923  1\n",
       "4     1  0.688062  0.490905  1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "'''归一化'''\n",
    "data['A'] = (data['A']-data['A'].mean()) / data['A'].std()\n",
    "data['B'] = (data['B']-data['B'].mean()) / data['B'].std()\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "开始实现逻辑回归， 因为在推导公式中，为了方便计算，类别是以，1,-1分类的， 因此需要将0换为-1.\n",
    "损失函数:$E_{in}(W)=\\sum\\limits_{n=1}^{N}ln(1+exp(-y_nW^Tx_n))$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def filterData(data):\n",
    "    '''构造输入和标签'''\n",
    "    col = data.shape[1]\n",
    "    x = data.iloc[:,0:col-1]\n",
    "    y = data.iloc[:,col-1:col].replace(0, -1) \n",
    "    return np.array(x.values), np.array(y.values)\n",
    "\n",
    "def getWLin(X, Y):\n",
    "    '''线性回归计算初始值'''\n",
    "    return np.linalg.inv(X.transpose().dot(X)).dot(X.transpose()).dot(Y)\n",
    "\n",
    "def sigmod(s):\n",
    "    '''sigmod函数'''\n",
    "    return 1.0 / (1.0 + np.exp(-s))\n",
    "\n",
    "def update(W, grad, ita):\n",
    "    '''更新W'''\n",
    "    return W - (ita * grad)\n",
    "\n",
    "def MBGD(W, X, Y):\n",
    "    '''批量梯度下降'''\n",
    "    N = len(X)\n",
    "    grad = np.zeros(X[0].shape)\n",
    "    for i in range(N):\n",
    "        sig = sigmod(-Y[i][0] * X[i].dot(W))\n",
    "        grad += -Y[i][0] * sig * X[i]\n",
    "    \n",
    "    return grad / N\n",
    "\n",
    "def SGD(W, X, Y, n):\n",
    "    '''随机梯度'''\n",
    "    sig = sigmod(-Y[n][0] * X[n].dot(W))\n",
    "    return -Y[n][0] * sig * X[n]\n",
    "    \n",
    "    \n",
    "def cost(W, X, Y):\n",
    "    '''损失函数'''\n",
    "    N = len(X)\n",
    "    cost = 0.0\n",
    "    for i in range(N):\n",
    "        err = np.exp(-Y[i][0] * X[i].dot(W))\n",
    "        cost += np.log(1 + err)\n",
    "    \n",
    "    return cost / N\n",
    "\n",
    "def train_with_grad(X, Y, W, GD=1, iterations=500):\n",
    "    '''训练'''\n",
    "    N = len(X)\n",
    "    for i in range(iterations):\n",
    "        if GD:\n",
    "            grad = MBGD(W, X, Y) # 全梯度\n",
    "        else:\n",
    "            grad = SGD(W, X, Y, i%N) # 随机梯度\n",
    "        err = cost(W, X, Y)\n",
    "        W = update(W, grad, 0.1)\n",
    "    return W, err\n",
    "    \n",
    "def train_with_fmin(W):\n",
    "    '''使用scipy的fmin优化'''\n",
    "    W = np.matrix(np.zeros(X[0].shape))\n",
    "    result = opt.fmin_tnc(func=cost, x0=W,fprime=MBGD, args=(X,Y))\n",
    "    return result\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''使用随机梯度下降迭代求解'''\n",
    "X, Y = filterData(data)\n",
    "W = getWLin(X,Y).reshape(X[0].shape)\n",
    "W_SGD = train_with_grad(X, Y, W, GD=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''使用全梯度下降迭代求解'''\n",
    "X, Y = filterData(data)\n",
    "W = getWLin(X,Y).reshape(X[0].shape)\n",
    "W_MBGD = train_with_grad(X, Y, W, GD=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''使用第三方库scipy的fmin直接优化'''\n",
    "X, Y = filterData(data)\n",
    "W = getWLin(X,Y).reshape(X[0].shape)\n",
    "W_fmin = train_with_fmin(W)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((array([ 1.12902306,  2.51740545,  2.37171023]), 0.22272664575554293),\n",
       " (array([ 1.02284455,  2.5412209 ,  2.32502707]), 0.22322532581110394),\n",
       " (array([ 1.71881679,  4.01402376,  3.74529924]), 20, 1))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W_SGD, W_MBGD, W_fmin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20349771521035276"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cost(W_fmin[0], X, Y) # 计算fmin的误差"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看出随机梯度和全梯度与直接使用fmin误差差不多，但是训练时间的话，fmin 最快，随机梯度次之，最慢的全梯度。因此以后的优化问题均使用fmin。\n",
    "\n",
    "另外基于原数据作了一个简单的测试，正确率85%+"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def predict(W, X, Y):\n",
    "    '''预测'''\n",
    "    predictions = [1 if sigmod(x.dot(W)) >= 0.5 else -1 for x in X]\n",
    "    correct = [1 if ((a == 1 and b == 1) or (a == -1 and b == -1)) else 0 for (a, b) in zip(predictions, Y)]\n",
    "    accuracy = (sum(map(int, correct)) % len(correct))\n",
    "    return predictions, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fmin accuracy = 89%\n",
      "SGD accuracy = 89%\n",
      "MBGD accuracy = 89%\n"
     ]
    }
   ],
   "source": [
    "result_fmin = predict(W_fmin[0], X, Y)\n",
    "print 'fmin accuracy = {0}%'.format(result_fmin[1])\n",
    "\n",
    "result_sgd = predict(W_SGD[0], X, Y)\n",
    "print 'SGD accuracy = {0}%'.format(result_sgd[1])\n",
    "\n",
    "result_mbgd = predict(W_MBGD[0], X, Y)\n",
    "print 'MBGD accuracy = {0}%'.format(result_mbgd[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    可以看出，正确率不到90%,因为我们没有加任何trick，比如正则化等等。下面我们再使用sklearn自带的lr模型试试如下:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sklearn accuracy = 89%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "LR = LogisticRegression()\n",
    "LR.fit(X, Y)\n",
    "lr_result = LR.predict(X)\n",
    "correct = [1 if ((a == 1 and b == 1) or (a == -1 and b == -1)) else 0 for (a, b) in zip(lr_result, Y)]\n",
    "accuracy = (sum(map(int, correct)) % len(correct))\n",
    "print 'Sklearn accuracy = {0}%'.format(accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "大致差不多。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}