regression-problem.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:ac67ecf0f39e58d891bed98f9baa82085a05933e1f4efdc90ce5769a8f51307f"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u555f\u52d5\u4e92\u52d5\u5f0f\u7e6a\u5716\u74b0\u5883\n",
      "%pylab inline\n",
      "import numpy as np\n",
      "import matplotlib.pyplot as plt"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Populating the interactive namespace from numpy and matplotlib\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u8f09\u5165 file \u8f38\u51fa feature,label(\u6700\u5f8c\u4e00\u6b04)\n",
      "def loadDataSet(fileName):      \n",
      "    numFeat = len(open(fileName).readline().split('\\t')) - 1 \n",
      "    dataMat = []; labelMat = []\n",
      "    fr = open(fileName)\n",
      "    for line in fr.readlines():\n",
      "        lineArr =[]\n",
      "        curLine = line.strip().split('\\t')\n",
      "        for i in range(numFeat):\n",
      "            lineArr.append(float(curLine[i]))\n",
      "        dataMat.append(lineArr)\n",
      "        labelMat.append(float(curLine[-1]))\n",
      "    return dataMat,labelMat"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#####\u77e9\u9663X \u5c0d\u61c9y  \n",
      "#####x:feature y:label\n",
      "\\\\[ y =\\{x^Tw}\\\\]\n",
      "#####\u5df2\u77e5x \u627e\u5230w \u5373\u53ef\u9810\u6e2cy\uff0c\u6545\u4e0b\u5217\u5404\u7a2e\u65b9\u6cd5\u70ba\u627ew\uff0c\u5efa\u7acbmodel\n",
      "\\\\[ w =\\({x^Tx})^{-1}{x^Ty}\\\\]"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# OLS \u666e\u901a\u6700\u5c0f\u4e8c\u4e58\u6cd5 ordinary least squares\n",
      "# \u7406\u5ff5\u70ba \u6c42\u89e3w \u800c\u5e73\u65b9\u8aa4\u5dee\u6700\u5c0f\u5247\u8d8a\u63a5\u8fd1\u9810\u6e2c\u60c5\u5f62\n",
      "def OLS(xArr,yArr):\n",
      "    xMat = np.mat(xArr); yMat = np.mat(yArr).T\n",
      "    xTx = xMat.T*xMat\n",
      "    # \u6709\u4e9bmatrix\u4e0d\u53ef\u8f49\u70ba\u9006\u77e9\u9663\n",
      "    if np.linalg.det(xTx) == 0.0:\n",
      "        print (\"This matrix is singular, cannot do inverse\")\n",
      "        return\n",
      "    ws = xTx.I * (xMat.T*yMat)\n",
      "    return ws"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "fileName = '/Users/wy/Desktop/ex0.txt'\n",
      "dataMat,labelMat = loadDataSet(fileName)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dataMat[1:3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "[[1.0, 0.42781], [1.0, 0.995731]]"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "labelMat[1:3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "[3.816464, 4.550095]"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u6c42\u51faw\n",
      "OLS_w = OLS(dataMat,labelMat)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "OLS_w"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "matrix([[ 3.00774324],\n",
        "        [ 1.69532264]])"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# dataMat\u70balist\uff0c\u9700\u8f49\u70bamatrix\u904b\u7b97\n",
      "xMat = np.mat(dataMat)\n",
      "yMat = np.mat(labelMat)\n",
      "yHat = xMat*OLS_w"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u628ayHat matrix \u8f49\u70ba yHat.flatten().A[0] list\u683c\u5f0f\n",
      "# \u76f8\u95dc\u4fc2\u6578\n",
      "corrcoef(yHat.flatten().A[0],yMat.flatten().A[0])\n",
      "# 0.98 \u5efa\u7acbmodel\u6240\u9810\u6e2c\u7684y\u548c\u539f\u5148y\u76f8\u6bd4\u7684\u76f8\u95dc\u4fc2\u6578 \u883b\u9ad8\u7684XD"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "array([[ 1.        ,  0.98647356],\n",
        "       [ 0.98647356,  1.        ]])"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# linear regression\u6700\u5e38\u51fa\u73fe\u7684\u554f\u984c\u70baunderfitting\uff0c\u6703\u6709\u4e00\u4e9bbias\uff0c\u6545\u5b9a\u6bcf\u4e00\u9ede\u7684\u9644\u8fd1\u7d66\u4e88weight\n",
      "# \u5c0d\u6bcf\u4e00\u9ede\u7d66\u4e88 Gaussian kernel \u4f86\u6539\u8b8a weight\n",
      "# \u5176\u7406\u5ff5\u70ba\u539f\u5148\u7528\u4e00\u689d\u7dda\u53bbfitting\uff0c\u6bcf\u4e00\u9ede\u7d66\u4e88weight\uff0c\u6709\u9ede\u50cf\u662f\u7528\u5f88\u591a\u689d\u5f88\u77ed\u7684\u76f4\u7dda\u53bb\u7d44\u6210model\n",
      "# Locally Weighted Linear Regression\n",
      "def lwlr(testPoint,xArr,yArr,k=1.0):\n",
      "    xMat = np.mat(xArr); yMat = np.mat(yArr).T\n",
      "    m = shape(xMat)[0]\n",
      "    weights = mat(eye((m)))\n",
      "    for j in range(m):                     \n",
      "        diffMat = testPoint - xMat[j,:]\n",
      "        weights[j,j] = exp(diffMat*diffMat.T/(-2.0*k**2))\n",
      "    xTx = xMat.T * (weights * xMat)\n",
      "    if linalg.det(xTx) == 0.0:\n",
      "        print (\"This matrix is singular, cannot do inverse\")\n",
      "        return\n",
      "    ws = xTx.I * (xMat.T * (weights * yMat))\n",
      "    return testPoint * ws\n",
      "\n",
      "def lwlrTest(testArr,xArr,yArr,k=1.0):  #loops over all the data points and applies lwlr to each one\n",
      "    m = shape(testArr)[0]\n",
      "    yHat = zeros(m)\n",
      "    for i in range(m):\n",
      "        yHat[i] = lwlr(testArr[i],xArr,yArr,k)\n",
      "    return yHat"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u6392\u9806\u5e8fx,y\u7531\u5c0f\u5230\u5927\n",
      "srtInd = xMat[:,1].argsort(0)\n",
      "xSort = xMat[srtInd][:,0,:]\n",
      "\n",
      "# # xMat[:,1] \u7b2c\u4e00\u7dad\u7686\u662f1\u53bb\u6389\n",
      "# # xMat[:,1] \uff1d matrix([[ 0.067732],[ 0.42781 ]])\n",
      "# # xMat[:,1].flatten() \uff1d matrix([[ 0.067732, 0.42781 ]])\n",
      "# # xMat[:,1].flatten().A[0] \uff1d matrix([0.067732, 0.42781])\n",
      "\n",
      "fig = plt.figure()\n",
      "ax = fig.add_subplot(111)\n",
      "# underfitting\n",
      "ax.plot(xSort[:,1].flatten().A[0],sort(yHat[srtInd].flatten().A[0]))\n",
      "ax.scatter(xMat[:,1].flatten().A[0],yMat.T[:,0].flatten().A[0],c='red')\n",
      "\n",
      "# overfitting\n",
      "yHat2 = lwlrTest(dataMat,dataMat,labelMat,0.003)\n",
      "ax.plot(xSort[:,1].flatten().A[0],yHat2[srtInd],c='white')\n",
      "# better fit \n",
      "yHat3 = lwlrTest(dataMat,dataMat,labelMat,0.01)\n",
      "ax.plot(xSort[:,1].flatten().A[0],yHat3[srtInd],c='green')\n",
      "# \u767d\u7dda\uff1ak=0.003 \u767c\u751foverfitting \u904e\u5ea6\u8026\u5408(\u5c0d\u76ee\u524d\u9810\u6e2c\u6e96\u78ba)\u5c0d\u65bc\u672a\u4f86\u9810\u6e2c\u6578\u64da\u53ef\u80fd\u6703\u767c\u751f\u4e0d\u6e96\u78ba\n",
      "# \u85cd\u7dda\uff1ak=1.0 \u767c\u751funderfitting \u7121\u8026\u5408\u76f4\u7dda\u9810\u6e2c\u4e0d\u6e96\n",
      "# \u5fc5\u9808\u8b93\u7dda\u5b58\u5728\u65bc\u63a5\u8fd1\u6578\u64da\u8d70\u5411\u4f46\u662f\u53c8\u4e0d\u80fd\u592a\u904e\u8cbc\u548c\u6578\u64da \u8abf\u6574k\u503c\n",
      "\n",
      "# \u5728 Locally Weighted Linear Regression\u4e2d \u7528k\u4f86\u8abf\u6574\u8a72\u9ede\u9644\u8fd1\u7684\u7bc4\u570d\n",
      "# k\u8d8a\u5c0f\u4ee3\u8868\u8d8a\u7cbe\u7d30\u5f71\u97ff\u7bc4\u570d\u8f03\u5927 k\u8d8a\u5927\u5247\u5f71\u97ff\u7bc4\u570d\u8f03\u5c0f \u56e0\u6b64k\u592a\u5927\u6703\u767c\u751funderfitting k\u592a\u5c0f\u6703\u767c\u751foverfitting\n",
      "# \u4f46\u6b64\u65b9\u6cd5\u7684\u904b\u7b97\u6210\u672c\u592a\u5927\uff0c\u56e0\u70ba\u5c0d\u6bcf\u500b\u9ede\u5fc5\u9808\u9032\u884cscan\u6574\u500bdataset"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "[<matplotlib.lines.Line2D at 0x1062a3f50>]"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEACAYAAACnJV25AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVNX7wPHPGRHFABFBc8s9LU1ES3FJsQTKyv1b5hJa\nqe2WuGS2WGJWipbVz6VFUDPNSsMyxi0sy33BpdzN3Nc0zVHEeX5/zECoKIvAwPi8X695defO4d6H\niXnm+NxzzzEiglJKKfdhcXUASimlcpcmdqWUcjOa2JVSys1oYldKKTejiV0ppdyMJnallHIzWUrs\nxpg/jTEbjDHrjDErr9JmnDFmuzEmyRgTnLthKqWUyiqPLLYTIFRETmT0ojGmDVBDRGoaYxoD44GQ\nXIpRKaVUNmSnFGOu8VpbIA5ARFYAfsaYstcTmFJKqZzJamIXYKExZrUxpncGr1cA9qZ7vg+oeL3B\nKaWUyr6slmKaichBY0wgsMAYs0VEfrmszeU9ep2rQCmlXCBLiV1EDjr/e9QYMxtoBKRP7PuBSume\nV3TuS2OM0USvlFI5ICLXKoVfIdNSjDGmhDHGx7l9ExAObLysWTzwmLNNCHBSRA5nEFyhfbzxxhsu\nj+FGjb8wx67xu/5R2OPPiaz02MsCs40xqe2/EJH5xpi+zmQ9UUTmGWPaGGN2AP8CvXIUjVJKqeuW\naWIXkd1A/Qz2T7zs+XO5GJdSSqkc0jtPsyg0NNTVIVyXwhx/YY4dNH5XK+zx54TJaQ0n2ycyRvLr\nXEop5S6MMUhuXzxVSilVuGhiV0opN6OJXSml3IwmdqWUcjOa2JVSys1oYldKKTejiV0ppdyMJnal\nlHIzmtiVUsrNaGJXSik3o4ldKaXcjCZ2pZRyM5rYlVLKzWhiV0opN6OJXSml3IwmdqWUcjOa2JVS\nys1oYldKKTejiV0pVWBYrVY6hYfTKTwcq9Xq6nAKLV3zVClVIFitViI7dOBdmw2AwV5exM2eTURE\nhIsjcy1d81QpVWhNionhXZuNSCASeNdmY1JMTI6O9e+/MGAAxMbmZoSFhyZ2pVShlVHp5scfoW5d\nOHwYHnjAxQG6iIerA1BK3ZisVmtaj7xPVBQNWrbkpUWLwG4HnKWYqKhr/nz60k33X7ZRN2QZe/aU\nY+JECA/P+9+hoNLErpTKE5cn7tRaudVqZfiQIfyelMRYZxLvsnAhKUCECK8C/wI+fn7XPObhY8fo\nbrPxHbCHXvx77h0O/PkTY9/3Z+LoUUwcfel5byR68VQpleuudiEUILJDB6rabDyFo5YOEAdEFy1K\n3RYt6JuczOFffuFFAE9PZsTHA/Dys8/y186dlAEeACYaQxGpQTkmch5v/qY35auf4fiBA251ATYn\nF081sSulcl2n8HDaLlhwSeKeHBxM6YAA2i5YwAIfHx6uWpW2pUtztEY5YhuV5PdSNk4fP4jF24s2\nc7eQMmsLn12EIsHBbNu8mVHJyQAMAC5QlGIM4iwv8p7Pe/QJ+JZpu3fyipcXbzsvwKaeNz4sjG/m\nz8//NyGX5CSxaylGKZV3ypaFoCCYP59NSUlUqloVgAnLlrHo5BY6JP0fi47PIXBrIJY1x7nn+Bma\nB9dnWqsAkuqVxnslnFq4gRrJF7kZiAC204QxTOI8fzKCBjwdPwWajaBoQADn/vnHpb9uQaE9dqVU\nrrNarTzapg3Wfv24tW9fatWuTQ1gM1A9vDalOpdl+ZYlXFwOxdbAzecd5ZVpwLsAxvDT8w9zrgn8\n+Mcc7lp9nvUrfGh9cSzWk/cTSD8O8g2+N5Xgz8OHOb5qFa9OmsTJL79kaeoxgJcsFr6cN++GK8Xo\ncEelVK6LiIjg9qAgTrVoQclatVi4YgU+T99L4BOGI82P4b3Pm/LjoPJv8MR52G8MnwLdgQlAlAiJ\n42by8Gtr+KPlTIJ6PAsDi3Fh4jymb0iieI313FK9Gm++/z6Hjh5l9rp1dHnsMR4CZlasyF/h4UwA\nbg8KKtRJPae0FKOUyhP3d+pEg7vvZvuudXRdFsnZqskMazKeR2/vzHd3NWJUChz192d3w4bMiopi\n9erVjH39dWra7fQEYi0W9uyw8UWYHd/QgTy7dTcfVJnLkvBEyj7mzW0L/mLa1skcqHSASqGVCAws\nR/dinnweHc2QRx/lz8hIHu7Z07VvgotoYldK5Ykd69djOX6MyFENeWSbUOFnWHXbB3RvvJzTu3ax\n3WLhi+nT03rUERER3HnnnUyKiWGXWGhS/EWivm9IO/tHNFvchVc8ocSpYtQ97EfXKa/wztI+BJf1\nInH9Io6fP8nhYoe5OKAIi0qvYeUHW+nz4os0btzYxe+Ca2gpRimVJypXrcoU6/ukGGHIL2CAlX/8\nQVxsLC9ZLLz01ltXlEkiIiIYFjOfg2cSOHnyPiZO2oolbBnTgutwAeh65gzrduwiwlaViaX+xyv3\njqLcygD2DtvI939H8suLazHbtzP21GjGbI6hfcS9N+ZkYiKSLw/HqZRSN4qFVqtUGHyzPFMHiQXx\nBynn7S0dw8IkISHhivZnz4q88opIQIDIxIkiFy/+91rHsDCJBemI41jSqpXIkSNy4exZqeznJxVB\nZteu7WjcqJFMKuMr7b5oJ9UaG/Hz9MzwfIWFM3dmK99qKUYplSfK1C3H4cVHWb0F1gJ2T08mf/11\nhhczFy+Gvn0hOBg2bIBy5TI5+E8/QXg4m9q25Z+TJ/EFem/ZgqVePf7euJFXgJUN32X9me30Sf6d\n4UOG3FAXUbM03NEYUwRYDewTkYcuey0U+A7Y5dz1jYhEZ3AMycq5lFLuIcoaxb6/9pEy628g49v7\njx93zMK4aBF8/DE89FBGR/rvTtbuNhufAe879w/GMZJmkjEYEW537v8d+GPaNMovXEhcbCzD/f3Z\ncfx47v+S+SAvb1Dqh+O98rnK60tEpG12TqyUcl/nUs4xZcMUVjy5gmq9q13xughMnw5RUdClC2ze\nDD5Xyy44au9xs2czKSYGr2XLmHDmDOVx3Fl6CGhQvz4pwKE9e7DZbPS22Sj/ySfgnFqgYuXKefFr\nFlyZ1WqAisBCoBUwN4PXQzPan0G7vCtCKaUKlKlJUyV8aniGr+3cKRIeLhIUJLJyZfaPnZCQIGW9\nvCTWWW8v6+V1SQ09ISFBAj09014PvAFr7FkZFTMWGAjYr/bdADQ1xiQZY+YZY26/SjullJtLnR/9\no2Uf0qJEi0teS0mBUaOgUSO4915YtQruuiv750jtvceHhREfFnbFJF8RERFMjY9Pe31qfPwNVV+H\nTGrsxpgHgftF5FlnLT1Krqyx+wAXReSsMeZ+4AMRuTWDY8kbb7yR9jw0NJTQ0NDc+S2UUi6XWgcf\nXLMk73UX7G+eYso3c4iIiGDVKujTBwIDYfx4qF7d1dEWXImJiSQmJqY9f/PNN3N3dkdjzNtADyAF\nKA744rg4+tg1fmY30FBETly2X651LqVU4WW1Wnm2a1deO3GC1S/eil+NutR47lu+bfUQ1YLi+fJL\nGD0aunUDk60UpXJ9rhgReUVEKolIVaALsPjypG6MKWuM43+VMaYRji+LExkcTinlhlJ76oEnTmAr\nBl/476fvFm/W04YFyz7h779h0ybo3l2Ten7J7jh2ATDG9AUQkYlAZ+BpY0wKcBbHF4BS6gaRugh1\nlYoVaV/pIK2q3cvTEzvxo6lD9Bt/8fLLZV0d4g0ny4ldRJYAS5zbE9Pt/xj4OPdDU0oVFsbDg6Zr\nVuH7aSNal3yBocc38O0cL9q2DXN1aDckvfNUKXVd+kRF8VmxGpzZ+iOVy1Xhr8VrGfBqsiZ1F9KF\nNpRSOXb+PLzzDpSveoIBO2vzbLEOFBv3HeP/+afQrzVaUOiap0qpfPPLL44hjLffDsUe6crRb79j\nwcyzgHusNVpQ6ApKSqk8d/KkY8Kujh3PUdLzTTzrduK3v37lf9+edXVoyklr7EqpLElIsPLG4JVs\n2Po09YL2YM48QOe/DjOyqC8P7r+fweYQxUgGYLCXF3FRUS6O+MalpRilVKamTVvCE5H/EmCvTC/6\nMMGynDeL2fnwScNznd4h8OEYxleoQOmAACDjmRxVzuTl7I5KqRvQxYvw0UcwaGB9HrSP4ssSXfG8\nqz4BIS/yeekfaFfzXp47Wo24I0coHRSkNfUCQnvsSqkMJSVB795QogR4Xnicbr9NJnL5clLkIqGL\n+nDo+Bne/saO7a+9jtKLjoLJE3rxVCl13c6ehZdfhrAweOopx2JFUa8/wluBgSTXqUPrkc1Ztf0P\nevj3Ymat2hnOsKhcS3vsShVCVquVSTExQO7WsxcscCTzxo1h7Fgom242gJUrVzL9r2nErYjl81Zx\ndGjTIVfOqa5Nx7ErdQNInXTrXefqQKllECDHyf7oUcdqRj//7JhW9/77r2yz6cgmWsW1YsNTGyjn\nk9mipCq36MVTpdxc6vS479psRKbutNkYOWQIW7ZsSUv2kUuXZqk8IgJTp8LAgY7ZFzdtAm/vjNoJ\n/RL68XqL1zWpFwKa2JUqJFJ76lVtNjYCnZz7qwL79uzJMNmn9uAbtGzJ2iVLgP968zt2OMouJ07A\nvHnQsOHVz/ta3AD+CblA3XN18/A3VLkmu2vp5fSBrnmqVLYkJCRIx7Aw6RgWlrYdCzKiSBFpcOut\n8mXp0hIL4gtyS7lyEgsiNWqIbN0qcR4eUtpikViQKGebWOejTHEf6dVrq5QuLTJ6tMiFC1eeK30M\npSsVE5/BRpZt+UluLlGiUK8fWhiRgzVPNbErVQBltGBzy+BgiQVZMm2a/Lt/v8ipUyK33SaxILf4\n+EhZLy/5a+5cERFpUreuRDkqLdLReQwBWeHRRCqSJKV8lsquXVc/V2ryfui+UCn7PBLbs77IwoUS\nC9IxLMxVb8sNKSeJXYc7KlUATYqJobvNRjzwCRBos7Frxw429+vHv7d40bpXZe4a25hePaqx/CYo\nWrQoP/zyC5agIFYtW0b4gw9yR7rj2fBm0Xur8ViayIOMxJxpwbZt1rRzpZZxIoF3bTYmxcRgFzur\nbt1Ew38CiXx7nuOqqioUtMauVAF0+NgxFuJYZPg08AFQPLg2i+7+l7Y/PoLlYgr91l3gQMdixD4P\nfkXsDN8+HN/ffSl2qgg333Izz1pgox1W8BB+T07Ep9EKthxfwLF2x+j1nT2t/p60Zg1tLzt/0po1\n1H3iVnxqleKTlxezuG9f9s6dq3PAFBI63FGpAqhejRrs2bkTD2AM4BFamv73GRrtKcGm+H8Ytv+k\n40LpunV8P3QQCxtVp3rz23llRH/urluevRWLsun4bkrsv5ubSpTgfNVfubg/mQqmJEcrnCfiZBX2\n/JzCrvU76W6zEQe87+1NypkzPO8BdZrDjmbF+emJFaxbmsScqVMBnQPGFXS4o1JuwGq1cnjPHnyB\nNhHhrO9bnR/3LuTHXcFsHP8Vf/j7/9d47FhC3ojm7urVad2kCR9tT6HH7iLI9q38+fcBftgyF5+b\nilNl3E38Ffc1EzjHoDuqsLh/bTb7zqNpxSK8aIX77N7U276N/w1ox/nSq/Dzq826blNZ1zGSOYGB\nOgdMIaOJXakCJHVI43spKQAs6VKeGau/ZnjcRTbu/4qXLBai+vdn8IgRYLNhmTGDiHff5fikSXhs\n385+bufuPydxuOJa/j7RmzEpGyhSvDiP2mx0B476+zPv5pr0Ld+LphcasbPbTurV/oRalgB2T6lP\n6WAfYqu9R6WSDdlxT3tO7N/vmFtAFS7Zvdqa0wc6KkapTKUOaRSQ9YN7SsBIf6ldykNCQPyNkejo\naBFxjGSp7u8vISA/V68uNk8fCeFNMRyRx3hKKmDSjiPO0S6l4IqhjM0bNpR1G5bL3K1z5f/urC72\n7dtFBg+WOItFQi4bIaNcgxyMitEeu1IFyJZdu1gJbIzsweKKa+j87UW+/juFw8YQNXw4Q4cOBSAi\nIoKPp08nskMH5u2swMP8wN+WPxg44Et2rNuJZbk3nD59ybErV69+RX1868aNbL33IdY2bcrtq3di\natYEQHD07uOmT9eaemGU3W+CnD7QHrtS1xQdHS2+IFM8PeU965tS8llvKQGS4OxxXz5+/PhxkYiI\nveJV7IiEBL1xRW880NMzbWx6oKfnFT3v9P86SAAJSHcTk/bUCw50HLtShdfkMWMYB7Tr0oFRq8Yw\n6KszBACX95dFYMYMqFMHatasyKEjgSxbP+ySnnVERART4+OJDwsjPiyMqfHx1+x5R+AYwz7c31+n\n4XUDWopRqoD5oFkRwrf4UeHoKexAHP+tIfrnn/DMM7B3L8yeDSEhVz9ORETENZNzn6goIpcuBefE\nYdO8vLT04ia0x65UAdGrf3+eLw4f/J1AI+s5XgB8q1cnPiyMz7+ew+bNEdx5J9x9N6xde+2knhUR\nERHEzZ6d1qvXXrr70BuUlCogrFYrT3/Zh8YRTfit7zz6DB7M0KFDWbvWsUSdnx9MmADO65vqBqFL\n4ylVSFmtVrp3a8/xSod4ztKK8ykp1K3bmAEDHItevPACLFyoSV1ljdbYlXKh1CXuktasoWLjc7Qs\neS/N1u2mm60FXbvUpWNnx+IXgYGujlQVJlqKUcpF0i9xd6QUvPF8cTY/8iujWu/h64NB3NrgE5au\nGenqMJWLaSlGqUIk/XS565+szgsN+xH+wC3sObgNU/wuXns71NUhqkJKSzFKudi8lqEsCtzCrYPb\nc+5EJ4qHFWNKlA47VDmniV0pF+n1wgC6/NSUWx9cTsSxh0j4szVT5nyjCV1dNy3FKOUCy5bByy+H\n06BLA04U2cr5LQc1qatcoxdPlcpHp07BK6847hodO1YYe7oJzzd6nm71urk6NFVA5dnFU2NMEWPM\nOmPM3Ku8Ps4Ys90Yk2SMCc5OAErdKGbPhho1zjHvu3k0rNWRrZbhnEs5x6N3POrq0JSbyWqNvR/w\nO+Bz+QvGmDZADRGpaYxpDIwHrvNmZ6Xcx7598PzzsGbNGS7+055hyYu4eBCGtFrKe61HYTFaEVW5\nK9O/KGNMRaAN8CmQ0T8H2uKYpwgRWQH4GWPK5maQSuUGq9VKp/BwOoWHY7Va8/x8Fy/Cxx9DcDAE\nBUEV/3t5PHkR8cCkRsW4tVxNlk9YmOdxqBtPVroKY4GBgP0qr1cA9qZ7vg+oeJ1xKZWrUm8Gartg\nAW0XLCCyQ4c8Te4bN0Lz5o7pdX/+GZo0sbJ+wyrigIii8Ge4Jw943MfGTZvzLAZ147pmYjfGPAgc\nEZF1ZNxbT2t62XO9SqoKlPQ3A0UC79psTIqJyfXz2GwwdCjccw/06gVLlsBttznOX1mE0cCeHlVo\nVfMe6oz7VscbqzyR2d9VU6Cts45eHPA1xkwRkcfStdkPVEr3vKJz3xWGDRuWth0aGkpoaGgOQlbq\nOlSoAB99BB06XLI7dc4WcMxTnpNhh4sXQ9++jtLLhg1QrpzjuCOHDOGPjRvxAg5UK8Entc6QlADz\n16+ntC4UrS6TmJhIYmLi9R0kq0stAS2BuRnsbwPMc26HAMuv8vO5vWKUUlmWkJAgfp6e8krnziIi\n0rhevbSl3xISEqSsl9cly8JFR0dLx7Aw6RgWlukScceOifTsKVKpkkh8/KXnTL883W23VJTaY2pJ\n9KiuEgtS2sNDl59TmSIfFrMWAGNMX2emnigi84wxbYwxO4B/gV7X91WjVN4oCjxSvz4pNhvt2rdn\n+JAhTIqJ4fixY44yzYcfwqJFbJwzh5jXX2es3XFZKXLp0gwXoRCB6dMhKgq6dIHNm8En3bixSTEx\njEpOJhKgXj3+HPkAM5dNZ030dL4Hat9xh96QpPJGdr8JcvpAe+zKhdIWbv7+e5GYGDm+fr2EOHvS\npS0W+dLfX+TUKZHt2+X7yZPlC19fEedCz7EZLCS9c6dIeLhIUJDIihUZn7NlcHDaYtG/Tx8nfq8X\nl8Ylr35MpTKCLmat1NUVCwiApk1h5EiKlylDw5o1iQR62u0k3Hsv+375hS/q1+fI+fO0TUqCli3T\nfvb4sWMApKTAqFHQqBHcey+sWuXYTi91WOW27dsZAEwrVZLHT0/DssjOqVP/rWHaJyoq3353dWPR\nKQWU20p/QbRBy5ZYLlygTlkYP+dNvFu3p8y/HhxYtBxzeB81BvXnnzOn2LNoK3/9cZB6t97K5KlT\n+bFZM/ps3Uqt4GDen7SW3r0di16MHw/Vq195jgXffMOmpCR62u38CjQDfulTEc+65WnUbyVfe/tw\nZ0hIji/QqhtPTqYU0NFWyi2lX8QCYOjy5Xy1LoHWcffi412KEltXUb/1XZy4pSrbNp3nX8smzhc7\nz6a7/8Be6wIBy3by3fdTORoSwqNb9/Hj8Vd58EEYPRq6dQNjrjzHCwsW0BvHRabBwBNeMOYeuPnm\ns3yZ3Jb2xTcSN2uWJnSV57QUo9xS6rj1m4F4oPODrej8TWc++9/n3La7Cm9OPsjseyaypGQ/dg3e\nyMy2X7B1wCri7ohjYKOXWRVYhKd2v8KW24rwEZs5fcaDO2t3JjDQijGXniNy9GgiixdnHJBcuzZ3\nrbXSM2EA//eKL7UE3gn6gGXJlgwvwCqVF7THrtzWRhw95+giMKHRTgK2lcT/gD/79uxB7HZ46y1Y\ntowFFy6w9847eefsWT4f+wHfzJ/PqbVeLFlanjkRg/Cs/xvd12/hjiUQuXLeJQm6Utu2jmExkydz\n+N/dbIuqQOvZHWiwNoVvhs1hwvKPeei+jpToWMKl74W6sWhiV24lteZ9+NgxEo1hjAhJXctQybcC\n7SctZNKOGG6uXJkBJ044JnLBkfzjNm7kECBiePHFTYz7oCvF+ZSloz8j7OQDnHm4J/cGNuPd3r2Z\nFBNDREQEfaKiCChThl37/6Bv6N+sLleU7qdK0mnkWQbbLdSs8gOjP/oIEcFqtWpvXeUbLcUot5F+\nPpje69YhwG/14Ntadj77oyZcdIxLf23kSFI8PRkN9Ae6A4eAqGJB7Dj8JVMme1CX1oxnKA3m/kZC\nr/nMuvA16xsEsKBuXVYtX07zBg1YNG8eW05vo9HkZtRo1oav757Fxpl7SWgVRtRbb/Hl559T/MAB\n/lmxIs/nplHqEtkdH5nTBzqOXeWhhIQEqe7vLyEgCRaLSFCQDHq8pXgPu0k27F0r00uVkrJeXpfc\nbdoxLEyaBQfL3UGN5LZqsVKy5Hn58EORDq0j0sa4i6eniK+vdKqOlHijhHw+rp/EggSAfDg3Rvxe\nLSbfvtVV5MABmVmuXNrY9LRx87Vri9xzj45bVzlGPtx5qlSBc/nolAPjxvFr40p8ktCdnpbuvPPM\nG+yvUoW4kSPTyiERERFERESwdCn06QM164L1I6hUCWrWfIkuP//EgORkcD4W/wMD53jyctfpvHxP\nCe6rVo63No/kudjzdOBnmNoS28GDULfupcFt2eJ4KJWPNLGrQin9+PG0KQEAmjdnU6v6NJh8Nx7f\nC3dumQg4bghK7+RJePll+P57GDfOMSdY6miXiIgIZsTHM3zIEIbv2UPFypWpB1Rdt46fnh3BsGZT\n8ap2O3dbz/J/+xKowb60c8Q5bzrqExVF5NKljukeL3tNqTyX3S5+Th9oKUblkssn7SptsTjKHkWK\nyL8b10q1wb7iHWJJu50//S38drvIrFki5cuLPP20yMmTWT9nKYtF4hs0EBGRX5cskU4gPsZIy+Dg\nDCcLSy33ZGUiMaWuBi3FqBtB+rnVATba7bxkseBXtw4zVr/JgV3/0uB8PWD9JT939lwg7drBjh3w\n1VfQrFnWzxkREUG5qlXpuXYtT7RrR7FffmEJUKVaNRLXrr3qz+hIGOUKmthVoWYFfgWKlyhBbLtK\nJB1NYkbPbyhepDiRHTqAzYYdC/2KvohJGsmAATBrFhQrlv1zBfr6MgiIjI8HoAYw2dc3F38bpXKH\nJnZV6KTWrzfabMQBo4HdvmeIKZLIrFZfc9+d9wEQN3s27w2bzbrfX6RK5TLMmuVJrVo5P2/pgIAs\n7VPK1XQcuyp0IiIiiJs9mzn+/owGHqx/K7GPWhhzaxSrvv0ZgLNn4aefIti4cwKjxtRm/Xr/60rq\n4PhCGezlRRw6Q6Mq2HR2R1VodQoPp8uO7Xz6ajVqBdzGqPDRPNy2Lc8NnM9TTzmm033/fShbNvfO\nmRtL6CmVHTmZ3VETuyqUrFYrQ196ibtea8EPm+fyzuSLlL2vCw/E3knJgE5MnuxFmzaujlKp65eT\nxK6lGFXoWK1Wuj34IP06NGLmrpn0/+AAKQfC6fr5EELthwmp84gmdXVD04unqtAZOWQIo4t5EFNu\nNUE/3MInZ2ZRDH/m0YZNrCXeI8zVISrlUtpjV4XOvj17WP3MnXif92VNwkL+5EeeoxGbWKsXNJVC\nE7sqZKxWK2e8b2O6/x94xb/E6zTCgzFEcZHJwcG6mIVSaClGFSKzZy+k6/92EBhVjB43P8YnPz/M\nPsBuDIOGD2fo0KGuDlGpAkETuyoUhg1by4jhtanb4DtSyu/j7akXqQsM9/fn6+nTtZeuVDqa2FWB\ndvAgPPzwIZb96kPt4j040nEzX6Z0oejiDwEIathQk7pSl9EauyqQ7HZ44YXNVKl8ks2rp9HPowGt\nRvpQ2/t2/hz0sd75qdQ1aI9dFTi//w7/+9/fbN9ymjfszanuewgzbQLPrH6ajz9M4QO7naP+/sRp\nCUapDGmPXWWL1WqlU3g4ncLDc30Nz3Pn4PXXoWVL8LRPZaK9KUNvOc0jy5fzwcph+Px4mvNHbOz2\n8uJjTepKXZUmdpUlVquV5g0a8GibNrRdsIC2Cxbk6gLNS5ZA/fqweTOsXw8lvWKxIPDww8z+YTR7\nDu+gyLZSxIeF6ZBGpTKT3ZU5cvpAV1AqtFJXLEpb4LlIEZEmTXJlgeYTJ0SefFKkYkWR2bP/O5+f\np6cEgGya8pmUffUmKVnTQ1chUjckcrCCkvbYVaYmxcTQ3WbjaOqOli0hMRFPP78cH1MEZs6EOnUc\ni15s3gzt2/93vveTk5kGTCy2guIHPajrfYf20pXKIr14qq5w+dS0W3btIhEoA/QHgu6+m/qenqx8\n5BEOr1wKW2nLAAAY9ElEQVRJaIMGpABlAwKyNJXtnj3wzDPw11/w7bcQEpJxu2oVvJj+59e8Nvtf\nfm6oC1oolVXaY1eXsFqtRHbokFZH79G2LXt27uTr9u2J69cPO3CueXNmTJlCh0ceofe6dfRat46t\n69ZRNZO6e0oKjB0LDRtCmTLbqFG2DaNe/+8ibOqF2e27dvE80K3zTTxd/mFeP5FCg5Yt8+9NUKqw\ny27tJqcPtMZeKHQMC3PU0Z2PWJD7AgJELl4UEZEZgYFy5p9/JOS22+Tc8eMi5cuntasOEnWVuvtH\nH/0qfj5bJdB/rbz44sdS1stLYp0/V9bLS6Kjo6Wsl5d0AqkIclNtpMqoyjK2Z4+rHlOpGwE5qLFr\nKUZl6rlRo5AxY0ioaGPm85UZNy2CjaG7efzzToS+3Ym2fwjV9v1N002biEtKotaxY2k/+913C3mm\nzwEOHongcQbTnDgGjLMQY7cTmdrIZmP4mDE0t9lYALzrCW918mbAbVGcHNSPO4DdLvi9lSqstBSj\nLnH5up7zH3iASk3q03jdyzz7+wc0btGZBueCubggmU0b9rPE5wS1b4plfMh5PlqUwEdly6b1FkaM\nWEPHDjVIPmLhA+7g0zI/0rNXL2ra7RmeewXwvoGEHjfx4B3tqDxsChNE9A5TpbJJE7u6REREBM8P\nHcp75coRvGoV3V57jGZxIew7dpF2P1ZicMvBdF5xnPo77Lz3xU7uWF6eqmPPsH/J19SdUJd/Oodw\nk28NunWDt0eU5yXpSwt64MtR6NoVPv+cvu3b85LFcsmi0M0feogzRWDO4+U5dlcdPlroyfHVq0n2\n8NBx60plV2a1GqA4js7UeuB3YGQGbUKBU8A65+PVDNrkSz1KXZ/o6GgpZbHIz926yawv3hYzEKlY\n3Tl+3WIR2bpVZpYrJyHOGnwCSICzVj6iVzPxi/YXrxbjZMBAu7S95yGJAmkGUgpk348/yoZ33pGj\nhw9L//79pbq/v1T395fHIh8Tv7qeUvx5JOSDxjK3W0eJBfEFiY6OdvVbopRLkRc1dhE5Z4xpJSJn\njTEewFJjTHMRWXpZ0yUi0jYXvmuUi1itVmJee42xIpS8sxKdtg5nwDcw46gPAz3PQ3Iy1KrFQE9P\nLnh6EpecDECKpycfV7+fAz++xsJlXtz7dyt+Kh9HK99W/N/iudQDbileHJ+mTWnbrx8DK1emc8f2\neBY9wY6NVqbLFHxbwrfHG9Hi1XiCBt3CaQ8PBg0bpnOsK5UDWbp4KiJnnZueQBHgRAbNsrWKtip4\nhg8ZghHhtLeh3cVpvL86gAu79nJXWAh9oqIYPmQIh/bs4fbKlQnr1In4JUuw2z1oV/EDZs+ujN02\nlMCNLZl9uDU99y7m49of0/7t/9Grakfmf/EFs5bOJKn6NrovehTPW4pRpVI1Kt5yOwGfHqRjlXbc\nP34itGnDa+fPEx8WpkldqRzKUmI3xliAtUB1YLyI/H5ZEwGaGmOSgP3AgAzaqAIo/c1Ie3fsoEcR\neLNPSbqXbc2FebG8AAxyjiHfsWUL79pscOIEg7dsYfCIxXz2WQhVikPTet3psvQLbpl1mFumfsGm\nnTv5bNkc3ts+ikN/78fcL3yycgBFTkPUAhiSfBM+Pdtz6rHHsKbMp+XDDzPmvvsovX694+KtXixV\nKueyU7cBSgLLgdDL9vsAJZzb9wPbMvjZPKxCqZyIjo6W0haLhDjHn/uBNGuPPDCmsXz05BMSAlLb\nOYY8/fj2k/jKPXwkxT2PycyZInZ7uvHvxoh4e0ssiD/Ipxbkw9fvk7j1cTKzVnXxT51vxvlY+OCD\n8vehQ/JdvXpS3d9fOoaF6ZwwSqVDXo9jF5FTxpgfgDuBxHT7T6fb/tEY83/GGH8RuaRkM2zYsLTt\n0NBQQkNDs3N6lYusVisxr7/OWOfQw8FAxWawpqIn+7vO4scat7Mb6M6lY8hn057n+ZAazKN10948\n/PC3gGOYZOTSpWCzwZkzDPbyohjgYbPx3HuJsOg0cVt3chEY6OnpqNcDg7//nikVKvD3xYsEhYXx\nzfz5+fcmKFUAJSYmkpiYeH0HySzzAwGAn3PbC/gZuPeyNmUB49xuBPyZwXHy+otNZUNqDzsBpIef\nn4xPeF8qjC4v2w5uldHDh6f14st6eUlCQoJMm/aTFLN8Jzfzhwzh7rT96SUkJKT17hMSEiQ6Olp8\nnT30WOcol8jISElISJCWwcFS2mKRqHR3n2pPXakrkYMee1YS+x046uvrgQ3AQOf+vkBf5/azwCZn\nm9+AkAyOk09vg8qKjmFh0gmktMUi1snvSam3vMW/ajGJdib7EJDq/v7yww8J8tFHIgEBIt26bZd2\n97bJcrkkISFBbvLwkIqp0wR4XDr17uVfBEqpK+Uksaf2svOcMUby61wqcyNGjGD0q6/yw2uD6e0z\ng/5f78djZQoTgKdw3DQ0bMwC4uKa4eEBkybBbbdl7xydwsMdC3I4n8cB8VpuUSpbjDGISLZGHeqd\npzeotUuWUMfDg28anCHo9xM8vjIFgKP+/sy+5wFatdvIa681o1cvx+pG2U3qSinX0UnAbkBWq5Wk\nNWto0j6UTzdP5c1vTjMFeMliYfCgBD799C6CS8OGDVCuXM7Pc8kFVdBhjErlEy3F3GBS51t/12Zj\n9gct2L/7FOfeT2K/CaBK/YUcOxbExx/DQw/l3vnSL9qhc74olT05KcVoYr/BNG/QgIvr1nFzcBlW\ndLXw1oh/GHqhOxc8x/HYY8UYPhx8fFwdpVIqVU4Su5ZibiBWq5Xfk5IYA4x/tDRNz9Qj+mQvznvX\nYH5CMRo1cnWESqncoBdPbyAjhwxhrN1Oyf8FccYTEt/7gKNmMV/M2KlJXSk3oon9BmG1WtmUlMSR\nihEMbHqWusueYbCtKXfVX8ADD4S7OjylVC7SUswNwGq18vSjvREZzeHRFrw3f859M58nymLhy5Hz\nXB2eUiqX6cVTN2e1WunSdgJFksfyZNff+aRuD6qNO4HHISgSHMzStWtdHaJS6hr0BiV1iUOH4MnI\nYngmj2Km59OYjnNpuMcTj0Ow28uL10aOdHWISqk8oIndDdnt8OmnUK8eGHYRzR3UfK4Ck/74gjbT\nD3HU31/XEVXKjWkpxs1s2QJ9+8K5c9Cz56+88uI9eCQn03pqB+w//MZP3/7N1Ph4TepKFRJairmB\nnT8Pb70FzZtD587w22/wxSfPUzs5mWYtb2Xx/qXs/vYwt9apo0ldKTenid0NLF0KwcGwZg2sWwfP\nPw8LFzpuRupjYPe95wiTUFokQ9mAAFeHq5TKY5rYC7GTJ+Gpp+CRR2D4cJgzBypVcrw2KSaGsXY7\n5ztWxqfMzTw47hc+Axo41y9VSrkvTeyFkAh8/TXUqQPGwObN0KkTzJ9vpVN4OJ3Cwzl+7BinboLX\nGp5i/PIAzh88REUc0/Uqpdyb3qCUx3J7dsO9e+HZZ2HHDvjqK2jW7L/zpM7aCPCcMex6phxdq9zP\n2qGTGcyV65cqpdyT9tjzkOPmoLYcWLCAAwsW0KVtW0aMGJHWq7Zardf82fTtLl6EceMctfS77nLU\n0lOTOjhKL+/abEQCkcCTNYSUSsKp708zWoTuwDQvL/rofOhKuT3tseeh4UOG4JGcTDPgV8AkJ/Pm\nq6/yifP1yKVLMxxPfnnvu+vPpwioHMLNN5fk11+hVq1rn3e/D0x5tASfNB3NmVceY6m/P7sbNiRO\n50NX6oagiT0PHdqzh0hgGvCuc9+rvr5UffFFWvj4wMCBTIqJuSTZWq1Wnu3alao2G354sYzXuXD+\ncbw9PuOnn/pjSfdvrPRlHp/y5elnDHYPCxOG1qRrlXsIeOxVTtvtBDVsqOuMKnUD0cSeR6xWKxcu\nXOBXHEn90bIBLH88nGmPdmf98vU0aNCM2s8+S/y2bZf8TGpPfROt6cQEmrOS4dTj5wr1sFj6p7Ub\nPmQIvyclMdZuB+AZwACfD6qLpXhRvnr8c4qdPcc0XY5OqRuO3nmai1J70IePHWPb5s08lpzM58BL\nITcz9ZESeNsucvrgYf6Uc3jfVITR7SZQ13IHjRs3Tuup+52w4EcMO2hBR55hGT+y28uLuNmzAdIS\nei27nadw1NOtwBMGvLqXwjfkFhaPO8WcrX8yyMODKd9/r+UXpQoxXUHJhUaMGEHM668z1m5nAvBh\nYCAPRr/BEY9E/u/EUvrvv50yo7/nBaBiuXJUCanI8F+jCakSQpdTXejbvivtznVmJqM4zzQGUZfq\n/Eu8vz9x06cDENmhA1VtNsYC8c7zLmvYkMd3bSDwITt+TWpTe3kVlm/9EgAfX19N6krdiEQkXx6O\nU7mfhIQEaRYcLKVAYh1DzOW9Tp1k5abFUvPNAOneJ0CeuL2qlAGpCFLC8S8XERE5m3xW+if0l4CR\ngVL59jclmDWymgYSC+IPck/TpmK1WkVEpGNYmMSCdHSeJwHk/saNZcb6L6XMO6Xl1W+elfMfj5NY\nkBAQX5Do6GhXvS1KqVzizJ3ZyrfaY78OqTXxqjYbaQNV7ruPwGdb0Gx6OD2/T6HZBhjMMaYAh4DX\nvL3Tft4DL05+1ZfBEQ8w5pEeVN/xDmcD28AnW3gl8nGiPvww7TxJa9bQFugDdAFuK+lJ+afL0fur\n3gzfWod+Xy2DtR8DsLNIEQa9+SZDhw7NvzdDKVVgaGK/Dqljx78vUgS5eJHnDSxrV4K5i16lyGcp\nzL/gzyxO8ASOpP4CMGjwYABWroTeveHQ/lP4zIli8bL5fL1lKk/t+JZ9r3jQptIRPpg+hruqhNCt\nTRvK2e28APQsCl4tS7KtpVDt5BlSRp/hreQV+DljGujpqbM3KnWD08R+nYzFwoSEBGZt28Rfez8g\n6eJfhIxPoeoJ2B3WkAYtWzJ5zBjmAIP69+eFF4bSr5/jrtExY2DW50MJXrieE490p0aPHvw1YAc9\n2rdm0c8/8G292dQ8fiuNIww3V76dkreX5PPTG2lW7k7etbciqN9I4pJhgJcXw728qFi5MlNHjtSk\nrtQNThP7degTFcWqRo3Yy2k+tn9Gg/pN+OS1FXiesBGH4/b9oUOHppVE5s51zO8SFuaY38XfH/z9\no4j8dSnvrl/PO+vXUxxoPHsBjYEoKzwZ15UzzX/m7B21KTt7KccTvXl09yKCWJQWR0kvL3YcP+6K\nt0ApVRBltyif0wduevF0+5HtctPLXlKrR1UJaxIiUc6Lm2W9vCQhIUFERA4cEOncWaRmTZHFi688\nRkJCgnQMC5MyHh5pF2DFeZzWd9zhaPTWWxILUg4kwPlarHO7WXBwPv7GSqn8RA4unmpivw77Tu2T\nSqMqiffdRdMSbSmLRZoFB0tCQoJcvCgyYYJIQIDIq6+K2GzXPl7L4OArEnspkPlhYTLF01MCQKJB\n/JwjX0JA/Dw9075AlFLuJyeJXUsx2WC1Whk5ZAj79uzhjYmjGbYvmpKbizP8lwtEpjay24kPCKBS\npQhatHCsP/rTT1C3bubHHzJyJD3atoXkZAAGAlHA6wsWcABIBioCTwCxFgt1g4KYoTV1pdRldHbH\nLLJarfRo25Ze69YxKqI5b+6N5ti8A5RO8r6kXTLF+H1nD1q2hG7dHKsbZSWpA0RERDA1Pp7JwcFE\nWSw8hiOR7wD+B1g8PZkcHMzusDC+mDePxLVrNakrpa6gUwpkUafwcNouWMBD3kVo8XZNgn47xa4Z\nBykSHMyOLVt412ZjCy0YZSbRuIkfX31VlgoVcn6+1OkJjh87RgqOJe1yYz53pVTholMK5IMS3qUY\nbG+CfcZkJuBIuC9OmcvAF89z+NhdvDJoH2+9lcm8ulkQERGhSVwplSPaY8+i1FLMKGf9ewBwoagn\nzwxYSWxsEB07wttvg6+va+NUSrmXnPTYNbFnQ/qLp37lQrCUiMVmC2TSJGjSxNXRKaXcUa4ndmNM\ncWAJUAzwBL4TkSEZtBsH3A+cBXqKyLoM2hT6xA6QkgIffggjRkD//jBgAHh6ujoqpZS7yvUau4ic\nM8a0EpGzxhgPYKkxprmILE130jZADRGpaYxpDIwHQnLyCxQGJ07Azz/DsmVQs6aro1FKqStlevFU\nRM46Nz2BIsCJy5q0BeKcbVcYY/yMMWVF5HCuRlpAlCkDzjUvlFKqQMp0HLsxxmKMWQ8cBn4Skd8v\na1IB2Jvu+T4cw6+VUkq5QFZ67HagvjGmJGA1xoSKSOJlzS6v/2RYTB82bFjadmhoKKGhodmJVSml\n3F5iYiKJiYnXdYxsjYoxxrwG2ERkdLp9E4BEEZnhfL4FaHl5KcZdLp4qpVR+ysnF02uWYowxAcYY\nP+e2FxAGXD7iJR54zNkmBDjprvV1pZQqDDIrxZQD4owxFhxfAlNFZJExpi+AiEwUkXnGmDbGmB3A\nv0CvvA1ZKaXUtegNSkopVYDleilGKaVU4aOJXSml3IwmdqWUcjOa2JVSys1oYldKKTejiV0ppdyM\nJnallHIzmtiVUsrNaGJXSik3o4ldKaXcjCZ2pZRyM5rYlVLKzWhiV0opN6OJXSml3IwmdqWUcjOa\n2JVSys1oYldKKTejiV0ppdyMJnallHIzmtiVUsrNaGJXSik3o4ldKaXcjCZ2pZRyM5rYlVLKzWhi\nV0opN6OJXSml3IwmdqWUcjOa2JVSys1oYldKKTejiV0ppdyMJnallHIzmtiVUsrNaGJXSik3o4ld\nKaXcjCZ2pZRyM5kmdmNMJWPMT8aYzcaYTcaYFzJoE2qMOWWMWed8vJo34SqllMpMVnrsF4CXRKQO\nEAI8a4y5LYN2S0Qk2PmIztUoC4DExERXh3BdCnP8hTl20PhdrbDHnxOZJnYROSQi653bZ4A/gPIZ\nNDW5HFuBUtj/OApz/IU5dtD4Xa2wx58T2aqxG2OqAMHAisteEqCpMSbJGDPPGHN77oSnlFIquzyy\n2tAY4w18DfRz9tzTWwtUEpGzxpj7gTnArbkXplJKqawyIpJ5I2OKAt8DP4rI+1lovxtoKCIn0u3L\n/ERKKaWuICLZKnVn2mM3xhjgM+D3qyV1Y0xZ4IiIiDGmEY4vjBPp22Q3MKWUUjmTlVJMM6A7sMEY\ns8657xXgFgARmQh0Bp42xqQAZ4EueRCrUkqpLMhSKUYppVThkWd3nhpj/I0xC4wx24wx840xfhm0\nyfTmp/xmjLnPGLPFGLPdGDP4Km3GOV9PMsYE53eMV5NZ7MaYbs6YNxhjfjXG1HNFnFeTlffe2e4u\nY0yKMaZjfsaXmSz+7YQ6b+LbZIxJzOcQrykLfz8BxpgEY8x6Z/w9XRBmhowxnxtjDhtjNl6jTYH8\n3ELm8Wf7sysiefIA3gMGObcHA+9k0OZmoL5z2xvYCtyWVzFlIeYiwA6gClAUWH95PEAbYJ5zuzGw\n3FXx5iD2JkBJ5/Z9BSX2rMafrt1iHBfzO7k67my+/37AZqCi83mAq+POZvzDgJGpsQPHAQ9Xx+6M\n524cQ7E3XuX1Avm5zUb82frs5uVcMW2BOOd2HND+8gaS9Zuf8ksjYIeI/CkiF4AZQLvL2qT9XiKy\nAvBzXjx2tUxjF5FlInLK+XQFUDGfY7yWrLz3AM/jGHZ7ND+Dy4KsxN8V+EZE9gGIyLF8jvFashL/\nQcDXue0LHBeRlHyM8apE5Bfg72s0KaifWyDz+LP72c3LxF5WRA47tw8D13wTr3HzU36qAOxN93yf\nc19mbQpCgsxK7Ok9AczL04iyJ9P4jTEVcCSb8c5dBekCUVbe/5qAv7P8uNoY0yPfostcVuL/BKhj\njDkAJAH98im23FBQP7c5kelnN8s3KGXEGLMARznlckPTPxERudY49kxufspPWU0Ulw/dLAgJJssx\nGGNaAY/jGPFUUGQl/veBl51/T4aCNY1FVuIvCjQA7gVKAMuMMctFZHueRpY1WYn/FWC9iIQaY6oD\nC4wxQSJyOo9jyy0F8XObLVn97F5XYheRsGsEcNgYc7OIHDLGlAOOXKVdUeAbYJqIzLmeeHLBfqBS\nuueVcHyzX6tNRec+V8tK7DgvunwC3Cci1/qna37LSvwNgRmOnE4AcL8x5oKIxOdPiNeUlfj3AsdE\nxAbYjDE/A0FAQUjsWYm/KTACQER2Om9ErAWszpcIr09B/dxmWXY+u3lZiokHIp3bkTimGbhEVm5+\nymergZrGmCrGGE/gERy/R3rxwGMAxpgQ4GS6kpMrZRq7MeYW4Fugu4jscEGM15Jp/CJSTUSqikhV\nHP/Ce7qAJHXI2t/Od0BzY0wRY0wJHBfxfs/nOK8mK/FvAVpD2k2JtYBd+RplzhXUz22WZPuzm4dX\nef2BhcA2YD7g59xfHvjBud0csOO4Ar/O+bjPxVen78cxOmcHMMS5ry/QN12bj5yvJwENXBlvdmIH\nPsUxkiH1vV7p6piz+96nazsZ6OjqmHPwtzMAx8iYjcALro45m38/AcBc59/9RqCrq2NOF/uXwAEg\nGce/jB4vLJ/brMSf3c+u3qCklFJuRpfGU0opN6OJXSml3IwmdqWUcjOa2JVSys1oYldKKTejiV0p\npdyMJnallHIzmtiVUsrN/D80NEfLYXJV9wAAAABJRU5ErkJggg==\n",
       "text": [
        "<matplotlib.figure.Figure at 0x1061eca10>"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# fidge regression"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### \u72c0\u6cc1\u4e00 \u7576 feature > n(\u6a23\u672c\u6578\u91cf)\uff0cinverse matrix \u4e0d\u5b58\u5728\n",
      "#### \u5f15\u5165 Ridge regression \u89e3\u6c7a\u8a72\u554f\u984c\n",
      "#### \u539f\u5148\n",
      "\\\\[ w =\\({x^Tx})^{-1}{x^Ty}\\\\]\n",
      "#### \u5f15\u5165 lambda\n",
      "\\\\[ w =\\({x^Tx+{\\lambda}I})^{-1}{x^Ty}\\\\]"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def ridgeRegres(xMat,yMat,lam=0.2):\n",
      "    xTx = xMat.T*xMat\n",
      "    denom = xTx + eye(shape(xMat)[1])*lam\n",
      "    if linalg.det(denom) == 0.0:\n",
      "        print (\"This matrix is singular, cannot do inverse\")\n",
      "        return\n",
      "    ws = denom.I * (xMat.T*yMat)\n",
      "    return ws\n",
      "# lambda \u7531 exp(i-10) \u53bb\u8abf\u6574\u7531\u975e\u5e38\u5c0f\u5230\u975e\u5e38\u5927\n",
      "def ridgeTest(xArr,yArr):\n",
      "    xMat = mat(xArr); yMat=mat(yArr).T\n",
      "    yMean = mean(yMat,0)\n",
      "    yMat = yMat - yMean     #to eliminate X0 take mean off of Y\n",
      "    #regularize X's\n",
      "    xMeans = mean(xMat,0)   #calc mean then subtract it off\n",
      "    xVar = var(xMat,0)      #calc variance of Xi then divide by it\n",
      "    xMat = (xMat - xMeans)/xVar\n",
      "    numTestPts = 30\n",
      "    wMat = zeros((numTestPts,shape(xMat)[1]))\n",
      "    for i in range(numTestPts):\n",
      "        ws = ridgeRegres(xMat,yMat,exp(i-10))\n",
      "        wMat[i,:]=ws.T\n",
      "    return wMat"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u8f09\u5165\u65b0\u7684\u6578\u64da\n",
      "fileName = '/Users/wy/Desktop/abalone.txt'\n",
      "dataMat,labelMat = loadDataSet(fileName)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dataMat[1:3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "[[1.0, 0.35, 0.265, 0.09, 0.2255, 0.0995, 0.0485, 0.07],\n",
        " [-1.0, 0.53, 0.42, 0.135, 0.677, 0.2565, 0.1415, 0.21]]"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "labelMat[1:3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 17,
       "text": [
        "[7.0, 9.0]"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# lambda \u7531\u975e\u5e38\u5c0f\u5230\u975e\u5e38\u5927\u5f97\u51fa\u7684w\u670930\u7a2e \u81ea\u884c\u5f9e\u5176\u4e2d\u627e\u65b9\u5dee\u6700\u5c0f\u9078\u70ba\u6700\u9069\u5408\u4e4bmodel\n",
      "ridgeTest(dataMat,labelMat)[1:3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 18,
       "text": [
        "array([[ 0.04304419, -0.02274164,  0.13214088,  0.02075182,  2.22403626,\n",
        "        -0.99895275, -0.11725417,  0.16622934],\n",
        "       [ 0.04304419, -0.02274164,  0.13214088,  0.02075182,  2.22403305,\n",
        "        -0.99895211, -0.117254  ,  0.16622966]])"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ridgeTest(dataMat,labelMat)[28:30]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "array([[ -8.34109967e-06,   9.05722756e-04,   1.13283336e-03,\n",
        "          2.59497753e-03,   2.15072069e-04,   3.65888384e-04,\n",
        "          8.94835365e-04,   8.86543150e-04],\n",
        "       [ -3.13618246e-06,   3.43488557e-04,   4.29265642e-04,\n",
        "          9.86279863e-04,   8.16188652e-05,   1.39858822e-04,\n",
        "          3.40121256e-04,   3.34847052e-04]])"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### \u70ba\u4e86\u8b93data\u53bb\u6389bias\uff0c\u964d\u4f4e\u6975\u7aefdata\u7684\u5f71\u97ff\uff0c\u6240\u4ee5\u5fc5\u9808\u8981data\u9032\u884cnormalization\n",
      "###z-score \u6a19\u6e96\u5316(zero-mean normalization)\n",
      "#####\u4e5f\u53eb\u6a19\u6e96\u5dee\u6a19\u6e96\u5316\uff0c\u7d93\u904e\u8655\u7406\u7684\u6578\u64da\u7b26\u5408\u6a19\u6e96\u5e38\u614b\u5206\u4f48\uff0c\u5373\u5e73\u5747\u503c\u70ba0\uff0c\u6a19\u6e96\u5dee\u70ba1\uff0c\u5176\u51fd\u6578\u70ba\uff1a\n",
      "\\\\[ \\dot{x} =\\frac{  x-u}{ \\sigma }\\\\]  \n",
      "#####\u5176\u4e2d\u03bc\u70ba\u6240\u6709\u6a23\u672c\u6578\u64da\u7684\u5e73\u5747\u503c\uff0c\u03c3\u70ba\u6240\u6709\u6a23\u672c\u6578\u64da\u7684\u6a19\u6e96\u5dee\u3002\n",
      "---\n",
      "### \u72c0\u6cc1\u4e8c \u5982\u4f55\u6e1b\u5c11\u8a08\u7b97\u91cf\uff0c\u53e6\u4e00\u7a2e\u8caa\u5fc3\u6f14\u7b97\u6cd5\n",
      "#### \u4e00\u958b\u59cb\u6240\u6709w\u7684\u6b0a\u91cd\u70ba1\uff0c\u6bcf\u4e00\u6b65\u5c0d\u6bcf\u500bw\u7684\u6b0a\u91cd\u589e\u52a0\u6216\u662f\u6e1b\u5c11\u4e00\u500b\u6b65\u9577\uff0c\u7136\u5f8c\u8a08\u7b97\u65b9\u5dee\uff0c\u65b9\u5dee\u8b8a\u5c11\u5c31\u57f7\u884c\u4e0b\u53bb\u76f4\u5230\u6536\u6582  "
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# zero-mean normalization \uff08x-\u5e73\u5747/\u6a19\u6e96\u5dee\uff09\n",
      "def regularize(xMat):#regularize by columns\n",
      "    inMat = xMat.copy()\n",
      "    inMeans = mean(inMat,0)   #calc mean then subtract it off\n",
      "    #\u6a19\u6e96\u5dee The variance is the average of the squared deviations from the mean, i.e., var = mean(abs(x - x.mean())**2).\n",
      "    inVar = var(inMat,0)      #calc variance of Xi then divide by it\n",
      "    inMat = (inMat - inMeans)/inVar\n",
      "    inMat = np.nan_to_num(inMat)\n",
      "    return inMat"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u65b9\u5dee\u8a08\u7b97 \n",
      "def rssError(yArr,yHatArr):\n",
      "    return ((yArr-yHatArr)**2).sum()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Forward stagewise linear regression\n",
      "def stageWise(xArr,yArr,eps=0.01,numIt=100):\n",
      "    xMat = mat(xArr); yMat=mat(yArr).T\n",
      "    yMean = mean(yMat,0)\n",
      "    yMat = yMat - yMean     #can also regularize ys but will get smaller coef\n",
      "    xMat = regularize(xMat)\n",
      "    m,n=shape(xMat)\n",
      "    returnMat = zeros((numIt,n)) \n",
      "    ws = zeros((n,1)); wsTest = ws.copy(); wsMax = ws.copy()\n",
      "    for i in range(numIt):\n",
      "        # print ws.T \u986f\u793a\u884c\u8d70\u7d93\u904e\n",
      "        # print ws.T\n",
      "        #inf \u6b63\u7121\u7aae\n",
      "        lowestError = inf; \n",
      "        # run every feature\n",
      "        for j in range(n):\n",
      "            for sign in [-1,1]:\n",
      "                wsTest = ws.copy()\n",
      "                wsTest[j] += eps*sign\n",
      "                yTest = xMat*wsTest\n",
      "                rssE = rssError(yMat.A,yTest.A)\n",
      "                \n",
      "                if rssE < lowestError:\n",
      "                    lowestError = rssE\n",
      "                    wsMax = wsTest\n",
      "        ws = wsMax.copy()\n",
      "        returnMat[i,:]=ws.T\n",
      "    return returnMat"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u6b65\u9577\u70ba0.001 iteration 5000\n",
      "stageWise_w = stageWise(dataMat,labelMat,0.001,5000)\n",
      "# \u4ee5\u4e0b\u70ba\u884c\u8d70\u7684\u904e\u7a0b\n",
      "# \u6536\u6582\u7d50\u679c w = 0.044 -0.011  0.12   0.022  2.023 -0.963 -0.105  0.187"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "stageWise_w"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "array([[ 0.   ,  0.   ,  0.   , ...,  0.   ,  0.   ,  0.   ],\n",
        "       [ 0.   ,  0.   ,  0.   , ...,  0.   ,  0.   ,  0.   ],\n",
        "       [ 0.   ,  0.   ,  0.   , ...,  0.   ,  0.   ,  0.   ],\n",
        "       ..., \n",
        "       [ 0.043, -0.011,  0.12 , ..., -0.963, -0.105,  0.187],\n",
        "       [ 0.044, -0.011,  0.12 , ..., -0.963, -0.105,  0.187],\n",
        "       [ 0.043, -0.011,  0.12 , ..., -0.963, -0.105,  0.187]])"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def OLS(xArr,yArr):\n",
      "    # \u56e0\u70ba\u8981normalization \u5728\u5916\u9762\u5c31\u8655\u7406\u6210matrix\u5f62\u5f0f\uff0c\u56e0\u6b64\u8a3b\u89e3\u6389list to matrix\u7684\u904e\u7a0b\uff0c\u5176\u9918\u4e0d\u8b8a\n",
      "    # xMat = np.mat(xArr); yMat = np.mat(yArr).T\n",
      "    xTx = xMat.T*xMat\n",
      "    # \u6709\u4e9bmatrix\u4e0d\u53ef\u8f49\u70ba\u9006\u77e9\u9663\n",
      "    if np.linalg.det(xTx) == 0.0:\n",
      "        print (\"This matrix is singular, cannot do inverse\")\n",
      "        return\n",
      "    ws = xTx.I * (xMat.T*yMat)\n",
      "    return ws"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "xMat = mat(dataMat)\n",
      "yMat = mat(labelMat).T\n",
      "xMat = regularize(xMat)\n",
      "yM = mean(yMat,0)\n",
      "yMat = yMat - yM\n",
      "OLS_w = OLS(xMat,yMat)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "OLS_w"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "matrix([[ 0.0430442 ],\n",
        "        [-0.02274163],\n",
        "        [ 0.13214087],\n",
        "        [ 0.02075182],\n",
        "        [ 2.22403814],\n",
        "        [-0.99895312],\n",
        "        [-0.11725427],\n",
        "        [ 0.16622915]])"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# OLS_w       0.0430442 -0.02274163 0.13214087 0.02075182 2.22403814 -0.99895312 -0.11725427 0.16622915 \n",
      "# stageWise_w 0.044     -0.011      0.12       0.022      2.023      -0.963      -0.105      0.187\n",
      "# \u5169\u8005\u7b97\u51fa\u4f86\u7d50\u679c\u5dee\u4e0d\u591a\n",
      "yHat_OLS_w = xMat*OLS_w\n",
      "# stageWise_w[-1] \u6536\u6582\u4e4bw\n",
      "yHat_stageWise_w = xMat*mat(stageWise_w[-1]).T"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u76f8\u95dc\u4fc2\u6578\u9810\u6e2cy\u548c\u539f\u672cy\u6bd4\u8f03\uff0cOLS \u548c stageWise\u5169\u8005\u7684\u5dee\u8ddd\u4e0d\u5927\n",
      "corrcoef(yHat_OLS_w.flatten().A[0],yMat.flatten().A[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 29,
       "text": [
        "array([[ 1.        ,  0.72656103],\n",
        "       [ 0.72656103,  1.        ]])"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corrcoef(yHat_stageWise_w.flatten().A[0],yMat.flatten().A[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 30,
       "text": [
        "array([[ 1.        ,  0.72642036],\n",
        "       [ 0.72642036,  1.        ]])"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u4ea4\u53c9\u9a57\u8b49\u7684\u65b9\u6cd5\uff0c\u5f9edata\u4e2d\u9078\u53d690%\u70batrain data 10%\u70batest data \u53bb\u8dd1\u6f14\u7b97\u6cd5\uff08Ridge regression\uff09\n",
      "# \u91cd\u898610\u6b21 \u5f9e\u4e2d\u53d6\u5f97\u65b9\u5dee\u8f03\u5c0f\u7684w\n",
      "def crossValidation(xArr,yArr,numVal=10):\n",
      "    # random 90%->train 10%->test\n",
      "    m = len(yArr)                           \n",
      "    indexList = range(m)\n",
      "    errorMat = zeros((numVal,30))#create error mat 30columns numVal rows\n",
      "    for i in range(numVal):\n",
      "        trainX=[]; trainY=[]\n",
      "        testX = []; testY = []\n",
      "        random.shuffle(indexList)\n",
      "        for j in range(m):#create training set based on first 90% of values in indexList\n",
      "            if j < m*0.9: \n",
      "                trainX.append(xArr[indexList[j]])\n",
      "                trainY.append(yArr[indexList[j]])\n",
      "            else:\n",
      "                testX.append(xArr[indexList[j]])\n",
      "                testY.append(yArr[indexList[j]])\n",
      "        wMat = ridgeTest(trainX,trainY)    #get 30 weight vectors from ridge\n",
      "        for k in range(30):#loop over all of the ridge estimates\n",
      "            matTestX = mat(testX); matTrainX=mat(trainX)\n",
      "            meanTrain = mean(matTrainX,0)\n",
      "            varTrain = var(matTrainX,0)\n",
      "            matTestX = (matTestX-meanTrain)/varTrain #regularize test with training params\n",
      "            yEst = matTestX * mat(wMat[k,:]).T + mean(trainY)#test ridge results and store\n",
      "            errorMat[i,k]=rssError(yEst.T.A,array(testY))\n",
      "            #print errorMat[i,k]\n",
      "    meanErrors = mean(errorMat,0)#calc avg performance of the different ridge weight vectors\n",
      "    minMean = float(min(meanErrors))\n",
      "    bestWeights = wMat[nonzero(meanErrors==minMean)]\n",
      "    return bestWeights"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# ordinary least squares,Forward stagewise linear regression,Ridge regression(\u52a0\u4e0acrossValidation)\n",
      "# \u4e09\u7a2e\u65b9\u5f0f\u9810\u6e2c\u51fa\u4f86\u7684\u7d50\u679c\u90fd\u6975\u5176\u76f8\u4f3c\n",
      "crossValidation_w = crossValidation(dataMat,labelMat)\n",
      "yHat_crossValidation_w = xMat*crossValidation_w.T"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corrcoef(yHat_crossValidation_w.flatten().A[0],yMat.flatten().A[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 33,
       "text": [
        "array([[ 1.        ,  0.72651684],\n",
        "       [ 0.72651684,  1.        ]])"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corrcoef(yHat_OLS_w.flatten().A[0],yMat.flatten().A[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 34,
       "text": [
        "array([[ 1.        ,  0.72656103],\n",
        "       [ 0.72656103,  1.        ]])"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corrcoef(yHat_stageWise_w.flatten().A[0],yMat.flatten().A[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 35,
       "text": [
        "array([[ 1.        ,  0.72642036],\n",
        "       [ 0.72642036,  1.        ]])"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# \u53c3\u8003\u4f86\u6e90 Machine Learning in Action"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 36
    }
   ],
   "metadata": {}
  }
 ]
}