ch12ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np               \n",
    "import matplotlib.pyplot as plt  \n",
    "import pandas as pd\n",
    "from sklearn import tree\n",
    "from sklearn import ensemble\n",
    "from sklearn import datasets "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 決定木"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = datasets.load_wine()              # wineデータ読込み\n",
    "dt = tree.DecisionTreeClassifier()    # 決定木\n",
    "dt.fit(d.data, d.target)              # フィッティング\n",
    "1-dt.score(d.data,d.target)           # トレーニング誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.022471910112359605"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.set_params(max_depth=3)            # 木の深さの最大値を設定\n",
    "dt.fit(d.data, d.target)              # フィッティング\n",
    "1-dt.score(d.data,d.target)           # トレーニング誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import graphviz            # graphviz.Source を使う\n",
    "# graphvizで決定木を描画\n",
    "graph = graphviz.Source(tree.export_graphviz(dt,out_file=None))\n",
    "# graph.render(\"DTplot\")     # 図をファイルに出力"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# バギング"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 4, 2, 0, 1])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1,2,3,4,5 の復元抽出(リサンプリング)\n",
    "np.random.choice(5,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 4, 2, 3])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 0，1,2,3,4 の非復元抽出\n",
    "np.random.choice(5,5,replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.020000000000000018"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = datasets.load_iris()                 # データ読込み\n",
    "i = np.random.choice(d.data.shape[0],100,replace=False)\n",
    "ti = np.delete(np.arange(d.data.shape[0]),i)\n",
    "x,    y = d.data[i,:],   d.target[i]        # トレーニングデータ\n",
    "tx, ty = d.data[ti,:], d.target[ti]      # テストデータ\n",
    "ba = ensemble.BaggingClassifier(n_estimators=100)\n",
    "ba.fit(x,y)                         # バギング(B=100)\n",
    "1-ba.score(tx,ty)                   # テスト誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.020000000000000018"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt = tree.DecisionTreeClassifier()  # 単独の決定木を学習\n",
    "dt.fit(x,y)\n",
    "1-dt.score(tx,ty)                   # テスト誤差"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ランダム・フォレスト"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08108108108108103"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d  = pd.read_csv('data/stagec.csv').values   # データ読込み\n",
    "xa, ya = np.array(d[:,:6]).astype('float'), d[:,7]\n",
    "i = ~np.isnan(xa).any(axis=1); xa, ya = xa[i,:], ya[i] # nan 除去\n",
    "x, y    = xa[:100,:], ya[:100]          # トレーニングデータ\n",
    "tx, ty = xa[100:,:], ya[100:]          # テストデータ\n",
    "# ランダムフォレスト：木の数 10\n",
    "rf = ensemble.RandomForestClassifier(n_estimators=10)\n",
    "rf.fit(x,y)\n",
    "1-rf.score(tx,ty)    # テスト誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.027027027027026973"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ランダムフォレスト：木の数 1000\n",
    "rf.set_params(n_estimators=1000)\n",
    "rf.fit(x,y)\n",
    "1-rf.score(tx,ty)    # テスト誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.054054054054054057"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt = tree.DecisionTreeClassifier()      #  決定木\n",
    "dt.fit(x,y)\n",
    "1-dt.score(tx,ty)    # テスト誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.027027027027026973"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ba = ensemble.BaggingClassifier(n_estimators=10)  # バギング(B=10)\n",
    "ba.fit(x,y)\n",
    "1-ba.score(tx,ty)    # テスト誤差"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ブースティング"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n = 4000                               # トレーニングデータ数\n",
    "d = pd.read_csv('data/soldat.csv').values;  # データ読込み\n",
    "d=np.delete(d,70,1)                    # nanを除去\n",
    "i  = np.random.choice(len(d),n,replace=False)  \n",
    "ti = np.delete(np.arange(len(d)), i)           \n",
    "x, y  = d[i,:71], d[i,71]             # トレーニングデータ\n",
    "tx, ty = d[ti,:71], d[ti,71]        # テストデータ\n",
    "T = 1000                                    # ブースティング：ラウンド数\n",
    "bl = tree.DecisionTreeClassifier(max_depth=1)               # 決定株\n",
    "# 決定株でアダブーストを実行\n",
    "ab = ensemble.AdaBoostClassifier(base_estimator=bl, n_estimators=T)\n",
    "ab.fit(x,y)                                # フィッティング\n",
    "# 各ラウンドの誤差\n",
    "ada_tr_err = np.zeros((T,))            \n",
    "for i, score in enumerate(ab.staged_score(x,y)): \n",
    "    ada_tr_err[i] = 1-score                     # トレーニング誤差\n",
    "ada_te_err = np.zeros((T,))            \n",
    "for i, score in enumerate(ab.staged_score(tx,ty)): \n",
    "    ada_te_err[i] = 1-score                     # テスト誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4VFX6wPHvSe89QCAEAoQOUiK9\nilQVFRHFhmVFXRdXXRu6NtTVXVdX3bWhIoqKIiryU1AEKaL0Ir1LCTWFJKSXOb8/ziSZSR1CJpPy\nfp4nT+bWOTd3ct85XWmtEUIIISrj5uoECCGEqPskWAghhKiSBAshhBBVkmAhhBCiShIshBBCVEmC\nhRBCiCpJsBBCCFElCRZCCCGqJMFCCCFElTxcnYCaEhERoVu3bu3qZAghRL2yadOmJK11ZFX7NZhg\n0bp1azZu3OjqZAghRL2ilDriyH5SDCWEEKJKEiyEEEJUSYKFEEKIKjWYOgshRN2Rn59PQkICOTk5\nrk6KsPLx8SE6OhpPT89qHS/BQghR4xISEggMDKR169YopVydnEZPa01ycjIJCQnExsZW6xxSDCWE\nqHE5OTmEh4dLoKgjlFKEh4dfUE5Pchand0JWMuSkQdwo8PB2dYqEaBAkUNQtF3o/JFh8ei2kHzev\n798OITGuTY8QQtRBUgzlE1LyOjvVdekQQtSY1NRU3nrrrWodO27cOFJTK38WPPXUUyxdurRa56+v\nJFj4BJe8zklzXTqEEDWmsmBRWFhY6bGLFi0iJCSk0n1mzJjBpZdeWu30na/SaS4oKHDoOEf3c4QE\nC1+bD0WO5CyEaAgee+wxDh48SI8ePXj44YdZsWIFw4cP54YbbqBbt24AXHXVVfTu3ZsuXbowc+bM\n4mNbt25NUlIShw8fplOnTtx555106dKFUaNGkZ2dDcCtt97K/Pnzi/d/+umn6dWrF926dWPPnj0A\nJCYmMnLkSHr16sVdd91Fq1atSEpKKpPWJUuW0L9/f3r16sW1115LRkZG8XlnzJjBoEGD+PLLLxk2\nbBiPP/44Q4cO5fXXX+fIkSOMGDGC7t27M2LECI4ePVqctgcffJDhw4fz6KOP1tjfVOospBhKCKf7\nz0/7eH3Zfof2ndynJS9O6G63bvrX25i7/ljx8l9HxPHAyPYVnuOll15ix44dbN26FYAVK1awfv16\nduzYUdx0dNasWYSFhZGdnc3FF1/MNddcQ3h4uN159u/fz9y5c3nvvfeYNGkSX331FTfddFOZ94uI\niGDz5s289dZb/Pvf/+b999/n2Wef5ZJLLmH69On88MMPdgGpSFJSEs8//zxLly7F39+ff/7zn7z6\n6qs89dRTgOkbsXr1agDeeecdUlNTWblyJQBXXHEFt9xyC1OmTGHWrFncd999LFiwAIB9+/axdOlS\n3N3dK/9jnwcJFlIMJUSj0KdPH7s+Bm+88QbffPMNAMeOHWP//v1lgkVsbCw9evQAoHfv3hw+fLjc\nc0+YMKF4n6+//hqA1atXF59/zJgxhIaGljlu7dq17Nq1i4EDBwKQl5dH//79i7dfd911dvvbLq9Z\ns6b4vW6++WYeeeSR4m3XXnttjQYKkGDBqXwfmllfZ6Un4+fS1AghnMXf37/49YoVK1i6dClr1qzB\nz8+PYcOGldsHwdu7pCm9u7t7cTFURfu5u7sX1xNoratMk9aakSNHMnfu3CrTXN6yLdumsZXtV12N\nPlgs2p/N7dbX6WcTJVgI4QQPjGxfabFRVV6c0L1M0VRlAgMDOXfuXIXb09LSCA0Nxc/Pjz179rB2\n7dpqp60igwYNYt68eTz66KMsWbKEs2fPltmnX79+3HvvvRw4cIB27dqRlZVFQkIC7dtX/bcaMGAA\nn3/+OTfffDOffvopgwYNqvFrsNXoK7jdbCq4C7KkzkKIhiA8PJyBAwfStWtXHn744TLbx4wZQ0FB\nAd27d+fJJ5+kX79+NZ6Gp59+miVLltCrVy8WL15MVFQUgYGBdvtERkYye/ZsJk+eTPfu3enXr19x\nBXlV3njjDT788EO6d+/OnDlzeP3112v8GmwpR7JK9UF8fLyuzuRHn3z0Djf9YVoMHIsYRMu/fF/T\nSROi0dm9ezedOnVydTJcKjc3F3d3dzw8PFizZg333HNPcYW7q5R3X5RSm7TW8VUd2+iLobwDSiq0\n3KSCWwhRQ44ePcqkSZOwWCx4eXnx3nvvuTpJF6TRBwvf4JIWCh756S5MiRCiIYmLi2PLli2uTkaN\nafR1FgEhJfOUexdUXCEmhBCNWaMPFsEhEcWvfQszXJgSIYSouxp9sIgIDSFPm84r3uRBvszsJYQQ\npUmwCPQhjZIOLDq7bFtoIYRo7Bp9sPD1cifDJlhkpKW4MDVCiJpwIUOUA7z22mtkZWXVYIrqv0Yf\nLAAs3iXjQ1myJFgIUd+5OliUHhrcFUOK17RG33QWoG1MNBwwvSaDlXybEKK+sx2ifOTIkbz88su8\n/PLLzJs3j9zcXK6++mqeffZZMjMzmTRpEgkJCRQWFvLkk09y+vRpTpw4wfDhw4mIiGD58uV25960\naRMPPvggGRkZREREMHv2bKKiohg2bBgDBgzg119/Zfz48Wzfvp2wsDC2bNlCr169eOKJJ7j99ts5\ndOgQfn5+zJw5k+7du/PMM89w4sQJDh8+TEREBJ999pmL/mqVc2qwUEqNAV4H3IH3tdYvldp+N3Av\nUAhkAFO11rus26YDd1i33ae1/tFpCZWRZ4VwnmeCq96n2ucu//+19BDlS5YsYf/+/axfvx6tNePH\nj2fVqlUkJibSvHlzvv/ejNyQlpZGcHAwr776KsuXLyciIsLuvPn5+UybNo1vv/2WyMhIvvjiC554\n4glmzZoFYDeE+K233mo3VPi0adPo2bMnCxYs4Oeff+aWW24pTt+mTZtYvXo1vr6+Tvkz1QSnBQul\nlDvwJjASSAA2KKUWFgUDq8+01u9Y9x8PvAqMUUp1Bq4HugDNgaVKqfZa68qnuKoumdNCiAZtyZIl\nLFmyhJ49ewKQkZHB/v37GTx4MA899BCPPvool19+OYMHD670PHv37mXHjh2MHDkSMDPYRUVFFW8v\nPaS47VDhq1ev5quvvgLgkksuITk5mbQ0E+zGjx9fpwMFODdn0Qc4oLU+BKCU+hy4EigOFlpr2y7T\n/kDRQFVXAp9rrXOBP5RSB6znW+OUlMpseUI0aFprpk+fzl133VVm26ZNm1i0aBHTp09n1KhRxRMP\nVXSeLl26sGZN+Y+iyoYUL28cvqJhxZ0xpHhNc2awaAEcs1lOAPqW3kkpdS/wIOAFXGJzrO2YwQnW\ndU6RXOBL0QhRew8fo8NQZ72TEI1QBUVFzlR6iPLRo0fz5JNPcuONNxIQEMDx48fx9PSkoKCAsLAw\nbrrpJgICApg9e7bd8aWLoTp06EBiYiJr1qyhf//+5Ofns2/fPrp06VJlmoYMGcKnn37Kk08+yYoV\nK4iIiCAoKKhGr9uZnBksVDnryoRWrfWbwJtKqRuAvwNTHD1WKTUVmAoQExNT7YSezPMpDhaJiafp\nUO0zCSHqAtshyseOHcvLL7/M7t27i2ehCwgI4JNPPuHAgQM8/PDDuLm54enpydtvvw3A1KlTGTt2\nLFFRUXYV3F5eXsyfP5/77ruPtLQ0CgoKuP/++x0KFs888wy33XYb3bt3x8/Pj48++sg5F+8kThui\nXCnVH3hGaz3aujwdQGv9YgX7uwFntdbBpfdVSv1oPVeFxVDVHaIc4ODKT2m7/M8ArPXqT7/Hf6jW\neYQQhgxRXjddyBDlzuxnsQGIU0rFKqW8MBXWC213UErF2SxeBhTN6L4QuF4p5a2UigXigPXOSmhA\ncElW0zNfBhMUQojSnFYMpbUuUEr9BfgR03R2ltZ6p1JqBrBRa70Q+ItS6lIgHziLKYLCut88TGV4\nAXCv01pCAUFhJcHCz3IOrbXdfLZCCNHYObWfhdZ6EbCo1LqnbF7/tZJjXwBecF7qSvgGhBW/DiST\n9JwCgn09a+OthWiw5EtX3XKhVQ4y3AfYNZ0NIpOkjFwXJkaI+s/Hx4fk5OQLfkCJmqG1Jjk5GR8f\nn2qfQ4b7APAOxoLCDU2QymZ3WhZtIwNcnSoh6q3o6GgSEhJITEx0dVKElY+PD9HR0dU+XoIFgJsb\nOW5++FkyAUg7mwQ0cW2ahKjHPD09iY2NdXUyRA2SYiirHI+SzjHnUpNcmBIhhKh7JFhY5XuWBIus\n9GQXpkQIIeoeCRZWXv6hxa97NZEWHEIIYUuChVVoeGTx6y5h0oJDCCFsSbAoIsOUCyFEhSRYFJEJ\nkIQQokISLIrInBZCCFEhCRZFbIqhlmzay7mcfBcmRggh6hYJFkVsgkVuRgqJ52TIDyGEKCLBoohN\nMVQwmZxOl2AhhBBFJFgUsangDlKZHEvJcmFihBCibpFgUcTHduTZLA4nZ7owMUIIUbdIsChiWwyl\nMiVYCCGEDQkWRWyKoYLJ5HCiBAshhCgiwaKIhzfaw9e8VBYSU1Jk4hYhhLCSYGHLpijKIy+dpIw8\nFyZGCCHqDgkWNpRtUZTK5IjUWwghBCDBwp5vWPHLCJXG4WRpPiuEECDBwl5Eu+KXz/VTjOgoU6sK\nIQTIHNz2mnUvfhlbcAj8vVyYGCGEqDskZ2GradeS16d3uC4dQghRx0iwsNW0S8nrxD1QIONDCSEE\nSLCw5xMEoa3Na0sB2Sd3kV9ocWmShBCiLpBgUVqzbsUvn3xnLtuPy6x5QgghwaK0piXBopM6Wnlf\ni9SjMGcCfHgZHNtQsj7tOHx+I7w9EE5sdWJihRCidkhrqNJschad1RHWJlXQ1yLjDHx8FaQcNMsf\njoXRL0BEe/jqDshKNusXPQR/WurkRAshhHNJsCitWUmLqE5uR/giKaPsPjlp8MmEkkABYMmHxY+U\n3TdhA5zabheEhBCivnFqMZRSaoxSaq9S6oBS6rFytj+olNqllNqmlFqmlGpls61QKbXV+rPQmem0\nE9ySAq8gAEJUJhmJR8z6rBT4YxVs+MAUPZ3abk2oG0R0qPycG2c5McFCCOF8TgsWSil34E1gLNAZ\nmKyU6lxqty1AvNa6OzAf+JfNtmytdQ/rz3hnpbMMpShsUpK78D+7C/Ysgn+3h4+ugO8fhOMbS/Yf\n/1+4axXE316yrtVAmGgTILbNg9xztZB4IYRwDmfmLPoAB7TWh7TWecDnwJW2O2itl2utiyoF1gLR\nTkyPw7xalPTk7pH/O5aF00wxU2mjXoCeN4GnD1z+H7h9CVw/F25ZCF0mlOQ48jJg+5e1lPpadnIb\nfHM37Pne1SkRQjiRM+ssWgDHbJYTgL6V7H8HsNhm2UcptREoAF7SWi8ofYBSaiowFSAmJuaCE1x8\nXpthP27z+BGKwplfOMSNgvB20HY4tOhtf2BMqcuLvx1+eNS83jAL4kbD+pmmw9+QRyC61PH1TfZZ\nmHM1ZCXBti/g3vUQEefqVAkhnMCZwUKVs67c2YSUUjcB8cBQm9UxWusTSqk2wM9Kqe1a64O2x2mt\nZwIzAeLj42tupiKbSm47V70D7Uc5fp6LroOlz0BBNpzeDq91A11otqX8AfeuA1Xen6meWP6iCRQA\n2gKrXoYJM12bJiHqOkuhqcfc9S20HwP9760XzwFnFkMlAC1tlqOBE6V3UkpdCjwBjNdaF4+vobU+\nYf19CFgB9HRiWu1FdsSi7ONoTqeJ5xcoAHxDoes1JctFgQIgaW9JJXl9kJViPtyZ1uBwagdseM9+\nn+1fQtL+2k+bEPXF8U3w3nDTpP7wL7DkCfi/+0wAqeOcGSw2AHFKqVillBdwPWDXqkkp1RN4FxMo\nztisD1VKeVtfRwADgV1OTKs9D29UZPvixQyPEDIveb5657r4Dvtl76CS1zu+qt45a1vyQXizL8y7\nBd7oBevfg8WPmtyEraLcRU04ssYEJ2dNbXvuNGz9DE7X3sdKNFLnTsOm2fDpJHhvBJz83X775o/h\ny1sdH4suPxu2zzf1hJbaG45IOXOeaaXUOOA1wB2YpbV+QSk1A9iotV6olFoKdANOWg85qrUer5Qa\ngAkiFkxAe01r/UFl7xUfH683btxY2S7nZ/GjsO4d83rih9B1QvXPtX2+6W/R9RrTWW/u9WZ9SAz8\ndVvdzoKmHYdZYyDtaPnb3Tzgitfh23vNsnKDG+fDvh9h59eg3E09RkQcdLgM4i61P77o82f7N9jx\nFcy3ti4bNh2GlWl1fX4KcqHQ2kAh/TisfQu2zoXCXHDzhFu+hdYDL+w9LoTFAm7lfG/LTIb9P5qH\nwuHV4OFtOn2GtzO/i/6uwTHlHw+mufePT5hOpD1vgr53Q0Ckc6+noavofpXZr9DkIDZ+SJkSeA8f\nU+d55NeSdTH9TevKiur9slJM0/1175QU/0b3MY1rKio6d4BSapPWOr7K/ZwZLGpTjQeLnDTzDTq8\nLXS5uubOW5AL/44z5wf40zKILuc+WQrNg7cmA4nWYCkAd0/H9s9MNj3Tk/ZWvE+/e2HMP+CTa+CA\nAz3VBz8El/zdXNfu7+CH6SY9V71tGgicPQzvDIbcdLO/pz/cvw38IxxLsy2tYcnfTXAonQuy5RcB\nU1dASMuK97FlsZhAeGxdyTr/JtDvHvAOcDx9hfmw6GHY9KFpPBHRHkJaQcYpSDoA6QmOnSe4JYx6\nDjpfVfJ5yThjrn3bF/b7evhAjxvNPfALK3suR2ltPsPeQY49OOu73HOw6SNY/675AhXaCsLjzEjV\nPW8yzwlbWpvipc0flz1Xu5Ew7l8Q0toUQ619q2SbuxcMvB/aDIPk/aZYN/mA+X32sH1RdhHlbj57\nw6af3+ev6HAJFnXYgnth6yfmddHD1tbSZ2D1f6DbtTDhvZoJGOknYNZoyM2Aa2dDm6EV71uQC79/\nbtJw9g+zzs0Drv0ITu+EX14x38oDo0wlvU+wGRvrg0srPqet+NvBKwB+e6NknaefycH98gokrLff\nf+D9MPLZ87pcAJY+C6tfrXi7ci/552vWHW7/Ebz8SrbnpMG6d80DNv52849YmA8L7im/KXTcaLjh\ni5L7pbUJejZzuxfLz4Yvb4N9i8tuq662I6DT5bD/Jzi43DSsqEh4HNz8TUmALCyA1CMlZef5WWaE\ngqT9kHqs5O9UmAcph0zRZG46NO9p+hSFtak6fbkZ5mHoUU8mFbNYTB3D7m9h08eQW9Ggogo6XQED\npkHzXuDuAT89Bb++XrJLdB/ochV0GGv/t9La/J/9/Hz5gaAigVGm/tC2SX9kR7jnN3BzP6/LlGDh\nBMdSsvh5zxmmDGh9YSc6sMwMFwIQ2Bwe2Fny7ezwaph9Wcm+V880raou1I9PwJr/mdd+EeZDFdi0\n7H4bP4QVL0LGaZuVCq55H7pNNIspf5hcRNzIkiHdAeZNgV3WFs6xQ2HgfRAaa74ZrZ/pWM6jPEW5\nC98w2PwRHPy5JKfgFw5977KfiwTg1zfgpydLlj18THBwczfZ/QHTzOuPrjC5LYCOl8Plr5limlPb\nTR1NyiGzLagFjHre1HMc+KnitI75J/S723wL/PwmSNxt3mvE0yVBJCcNPrsejv5W+XUrd1NU0XEc\ndBhnAnbyAUjaV/KN89T2klxYRbpMMK1u1r0NJ7aUrA9qAZPmmKKQtW/DuTLtTxzjHQxXvw0dL6t4\nn61zTZGMmzuMffn8P9P5OZCZaNJcUU4mK8UU84a0qjwgnTttgl5F5zp32tS97foWMs+U3V4ZN08I\nji75kgVw0WS48q3Kc2Ant8F3D9h3+C1P1EXmC2bXCeaz+f3fTEU5wOgXof+fzy+9SLCoURaL5u2V\nB/nvz/vJybfw5d39ubj1BWThCwvglfYlgw3ethhaDTDr3x0CZ3aW7BvQFP6y0cy1cSHv95/O9gGg\n7SVw41f2H+DSD1gwxQxj/wU9JjvwPvmmriK0VdmxsArzYcGfYfs8+/VtR5gHX+k6kUuehB1fl/wt\net8GaQnlP6g9/eH6T03fF4vFlOn+OL1ke/uxcN2c8ovfNs4y/6RFPHzMQ2/P91CQU/n1dr/OPMyP\nrStprODuZR4MS/5uipOKDH0Mhk831/rFzSaIFBn0gMm5JO03IxkHNDFFUqGtqy4yzE4130o3vE+Z\ncvGm3UyOrN0Is6y1Sec3d5ffyfRC9bzZfKFoNdA+3WvehB8ft983/nYY85IJ1EXFLEn7TdGLtpgv\nGx3GAsrczw0fmG/2/pEm8LUebD7PRUU1SftLyvEDmplimfjb7HN1qUdNXeTeRWbZ088UHzXvZYJx\n7GDzZWDZc+XnIsLamsDf5SpTFJW4x+TAK/vy0GGcCcjuDvRSsFjMl6HNH5tcRkR7kwOMsNZRhbW1\nz/mCuafb5pnixhvmOfY+pUiwqGF3zdnIjzvNw7ZD00C+u28Qnu4XUFb73QMlY0Z1v96U2W94r/zB\nCAdMM99qq8s2J2Nr1PPm3GA+oAunlWwLjIJ+f4bet15YoLJlsZiH+Lp3TH3M8Cdg0INw7qRJX+Ie\ns1/rwabSec935tu9I9y94NJnTc7Gti6h1UC46Svw9K342O8fKtsMuIhXgAkgRQ+iIkMegeGPm9xC\nQS68fymc2lZ5GnveBDsXmB79RUY+Z3JgF+r4Jlj+DxOU40aZB23pcvQiB382uZ78UsPv+4aaXCeY\nXExoa/OgCmtj/gZg7ltwtHmIpSfAvFvLBnrvYFP/FNHeXOum2eWnwzuo6lyRm2f1A5tXILQdZtKB\nMnUD+RWMIg3m2krXbflFQIcx0PEKk5Mur4jn1A4TEA/+bP8FIXYI3PClGeGhDpNgUcNOpGZz6asr\nycoz5YqPj+vI1CEV/DM64o9f4KPLS5Zb9jPfNosqvlsNgiOrzWs3D7hnDdg05z0vX08tqej0b1KS\ntXbzNGXc/k3Mw7LoH6XVQNOiqfS3mJpyepcJQME2o7tkpZhcTUEejP6HKQqyWOCdQfY5LYD+f4GY\nfuYh/dNTpoVTeaJ6wJSF5dcZ2NLaBJnVr8FJm/lHIjvBpI9NWpY+ax56ys0E2dLZ/aT9Jldo+zDy\nCjTFY8fWln1PDx9T5OVIjs0ZEjbCZ5NM7jY8zgSs7teZFlfnIysFvrkL9i+pet+W/SAoCnZ+U700\nu3uZ4qOKePiY9OdUVLdgwycEclIr3h7W1uR82o04vzqAnHSTU8pNN//D1fimX9skWDjBuysP8uJi\n8+3Xz8udpQ8OpXlIJd9YK2MphI/GlwQEW2FtTHCYc3VJuXaL3qaJXNRF5/c+uRmm9VXRQ+xPy0wL\nnBOby9+/WXe49buqH7C1ZddCmHezee0dDFe/Y8rwi6QegzlXmX/QIm6eJsc05OHzC3ham/LfLZ9C\nYDMY+gh4+ZdsTz9hAqptkLNlmzvzi4CbvzYP4jlX2weM0FgThKK6l3+e2pKTbiq1m3S5sBZNWpsm\nunu+g72LIe1Y2X3iRpuGFZ6+ptHAkr+bHIObh/l7RMRZmwTHmc/s3kVw5DdTHNOit2nk0GGsqXPZ\n8z0k7oXgFvbFNEHR5pzbvoDf/mvqdkpr2hUue9XkfDKT4cwuU4y0Z5Ep0nL3gsF/M+9Xx3MENUWC\nhRPkF1q4/I3V7D1tRpC9rFsUb97Yq/onzMs0FWm//bekkhVM1rX9KFN5+e4Q+6xxm2Gm/LtV/5J1\nB5fDD4+Zh9jED+2LjX7/Ar6Zal5HdoQ/rzWVbx+MLlt5F94ObvuhbrXD19qUxyfuNd/my2t1k5lk\nZiY8ttYUYV32CkRWMWy8s9K66UNTLDFgGoTFmvXZqTB3MhxdA53HwxVvgG9I7aevNmht7lXibpPb\nSjlkvqUPut++HiPjjAlWoa0qrpfJTjW5x4Am598i0GIxxYKJe03QOHfSNFHveUvF3/ZTj5r6L//w\n83uvek6ChZOs/yOFSe+uKV7+7E99GdCuGn0AbJ3ZU9L9v9cUGG/TpPSXV2DZjFIHKNNRbcgjplnf\nV3eWlOsOuM+0uS8yZwIcXGZej3jKfGsCyMuC0zusLWv2mZzOgGnmG3V9pLUZ2PBC+g44U1G/hIYa\nJES9JcHCie7/fAsLtppmhu2bBvD9fYMvrLK7SF5W+cUmJ7aaPgk7v7HPZUT1sA4dYHMP3b3hvs0m\nl3HuFLzaqeSY+3c43vFMCNEoOBosGkHXy5o3fVwn/L1Mpde+0xnMWXOkZk5cUfl68x6m49O0zaaY\npcjJrZRpLlmYa1rFWApN7+jiSutBEiiEENUmwaIamgb5MG1Eyfgt//lpH+k5Tmi3XlpYLNy8wDQ3\ntdW8p6k8LLL1M9PkdOfXJevib3N++oQQDZYEi2q6fWAsbSL8aRvpz1s39SLIx8Hxli6Uuwdc+rTp\ngNOyH/S6Bab8nxm/qt1I607atEwpcvGd9kOlCyHEeZI6iwtwNDmLZsE+eHnUkZh7aofpl2BbNNXt\nWjNkSGMY7E0Icd6kzqIWxIT71Z1AAWaY4otsOnnFjTY9wyVQCCEuUN3vXijOz7h/mdFRvQJMpzJH\nhyMXQohKSLCoIRaLZtmeM+QVWBjZuSlLd5/mTHoOV/eKJti3Fh/Y3oEwroZmqxNCCCsJFjXgUGIG\nd83ZxP4zGYT5ezHju52cTjdTJL654iBPX9GZy7pFoeryjHhCCFEJKcyuAc1DfDmbZZrOpmTmFQcK\ngMRzufzlsy3cPnsDKZmVDIImhBB1mASLGuDj6c4dg2Lt1kUEeNMksGQEz+V7E3n4y99LHyqEEPWC\nFEPVkDsGxXIyLZvjZ7MZ1y2Kyy+KIrfAwr9+2MMna814/5uPniU1K48Qv3oyraQQQlhJsKghXh5u\nzLiyq906bw93nr+qG57ubrSNDOCaXtH4ep3f/LhCCFEXSLCoBU9f0aXqnYQQog6TOgshhBBVqjJY\nKKXclVIPVLWfEEKIhqvKYKG1LgSurIW0NHhaazYdSeGeTzbx1aYEVydHCCEc5midxa9Kqf8BXwCZ\nRSu11hVM5CzK8+WmBB6Zvw2AfafPcVXPFri7SUc9IUTd52idxQCgCzADeMX6829nJaqhGt25GQHe\nJj4fTMzk682SuxBC1A8OBQut9fByfi5xduIammA/T6YOaVO8/J+f9pGTX+jCFAkhhGMcChZKqWCl\n1KtKqY3Wn1eUUsHOTlxDdMdYyVx8AAAd1ElEQVSgWML9Tae8E2k5fLK2hqZkFUIIJ3K0GGoWcA6Y\nZP1JBz6s6iCl1Bil1F6l1AGl1GPlbH9QKbVLKbVNKbVMKdXKZtsUpdR+688UB9NZ5/l7ezDtknbF\ny28uP8C52piSVQghLoCjwaKt1vpprfUh68+zQJvKDlBKuQNvAmOBzsBkpVTnUrttAeK11t2B+cC/\nrMeGAU8DfYE+wNNKqVBHL6qum9w3huhQXwDOZuXzj0W7pThKCFGnORosspVSg4oWlFIDgewqjukD\nHLAGlzzgc0o1wdVaL9daZ1kX1wLR1tejgZ+01ila67PAT8AYB9Na53l7uPPgyPbFy3PXH2PEKyv5\nbtsJF6ZKCCEq5miwuBt4Uyl1WCl1GPgfcFcVx7QAjtksJ1jXVeQOYHE1j613ruzRgsFxEcXLx1Oz\nWb4n0YUpEkKIilXZz0Ip5QZ00FpfpJQKAtBapztw7vI6EOgK3uMmIB4Yej7HKqWmAlMBYmJiHEhS\n3eHupvjw1ouZu/4or/60j5x8Cw+P7uDqZAkhRLmqDBZaa4tS6i/APAeDRJEEoKXNcjRQppxFKXUp\n8AQwVGuda3PssFLHrignbTOBmQDx8fHlBqK6zMPdjZv7t2Z8jxZsS0ilWbCPq5MkhBDlcrQY6iel\n1ENKqZZKqbCinyqO2QDEKaVilVJewPXAQtsdlFI9gXeB8VrrMzabfgRGKaVCrRXbo6zrGqRgX08G\nx0XarcsrsHDLrPXMWXOYtCxpLSWEcC1Hh/u43fr7Xpt1mkpaRGmtC6w5kh8Bd2CW1nqnUmoGsFFr\nvRB4GQgAvrTOT31Uaz1ea52ilHoOE3AAZmitUxy+qgZg+d4zrNqXyKp9iTz33W5u6d+K6eM6yfAg\nQgiXUFpXXnpjrbPor7X+tXaSVD3x8fF648aNrk5Gjbl7ziZ+2HnKbt2EXi14eeJFEjCEEDVGKbVJ\nax1f1X6OjDprQcaBqnUvTujGjCu70KV5UPG6rzcf55H52yi01LvqGSFEPVdlzgJAKfUssA34Wjty\ngAs0tJxFEYtFM/3r7XyxsaQlcZtIf/y9PHBTMKBdBA+ObI+nu8xjJYQ4f47mLByts3gQ8AMKlVI5\nmKatWmsdVPlh4kK5uSlenNANoDhgHEosHiWeuKaBeEixlBDCyRwNFsHAjUCs1nqGUioGiHJesoSt\nooDh7enGnLVHKMrbXdQyhBeu7oq1cYAQQjiNo8VQbwMW4BKtdSdrc9YlWuuLnZ1ARzXUYqjSTqZl\nk3jOdEcJ8vGkdYS/3XattQQPIYTDaroYqq/WupdSaguA1vqste+EqGVRwb5EBfuWu01rzRMLdtAy\n1I+7h7aRoCGEqDGOBot86yiyGkApFYnJaYg65K0VB/ls3VEAjqdm8ez4rtLMVghRIxxtQvMG8A3Q\nRCn1ArAa+IfTUiXOW0GhhdX7k4qXP1l7lFd/2uvCFAkhGhJHp1X9FHgEeBE4CVyltf7SmQkT58fD\n3Y3Zt1/MlT2aF697c/lBFm8/WbxssWjyCiRDKIQ4f44WQ6G13gPscWJaxAXy9nDn1Uk9SMvOZ8Ve\nM9z53778nQAfD349kMznG45SaNHMuaMvPVqGuDi1Qoj6RHpyNTDuborXr+tJq3A/ALLyCrn5g/W8\ns/IgnaOCeHlid7o2l+4xQojzI8GiAQr282TmzfH4ebkXr1MKArw9aBnmh4f09hZCnCd5ajRQHZoF\n8sq1F+FlDQx9WocxsXc0HZtJrkIIcf4crrMQ9c/YblFc1DKEQoumZZifq5MjhKjHJFg0cM1Dynbg\nKyi0sHT3GTzcFJd2buqCVAkh6hsJFo3M9oQ07pqzkRNpObRrEsCITk2kp7cQokpSZ9HIxEb6k55T\nAMCBMxl8u/UEtuODZecV8u3W48XjTwkhBEjOotEJ8PZgQq8WfLzmCAD3f7GVz9YdZVy3Zvx2MJlV\n+xPJybfw+LiOTB3S1sWpFULUFZKzaITuHNyGcP+ScSDXH07hmf/bxZJdp8nJNz28529KsMtx5OQX\nsvtkeq2nVQhRN0iwaIRahvmxcNogbugbg6d72fqKuCYBXNmjBQXW6VuPJGdyzdu/ccN7azmRml3b\nyRVC1AEOzWdRHzSW+SxqWsLZLGatPsyxs1n0bhXKqM5NaRMZULzdYtGMfm0V+89kANA5Kog5d/Qh\nPMDbVUkWQtQgR+ezkGAhqrT+jxQmv7eWQmtOo22kP5/+qR/Ngn1cnDIhxIVyNFhIMZSoUp/YMF6c\n0I2iFrYHEzOZ+M5vHEnOrPxAIUSDIcFCOGRSfEv+O7knHtbJlBLOZnP9zLWcTJM6DCEaAwkWwmGX\nd2/Oe7fE4+1hPjYn03K47cMNpOfkuzhlQghnk2Ahzsvwjk1475b44hzGnlPnuOvjTeQWFLo4ZUII\nZ5JgIc7bkPaR/POa7sXL6Tn5nErLcWGKhBDOJsFCVMs1vaN5eHQHANpGBtAq3N9ue3ZeIRm5Ba5I\nmhDCCWS4D1Ftfx7Wluy8Qtzcynbs+8ei3azcl8jLE7vTt024C1InhKhJEixEtSmleGh0BywW+746\nK/clMmetGXvquplrGdo+kvsvjaNnTKgrkimEqAFOLYZSSo1RSu1VSh1QSj1WzvYhSqnNSqkCpdTE\nUtsKlVJbrT8LnZlOcWFK5yyy8woI9Cn5HrJyXyJXv/Ubt8xaz6p9iTSUjqBCNCZO68GtlHIH9gEj\ngQRgAzBZa73LZp/WQBDwELBQaz3fZluG1joAB0kP7rrlRGo2//phDwt/P0GpjAdxTQK4bWAsE3q1\nwMfTvfwTCCFqRV3owd0HOKC1PqS1zgM+B6603UFrfVhrvQ2wODEdwgWah/jy2vU9WfLAUK7q0Rzb\n+ZX2n8ng8W+20//FZSzefrLMsRaL5lRaDskZMqeGEHWFM4NFC+CYzXKCdZ2jfJRSG5VSa5VSV9Vs\n0kRtadckgNeu78mKh4Zx28DW+HuV5CTOZuWXmRt87vqjdHhyMf1eXEbffyxj7vqjtZ1kIUQ5nFnB\nXd5cnedT5hWjtT6hlGoD/KyU2q61Pmj3BkpNBaYCxMTEVD+lwulahfvz9BVdeGBke+ZtOMbs3w7T\nIsSXri2C7fYL8fUkv9B8TAosmse/2Y6flztX9jif7xlCiJrmzGCRALS0WY4GTjh6sNb6hPX3IaXU\nCqAncLDUPjOBmWDqLC4wvaIWBPl48qfBbbh1QGtSMvPKbI8ONTkNLw838gosaA1/m/c7Ad4ejOjU\ntLaTK4SwcmYx1AYgTikVq5TyAq4HHGrVpJQKVUp5W19HAAOBXZUfJeoTD3c3mgSVHeK8U1QgO58d\nzbrpI4hrYto3FFg093y6maW7Ttd2MoUQVk4LFlrrAuAvwI/AbmCe1nqnUmqGUmo8gFLqYqVUAnAt\n8K5Saqf18E7ARqXU78By4CXbVlSi4fJwd8Pf24NQfy8++VNfYqx1GnkFFv708UZeXLSb/EJpDyFE\nbZPJj0Sddiwli+tnruW4zXSuY7s2480bepXbc1wIcX7qQtNZIS5YyzA/vps2iOEdIovX+Xq6lwkU\nu06ks3JfYpne5EKImiHBQtR5of5efDDlYh4d0xF3N0V0qG+ZfT767TBTZq3n1tkbyM6T4dKFqGkS\nLES94OamuGdYWxb/dTBju0XZbSu0aJbuNpXfq/YlcufHG8nJNwFDa01uQaEMMSLEBZKBBEW90r5p\nYJl12fmFDGkfyTdbjgOw+kASd3y0ga4tgvlp52kOJWXSKtyPib2imTKwNUE+nrWdbCHqPclZiHov\nwNuD/1zXg4dGtS9e9+uBZN5deYhDSZkAHEnO4u2VB3FXUikuRHVIsBANxl8uieNvI9tXuH1ctyj8\nvSUzLUR1yH+OaFCmjYgjxM+TeRsT6BQVyKjOzYhvHcqq/Um0jfQvs//KfYlk5hbQOSqImDA/aY4r\nRAWkn4VotI6nZnPZG7+QmpUPQHSoL7cOaM2ki1tKvYZoNKSfhRCVyC+0MO2zzcWBAiDhbDbPf7+b\n/v9YxnPf7eJsOWNXCdFYSbAQjdafBrfhriFtGBwXYTezX2ZeIR+s/oOhLy/n/V8OkVcgw4sIIcVQ\nQgA5+YUs2HKcWb/+wb7TGXbbWoX78fLEi+gTG+ai1AnhPI4WQ0kFtxCAj6c71/eJ4bqLW7Jk12le\nXLSbw8lZABxNySLAphVVfqGFYS+vIDOvgN4xofRrE86wDpHEldMHRIiGQoqhhLChlGJ0l2YseWAo\nT17emSAfDyb1bknn5kHF+3i6u9EkyJvUrHyW7TnDC4t2M/q1VcxZe8SFKRfCuSRnIUQ5vDzcuGNQ\nLBN6tih3esc+sWFsOZpavGzR8Nz/7SK+VSidooLKOUKI+k1yFkJUItTfizB/rzLr/zoijp//NpSX\nJnSjfVMzSVNeoYUHvthKboEMZCgaHgkWQlSDn5cHbSIDuL5PDG/f1BsfT/OvtOfUOV79aZ+LUydE\nzZNiKCEuUNvIAKaP7cTTC81EjzNXHcJi0dwzrJ1druSD1X/wxrL9BPl6EBHgzYRe0dzQJwZ36TUu\n6gHJWQhRA27u14rBcREAaA2zfj2Ml4f9v1dqVh5p2fkcS8lmy9FUnlywgwlv/8auE+muSLIQ50WC\nhRA1wM1N8fLEi2gdbuYM79Yi2K65LUB6dn6Z434/lsoV/1vN9K+3c+BMRpntQtQV0ilPiBqUnVfI\nL/sTcVOKSzs3tdtWUGjhXE4Badn5fLv1BG8uP0BeoX3v8GEdIvnzsHbSAVDUGkc75UmwEMJFDiZm\n8MQ321l7KMVuvVKw8qHhxFhzKcdTs5m34RidmwcxukszVyRVNGDSg1uIOq5tZABz7+zH2kMpzPr1\nD5buPo3WMLFXdHGgAMgvsPD6sv0AXNMrmueu6oKfl/zritolnzghXEgpRf+24fRvG87hpEzeXH6A\nh0Z3sNunVbgfzYJ8OJWew1ebE/g9IZWXJnSjY1RQmXoRIZxFPmlC1BGtI/x5+dqLyqzXGjo0C+RU\neg4AB85kMPGdNQCE+HkS7Fsy98bkPjHcPbRt7SRYNCrSGkqIOs7NTfHR7X3418TuxZ3/iqRm5XMk\nOav4p0WIb5njjyZnUWgpWzd5Oj2H1fuTSMsq20pLiNIkZyFEPTEpviUXRYfw+rJ97Dl5joSz2WVa\nU/VtY9+KqtCimfTuGtwU3DW0Lddd3BI3pXjvl0O8sWw/uQUWlIJOzYJ4b0p8ucFGCJBgIUS90qFZ\nIG/d2BsAi0WTlJFLVp4Zi0opaBLoY7f/6gNJxcVXTy/cydsrDhLo48F+mz4dWsMfSZk0CfSupasQ\n9ZEECyHqKTc3RZMgn0r3Sc7IJczfixTrFLGn0nM4ZdNhvEmgN8mZecS3DsXTXUqlRcUkWAjRgE3o\nFc24blF8tu4ob688SOK5XAD8vNx5cGR7bh3Qmuz8Qs5mll9vsWLvGZbuPs3Q9k24tFMTlJJxrBor\np3bKU0qNAV4H3IH3tdYvldo+BHgN6A5cr7Web7NtCvB36+LzWuuPKnsv6ZQnROVy8guZvymBk2nZ\nTO4TQ3SoX4X7/pGUyT8X7+GHnaeK1/VrE8bfL+tM1xbBtZFcUUtc3oNbKeUO7ANGAgnABmCy1nqX\nzT6tgSDgIWBhUbBQSoUBG4F4QAObgN5a67MVvZ8ECyFqxrdbj/PXz7eWu00paBnqR5CvB/+d3IvY\nCH+77WlZ+QT7eZZ7rKib6kIP7j7AAa31IWuCPgeuBIqDhdb6sHWbpdSxo4GftNYp1u0/AWOAuU5M\nrxCNXnmBol+bMDYcPkuhRaO1mZMcwKPU0OqFFs3o11bRNNiHib2jGd+9eXHgyM4rZOORFDzc3Ojd\nKrTMiLyi7nNmsGgBHLNZTgD6XsCxLWooXUKICozu0oyLooP5PSGNNhH+vHB1N/q3DefAmQxeWryb\npbvPFO8b5GOfgyhqeXUqPYffj6Xy3He7GNm5KQWFFlbuSyQn33wnDPT2YOqQNkwbEVer1yYujDOD\nRXk1YY6WeTl0rFJqKjAVICYmxvGUCSHK5ePpzpd3D+BwciaxEf7FLaTaNQng/SkXk5adT0pmHudy\n8gn0sX98HDyTgZe7W3Hfj7wCC99vO1nmPc7lFuDj6e78ixE1ypl5wQSgpc1yNHCiJo/VWs/UWsdr\nreMjIyOrnVAhRAkvDzfaNw0styltsK8nsRH+dI8Owa1UMdTtg2JZ9/gInh3fhW7lVIK3jfQnOtR0\n+hvVpWmZ7cv3nCEnX+Yvr6ucmbPYAMQppWKB48D1wA0OHvsj8A+lVKh1eRQwveaTKISoSaH+XkwZ\n0JopA1qz51Q6y3afwdvDjeEdm9A2MgCtNQfOZNAq3L5i/MCZc9w2ewOhfp5cG9+SDk0DiQ71JcDH\ngxOpORw/m0VOgYWJvaOJCJDOg67gtGChtS5QSv0F8+B3B2ZprXcqpWYAG7XWC5VSFwPfAKHAFUqp\nZ7XWXbTWKUqp5zABB2BGUWW3EKJ+6NgsiI7NguzWKaWIaxpYZt8Pfz0MwNmsfGauOlThOT/+7TCz\nb+9D+3LOUVpSRi7h/l7SN6SGyORHQgiXm7PmMO+sPMTx1Owq9x3VuSkzbzEtPc/l5PPfnw8wtmsz\nesaEFq97ZP42Fu84RbsmAbxy7UVc1DLEmcmv11zez6K2SbAQon4rKLSwbM8ZNh89S8LZbBLOZpOZ\nW0DzEF+ignz4btsJmof4Mv+eAcXDslssmvgXlnI2K48b+8ZwXXwMD87bajf2lbub4t7h7bjvknZ4\nyJAmZdSFfhZCCOEwD3c3RndpVuHUsTf3b0Wov5fd/B1uboq+sWEs3nGKT9Ye5ZO1R8scV2jRbDl6\nFrdSxVGfrD1CqJ8X3aODaRlWcW92YUiwEELUC+UNM5JbUFhuhbeXhxuPjunIjztPsedkOv+a2N2u\n9VZBoYVnFu6kwDrPR5/WYUzsHc247lEy+2AFpBhKCFGvaa1ZvOMUzyzcyZlzuTQJ9GbmLfH0aBlC\noUVzKDGjTKX68dRsBr70c5lzBfl48OjYjky+OKZM0+CGSuoshBCNSmZuAb8fS6V7y5Aqcwen0nJ4\nZ+VBDpzJYM2h5DIzCca3CuVvozoQ6ONBq3A/An0a7nhXEiyEEMIBZ87lsGDLceasPcKxlLKtsebc\n0YfBcQ2306+jwUKaBgghGrUmgT5MHdKWnx4Yyr3D25YZIFEYEiyEEAIzLtbDozuy6K+DuaxbFJ2j\ngugcFYR/qSItrTVp2eVPFlWROWuPcODMuZpMbq2Tan8hhLDRvmkgb97Yq8LtryzZx7e/H+fTO/oR\nE+5HUkYuz3+3i20JaQztEMm1vVvSubnpuZ5faGHG/+1iztojxIT5seDegYT5e9XWpdQoqbMQQggH\nzVx1kH8s2gNA0yBv7h3ejteX7ifZOsd5kZv7teK5q7py4EwG4/+3mqw8M0Bin9gwPrmjb52az0Pq\nLIQQooa1jQwoftCfTs/lqW93lgkUABfHhgFmaPf/XNeDov6A6/9IYfz/VvPS4j2s2HuG1Kyyx9ZV\nkrMQQojz8NvBJO78aCOZeSXDqTcL8mHaiHasO5TCmkPJ/PLIcLs5O95ecZB//rCn3PO1CPGlU1QQ\nN/aNYXjHJk5Pf2ky3IcQQjjBgLYRfHpnP+78eCOJ53K5skdzZozvSrCfJzf2bUV+oaXMXCB3D21D\nWnY+M1cdpFSXDo6nZnM8NZtretlPBppXYOF4ajYxYX6414EWWpKzEEKIasjJLyQ9O58mQT4OH5Oa\nlce6P1JYeyiZjYfPsvfUOfIKLYT4ebLu8RF4e5TkRvacSmfMa7/QPNiHh0Z34KoeLZzSq1xyFkII\n4UQ+nu7nPT1siJ+X3WCJ+YUWDiVmkpqVZxcoABKsHQRPpOXw4Lzfmf3bYe4YFEtEgDfBvp40C/ap\n1YmgJFgIIYSLeLq70aFZ+RM55RVa8PdyL64b2ZaQxl8/32q3T2SgN12aB/Hk5Z1pGxng1LRKaygh\nhKiDxnWLYt0Tl/LnYW0rbGqbeC6XFXsT8fM6vxxOdUjOQggh6qgAbw8eGdORyX1i+Oi3wxxNySI9\nJ5/UrHyOJGeRnV9ImL8Xzc6j3qS6JFgIIUQd1zLMj79f3tluXaFFczg5k9PpObUyz7gECyGEqIfc\n3RRtIwOcXldRROoshBBCVEmChRBCiCpJsBBCCFElCRZCCCGqJMFCCCFElSRYCCGEqJIECyGEEFVq\nMKPOKqUSgSMXcIoIIKmGklNfNMZrhsZ53Y3xmqFxXvf5XnMrrXVkVTs1mGBxoZRSGx0ZprchaYzX\nDI3zuhvjNUPjvG5nXbMUQwkhhKiSBAshhBBVkmBRYqarE+ACjfGaoXFed2O8Zmic1+2Ua5Y6CyGE\nEFWSnIUQQogqNfpgoZQao5Taq5Q6oJR6zNXpcRalVEul1HKl1G6l1E6l1F+t68OUUj8ppfZbf4e6\nOq01TSnlrpTaopT6zrocq5RaZ73mL5RSXq5OY01TSoUopeYrpfZY73n/hn6vlVIPWD/bO5RSc5VS\nPg3xXiulZimlziildtisK/feKuMN6/Ntm1KqV3Xft1EHC6WUO/AmMBboDExWSnWu/Kh6qwD4m9a6\nE9APuNd6rY8By7TWccAy63JD81dgt83yP4H/WK/5LHCHS1LlXK8DP2itOwIXYa6/wd5rpVQL4D4g\nXmvdFXAHrqdh3uvZwJhS6yq6t2OBOOvPVODt6r5pow4WQB/ggNb6kNY6D/gcuNLFaXIKrfVJrfVm\n6+tzmIdHC8z1fmTd7SPgKtek0DmUUtHAZcD71mUFXALMt+7SEK85CBgCfACgtc7TWqfSwO81ZjI3\nX6WUB+AHnKQB3mut9SogpdTqiu7tlcDH2lgLhCiloqrzvo09WLQAjtksJ1jXNWhKqdZAT2Ad0FRr\nfRJMQAGauC5lTvEa8AhgsS6HA6la6wLrckO8522AROBDa/Hb+0opfxrwvdZaHwf+DRzFBIk0YBMN\n/14Xqeje1tgzrrEHi/Imrm3QzcOUUgHAV8D9Wut0V6fHmZRSlwNntNabbFeXs2tDu+ceQC/gba11\nTyCTBlTkVB5rGf2VQCzQHPDHFMGU1tDudVVq7PPe2INFAtDSZjkaOOGitDidUsoTEyg+1Vp/bV19\nuihbav19xlXpc4KBwHil1GFMEeMlmJxGiLWoAhrmPU8AErTW66zL8zHBoyHf60uBP7TWiVrrfOBr\nYAAN/14Xqeje1tgzrrEHiw1AnLXFhBemQmyhi9PkFNay+g+A3VrrV202LQSmWF9PAb6t7bQ5i9Z6\nutY6WmvdGnNvf9Za3wgsByZad2tQ1wygtT4FHFNKdbCuGgHsogHfa0zxUz+llJ/1s150zQ36Xtuo\n6N4uBG6xtorqB6QVFVedr0bfKU8pNQ7zbdMdmKW1fsHFSXIKpdQg4BdgOyXl949j6i3mATGYf7hr\ntdalK8/qPaXUMOAhrfXlSqk2mJxGGLAFuElrnevK9NU0pVQPTKW+F3AIuA3z5bDB3mul1LPAdZiW\nf1uAP2HK5xvUvVZKzQWGYUaXPQ08DSygnHtrDZz/w7SeygJu01pvrNb7NvZgIYQQomqNvRhKCCGE\nAyRYCCGEqJIECyGEEFWSYCGEEKJKEiyEEEJUSYKFEC6mlJqtlJpYxT7DlFIDaitNQpQmwUKIKlg7\nNLn6f2UYpkeyEC7h6n8AIeokpVRr6zwQbwGbgZZKqclKqe3W+RL+abNvhs3riUqp2dbXs61zCfym\nlDpUlHuwBp//KaV2KaW+p4oB/awDP94NPKCU2qqUGlzT1ytEVSRYCFGxDpjhnXsC+Zi5ES4BegAX\nK6UcGe46ChgEXA68ZF13tfXc3YA7sckxKKVmKKXG255Aa30YeAczL0MPrfUvF3JRQlSHR9W7CNFo\nHbHOAQBwMbBCa50IoJT6FDNnxIIqzrFAa20BdimlmlrXDQHmaq0LgRNKqZ+LdtZaP1WjVyBEDZGc\nhRAVy7R5Xd5Qz0Vsx8zxKbXNdhwi23PIODuiXpFgIYRj1gFDlVIR1ul4JwMrrdtOK6U6WSvBr3bg\nXKuA65WZGzwKGO7AMeeAwOokXIiaIMFCCAdYh3Wejhny+ndgs9a6aBjox4DvgJ8xs7RV5RtgP2YE\n4LcpCTrl1llY/R9wtVRwC1eRUWeFEEJUSXIWQgghqiTBQgghRJUkWAghhKiSBAshhBBVkmAhhBCi\nShIshBBCVEmChRBCiCpJsBBCCFGl/wdj/KgshJffKgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a15cf56d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.xlabel('round: t')\n",
    "plt.ylabel('error')\n",
    "plt.plot(ada_tr_err[::10],label='training error',linestyle='dashed',lw=3)\n",
    "plt.plot(ada_te_err[::10],label='test error',    linestyle='solid', lw=3)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.23605150214592274"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb          # xgboostを使う\n",
    "# データ読込み\n",
    "n = 4000\n",
    "d = pd.read_csv('data/soldat.csv').values; d = np.delete(d,70,1)\n",
    "i  = np.random.choice(len(d),n,replace=False)\n",
    "ti = np.delete(np.arange(len(d)), i)\n",
    "x, y = d[i,:71], d[i,71]\n",
    "tx, ty = d[ti,:71], d[ti,71]\n",
    "T = 1000                       # ラウンド数 1000\n",
    "xg = xgb.XGBClassifier(n_estimators=T, max_depth=1, objective='binary:logistic')\n",
    "\n",
    "xg.fit(x,y)                    # フィッティング\n",
    "1-xg.score(tx,ty)              # テスト誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-logloss:0.67494\n",
      "Will train until validation_0-logloss hasn't improved in 10 rounds.\n",
      "[1]\tvalidation_0-logloss:0.663866\n",
      "[2]\tvalidation_0-logloss:0.650502\n",
      "[3]\tvalidation_0-logloss:0.641174\n",
      "[4]\tvalidation_0-logloss:0.632024\n",
      "[5]\tvalidation_0-logloss:0.626014\n",
      "[6]\tvalidation_0-logloss:0.6187\n",
      "[7]\tvalidation_0-logloss:0.614404\n",
      "[8]\tvalidation_0-logloss:0.608655\n",
      "[9]\tvalidation_0-logloss:0.603559\n",
      "[10]\tvalidation_0-logloss:0.599429\n",
      "[11]\tvalidation_0-logloss:0.597102\n",
      "[12]\tvalidation_0-logloss:0.593414\n",
      "[13]\tvalidation_0-logloss:0.590661\n",
      "[14]\tvalidation_0-logloss:0.588863\n",
      "[15]\tvalidation_0-logloss:0.586012\n",
      "[16]\tvalidation_0-logloss:0.583897\n",
      "[17]\tvalidation_0-logloss:0.582829\n",
      "[18]\tvalidation_0-logloss:0.581178\n",
      "[19]\tvalidation_0-logloss:0.579685\n",
      "[20]\tvalidation_0-logloss:0.578658\n",
      "[21]\tvalidation_0-logloss:0.576264\n",
      "[22]\tvalidation_0-logloss:0.575066\n",
      "[23]\tvalidation_0-logloss:0.574417\n",
      "[24]\tvalidation_0-logloss:0.573423\n",
      "[25]\tvalidation_0-logloss:0.572647\n",
      "[26]\tvalidation_0-logloss:0.572112\n",
      "[27]\tvalidation_0-logloss:0.571807\n",
      "[28]\tvalidation_0-logloss:0.570764\n",
      "[29]\tvalidation_0-logloss:0.569685\n",
      "[30]\tvalidation_0-logloss:0.56906\n",
      "[31]\tvalidation_0-logloss:0.56857\n",
      "[32]\tvalidation_0-logloss:0.567647\n",
      "[33]\tvalidation_0-logloss:0.566589\n",
      "[34]\tvalidation_0-logloss:0.565845\n",
      "[35]\tvalidation_0-logloss:0.564919\n",
      "[36]\tvalidation_0-logloss:0.564419\n",
      "[37]\tvalidation_0-logloss:0.56316\n",
      "[38]\tvalidation_0-logloss:0.562632\n",
      "[39]\tvalidation_0-logloss:0.56235\n",
      "[40]\tvalidation_0-logloss:0.561566\n",
      "[41]\tvalidation_0-logloss:0.561376\n",
      "[42]\tvalidation_0-logloss:0.560714\n",
      "[43]\tvalidation_0-logloss:0.560322\n",
      "[44]\tvalidation_0-logloss:0.559659\n",
      "[45]\tvalidation_0-logloss:0.559169\n",
      "[46]\tvalidation_0-logloss:0.558985\n",
      "[47]\tvalidation_0-logloss:0.558299\n",
      "[48]\tvalidation_0-logloss:0.557695\n",
      "[49]\tvalidation_0-logloss:0.557535\n",
      "[50]\tvalidation_0-logloss:0.556936\n",
      "[51]\tvalidation_0-logloss:0.556633\n",
      "[52]\tvalidation_0-logloss:0.556376\n",
      "[53]\tvalidation_0-logloss:0.555628\n",
      "[54]\tvalidation_0-logloss:0.555098\n",
      "[55]\tvalidation_0-logloss:0.554853\n",
      "[56]\tvalidation_0-logloss:0.554551\n",
      "[57]\tvalidation_0-logloss:0.554379\n",
      "[58]\tvalidation_0-logloss:0.553819\n",
      "[59]\tvalidation_0-logloss:0.553454\n",
      "[60]\tvalidation_0-logloss:0.553071\n",
      "[61]\tvalidation_0-logloss:0.553019\n",
      "[62]\tvalidation_0-logloss:0.552524\n",
      "[63]\tvalidation_0-logloss:0.552362\n",
      "[64]\tvalidation_0-logloss:0.552023\n",
      "[65]\tvalidation_0-logloss:0.551417\n",
      "[66]\tvalidation_0-logloss:0.551253\n",
      "[67]\tvalidation_0-logloss:0.551049\n",
      "[68]\tvalidation_0-logloss:0.550737\n",
      "[69]\tvalidation_0-logloss:0.550319\n",
      "[70]\tvalidation_0-logloss:0.54984\n",
      "[71]\tvalidation_0-logloss:0.549214\n",
      "[72]\tvalidation_0-logloss:0.549167\n",
      "[73]\tvalidation_0-logloss:0.549226\n",
      "[74]\tvalidation_0-logloss:0.548494\n",
      "[75]\tvalidation_0-logloss:0.548522\n",
      "[76]\tvalidation_0-logloss:0.548158\n",
      "[77]\tvalidation_0-logloss:0.547719\n",
      "[78]\tvalidation_0-logloss:0.547549\n",
      "[79]\tvalidation_0-logloss:0.547081\n",
      "[80]\tvalidation_0-logloss:0.546613\n",
      "[81]\tvalidation_0-logloss:0.546369\n",
      "[82]\tvalidation_0-logloss:0.545614\n",
      "[83]\tvalidation_0-logloss:0.545797\n",
      "[84]\tvalidation_0-logloss:0.545669\n",
      "[85]\tvalidation_0-logloss:0.545519\n",
      "[86]\tvalidation_0-logloss:0.545272\n",
      "[87]\tvalidation_0-logloss:0.545153\n",
      "[88]\tvalidation_0-logloss:0.545202\n",
      "[89]\tvalidation_0-logloss:0.545292\n",
      "[90]\tvalidation_0-logloss:0.545084\n",
      "[91]\tvalidation_0-logloss:0.545006\n",
      "[92]\tvalidation_0-logloss:0.54484\n",
      "[93]\tvalidation_0-logloss:0.544914\n",
      "[94]\tvalidation_0-logloss:0.544515\n",
      "[95]\tvalidation_0-logloss:0.544455\n",
      "[96]\tvalidation_0-logloss:0.544074\n",
      "[97]\tvalidation_0-logloss:0.543776\n",
      "[98]\tvalidation_0-logloss:0.543322\n",
      "[99]\tvalidation_0-logloss:0.542656\n",
      "[100]\tvalidation_0-logloss:0.542558\n",
      "[101]\tvalidation_0-logloss:0.542483\n",
      "[102]\tvalidation_0-logloss:0.542274\n",
      "[103]\tvalidation_0-logloss:0.542165\n",
      "[104]\tvalidation_0-logloss:0.542275\n",
      "[105]\tvalidation_0-logloss:0.542382\n",
      "[106]\tvalidation_0-logloss:0.542108\n",
      "[107]\tvalidation_0-logloss:0.542175\n",
      "[108]\tvalidation_0-logloss:0.541851\n",
      "[109]\tvalidation_0-logloss:0.541692\n",
      "[110]\tvalidation_0-logloss:0.541516\n",
      "[111]\tvalidation_0-logloss:0.541218\n",
      "[112]\tvalidation_0-logloss:0.541075\n",
      "[113]\tvalidation_0-logloss:0.540885\n",
      "[114]\tvalidation_0-logloss:0.540797\n",
      "[115]\tvalidation_0-logloss:0.5407\n",
      "[116]\tvalidation_0-logloss:0.540654\n",
      "[117]\tvalidation_0-logloss:0.540127\n",
      "[118]\tvalidation_0-logloss:0.54025\n",
      "[119]\tvalidation_0-logloss:0.540231\n",
      "[120]\tvalidation_0-logloss:0.54031\n",
      "[121]\tvalidation_0-logloss:0.540056\n",
      "[122]\tvalidation_0-logloss:0.539941\n",
      "[123]\tvalidation_0-logloss:0.539561\n",
      "[124]\tvalidation_0-logloss:0.539391\n",
      "[125]\tvalidation_0-logloss:0.539228\n",
      "[126]\tvalidation_0-logloss:0.539008\n",
      "[127]\tvalidation_0-logloss:0.539091\n",
      "[128]\tvalidation_0-logloss:0.538829\n",
      "[129]\tvalidation_0-logloss:0.539052\n",
      "[130]\tvalidation_0-logloss:0.539029\n",
      "[131]\tvalidation_0-logloss:0.538861\n",
      "[132]\tvalidation_0-logloss:0.538689\n",
      "[133]\tvalidation_0-logloss:0.538336\n",
      "[134]\tvalidation_0-logloss:0.537973\n",
      "[135]\tvalidation_0-logloss:0.538114\n",
      "[136]\tvalidation_0-logloss:0.538225\n",
      "[137]\tvalidation_0-logloss:0.538206\n",
      "[138]\tvalidation_0-logloss:0.537964\n",
      "[139]\tvalidation_0-logloss:0.537858\n",
      "[140]\tvalidation_0-logloss:0.53761\n",
      "[141]\tvalidation_0-logloss:0.537608\n",
      "[142]\tvalidation_0-logloss:0.537233\n",
      "[143]\tvalidation_0-logloss:0.537086\n",
      "[144]\tvalidation_0-logloss:0.536981\n",
      "[145]\tvalidation_0-logloss:0.536783\n",
      "[146]\tvalidation_0-logloss:0.536788\n",
      "[147]\tvalidation_0-logloss:0.53666\n",
      "[148]\tvalidation_0-logloss:0.536555\n",
      "[149]\tvalidation_0-logloss:0.536388\n",
      "[150]\tvalidation_0-logloss:0.536265\n",
      "[151]\tvalidation_0-logloss:0.536033\n",
      "[152]\tvalidation_0-logloss:0.536005\n",
      "[153]\tvalidation_0-logloss:0.536104\n",
      "[154]\tvalidation_0-logloss:0.535907\n",
      "[155]\tvalidation_0-logloss:0.535769\n",
      "[156]\tvalidation_0-logloss:0.535767\n",
      "[157]\tvalidation_0-logloss:0.535883\n",
      "[158]\tvalidation_0-logloss:0.535827\n",
      "[159]\tvalidation_0-logloss:0.535504\n",
      "[160]\tvalidation_0-logloss:0.535409\n",
      "[161]\tvalidation_0-logloss:0.535434\n",
      "[162]\tvalidation_0-logloss:0.535222\n",
      "[163]\tvalidation_0-logloss:0.535224\n",
      "[164]\tvalidation_0-logloss:0.534898\n",
      "[165]\tvalidation_0-logloss:0.534925\n",
      "[166]\tvalidation_0-logloss:0.534566\n",
      "[167]\tvalidation_0-logloss:0.534098\n",
      "[168]\tvalidation_0-logloss:0.533953\n",
      "[169]\tvalidation_0-logloss:0.533896\n",
      "[170]\tvalidation_0-logloss:0.533874\n",
      "[171]\tvalidation_0-logloss:0.533792\n",
      "[172]\tvalidation_0-logloss:0.533907\n",
      "[173]\tvalidation_0-logloss:0.533721\n",
      "[174]\tvalidation_0-logloss:0.533555\n",
      "[175]\tvalidation_0-logloss:0.533594\n",
      "[176]\tvalidation_0-logloss:0.533295\n",
      "[177]\tvalidation_0-logloss:0.533461\n",
      "[178]\tvalidation_0-logloss:0.533477\n",
      "[179]\tvalidation_0-logloss:0.533388\n",
      "[180]\tvalidation_0-logloss:0.533144\n",
      "[181]\tvalidation_0-logloss:0.533144\n",
      "[182]\tvalidation_0-logloss:0.533185\n",
      "[183]\tvalidation_0-logloss:0.532777\n",
      "[184]\tvalidation_0-logloss:0.532672\n",
      "[185]\tvalidation_0-logloss:0.532967\n",
      "[186]\tvalidation_0-logloss:0.532807\n",
      "[187]\tvalidation_0-logloss:0.532688\n",
      "[188]\tvalidation_0-logloss:0.532594\n",
      "[189]\tvalidation_0-logloss:0.532592\n",
      "[190]\tvalidation_0-logloss:0.532622\n",
      "[191]\tvalidation_0-logloss:0.532734\n",
      "[192]\tvalidation_0-logloss:0.532703\n",
      "[193]\tvalidation_0-logloss:0.532767\n",
      "[194]\tvalidation_0-logloss:0.532607\n",
      "[195]\tvalidation_0-logloss:0.532559\n",
      "[196]\tvalidation_0-logloss:0.532258\n",
      "[197]\tvalidation_0-logloss:0.532092\n",
      "[198]\tvalidation_0-logloss:0.532138\n",
      "[199]\tvalidation_0-logloss:0.531868\n",
      "[200]\tvalidation_0-logloss:0.531647\n",
      "[201]\tvalidation_0-logloss:0.531519\n",
      "[202]\tvalidation_0-logloss:0.531824\n",
      "[203]\tvalidation_0-logloss:0.531974\n",
      "[204]\tvalidation_0-logloss:0.532006\n",
      "[205]\tvalidation_0-logloss:0.532011\n",
      "[206]\tvalidation_0-logloss:0.531873\n",
      "[207]\tvalidation_0-logloss:0.531637\n",
      "[208]\tvalidation_0-logloss:0.531585\n",
      "[209]\tvalidation_0-logloss:0.531499\n",
      "[210]\tvalidation_0-logloss:0.531616\n",
      "[211]\tvalidation_0-logloss:0.531466\n",
      "[212]\tvalidation_0-logloss:0.531213\n",
      "[213]\tvalidation_0-logloss:0.531265\n",
      "[214]\tvalidation_0-logloss:0.531204\n",
      "[215]\tvalidation_0-logloss:0.531182\n",
      "[216]\tvalidation_0-logloss:0.530934\n",
      "[217]\tvalidation_0-logloss:0.53103\n",
      "[218]\tvalidation_0-logloss:0.531135\n",
      "[219]\tvalidation_0-logloss:0.531024\n",
      "[220]\tvalidation_0-logloss:0.531069\n",
      "[221]\tvalidation_0-logloss:0.530697\n",
      "[222]\tvalidation_0-logloss:0.530604\n",
      "[223]\tvalidation_0-logloss:0.530483\n",
      "[224]\tvalidation_0-logloss:0.530215\n",
      "[225]\tvalidation_0-logloss:0.530215\n",
      "[226]\tvalidation_0-logloss:0.530468\n",
      "[227]\tvalidation_0-logloss:0.530617\n",
      "[228]\tvalidation_0-logloss:0.530415\n",
      "[229]\tvalidation_0-logloss:0.530178\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[230]\tvalidation_0-logloss:0.530295\n",
      "[231]\tvalidation_0-logloss:0.530212\n",
      "[232]\tvalidation_0-logloss:0.53033\n",
      "[233]\tvalidation_0-logloss:0.530416\n",
      "[234]\tvalidation_0-logloss:0.5303\n",
      "[235]\tvalidation_0-logloss:0.530194\n",
      "[236]\tvalidation_0-logloss:0.530142\n",
      "[237]\tvalidation_0-logloss:0.53013\n",
      "[238]\tvalidation_0-logloss:0.530132\n",
      "[239]\tvalidation_0-logloss:0.530081\n",
      "[240]\tvalidation_0-logloss:0.530331\n",
      "[241]\tvalidation_0-logloss:0.529983\n",
      "[242]\tvalidation_0-logloss:0.529927\n",
      "[243]\tvalidation_0-logloss:0.529738\n",
      "[244]\tvalidation_0-logloss:0.530042\n",
      "[245]\tvalidation_0-logloss:0.529888\n",
      "[246]\tvalidation_0-logloss:0.529755\n",
      "[247]\tvalidation_0-logloss:0.529776\n",
      "[248]\tvalidation_0-logloss:0.529789\n",
      "[249]\tvalidation_0-logloss:0.529828\n",
      "[250]\tvalidation_0-logloss:0.529737\n",
      "[251]\tvalidation_0-logloss:0.529841\n",
      "[252]\tvalidation_0-logloss:0.529727\n",
      "[253]\tvalidation_0-logloss:0.529848\n",
      "[254]\tvalidation_0-logloss:0.529803\n",
      "[255]\tvalidation_0-logloss:0.529863\n",
      "[256]\tvalidation_0-logloss:0.529243\n",
      "[257]\tvalidation_0-logloss:0.529038\n",
      "[258]\tvalidation_0-logloss:0.52904\n",
      "[259]\tvalidation_0-logloss:0.529061\n",
      "[260]\tvalidation_0-logloss:0.529094\n",
      "[261]\tvalidation_0-logloss:0.529197\n",
      "[262]\tvalidation_0-logloss:0.529283\n",
      "[263]\tvalidation_0-logloss:0.529556\n",
      "[264]\tvalidation_0-logloss:0.52934\n",
      "[265]\tvalidation_0-logloss:0.529321\n",
      "[266]\tvalidation_0-logloss:0.529385\n",
      "[267]\tvalidation_0-logloss:0.529506\n",
      "Stopping. Best iteration:\n",
      "[257]\tvalidation_0-logloss:0.529038\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=1, min_child_weight=1, missing=None, n_estimators=1000,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg = xgb.XGBClassifier(n_estimators=1000, max_depth=1, objective= 'binary:logistic')\n",
    "cvx = x[:3500,:]; cvy = y[:3500];       # トレーニングデータ\n",
    "eval_set = [(x[3500:,:], y[3500:])]    # 検証用のデータ\n",
    "xg.fit(cvx, cvy, early_stopping_rounds=10, eval_metric=\"logloss\", eval_set=eval_set,verbose=True) # verboseを設定：途中経過をプリント"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "257"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg.best_iteration          # 最適なラウンド数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25321888412017168"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1-xg.score(tx,ty)          # テスト誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.87767226,  0.12232777],\n",
       "       [ 0.55203164,  0.44796836],\n",
       "       [ 0.73098552,  0.26901448],\n",
       "       ..., \n",
       "       [ 0.2073034 ,  0.7926966 ],\n",
       "       [ 0.05452764,  0.94547236],\n",
       "       [ 0.16084927,  0.83915073]], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg.predict_proba(tx)      # 予測点における各ラベルの条件付き確率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}