From 1c32afe545fb82575a80a7209743f4f12a2fb332 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?xavier=20dupr=C3=A9?= <xavier.dupre@gmail.com>
Date: Sat, 20 Oct 2018 10:20:21 +0200
Subject: [PATCH] use of xgboost

---
 .circleci/config.yml                          |   1 +
 .travis.yml                                   |   1 +
 .../notebooks/dsgarden/discret_gradient.ipynb | 955 ++++++++----------
 appveyor.yml                                  |   1 +
 requirements_conda.txt                        |   1 +
 5 files changed, 423 insertions(+), 536 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d3ef2ad..58f7604 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -57,6 +57,7 @@ jobs:
           command: |
             python3 -m venv venv
             . venv/bin/activate
+            conda install -c conda-forge xgboost
             pip install -r requirements_conda.txt
             
       - run:
diff --git a/.travis.yml b/.travis.yml
index e05865a..b057e88 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,6 +21,7 @@ install:
   - source activate test-environment
   # - make all
   #- conda build build_tools/conda-recipe --quiet
+  - conda install -c conda-forge xgboost
   - conda install -q --file=requirements_conda.txt
   - pip install -r requirements.txt
   - pip install -U git+https://github.com/quantopian/qgrid --no-deps  
diff --git a/_doc/notebooks/dsgarden/discret_gradient.ipynb b/_doc/notebooks/dsgarden/discret_gradient.ipynb
index 1f2b069..73877f9 100644
--- a/_doc/notebooks/dsgarden/discret_gradient.ipynb
+++ b/_doc/notebooks/dsgarden/discret_gradient.ipynb
@@ -403,94 +403,94 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>89</th>\n",
-              "      <td>4.8</td>\n",
-              "      <td>3.0</td>\n",
-              "      <td>0.0</td>\n",
-              "      <td>1.0</td>\n",
+              "      <th>414</th>\n",
+              "      <td>6.7</td>\n",
+              "      <td>3.3</td>\n",
               "      <td>0.0</td>\n",
               "      <td>0.0</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>361</th>\n",
-              "      <td>6.4</td>\n",
-              "      <td>2.7</td>\n",
+              "      <th>125</th>\n",
+              "      <td>5.5</td>\n",
+              "      <td>2.5</td>\n",
               "      <td>0.0</td>\n",
-              "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
+              "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>363</th>\n",
-              "      <td>6.1</td>\n",
-              "      <td>2.6</td>\n",
-              "      <td>1.0</td>\n",
+              "      <th>394</th>\n",
+              "      <td>6.7</td>\n",
+              "      <td>3.1</td>\n",
               "      <td>0.0</td>\n",
               "      <td>0.0</td>\n",
+              "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>75</th>\n",
+              "      <th>411</th>\n",
               "      <td>6.0</td>\n",
-              "      <td>2.9</td>\n",
+              "      <td>2.2</td>\n",
               "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>76</th>\n",
-              "      <td>6.6</td>\n",
-              "      <td>2.9</td>\n",
+              "      <th>95</th>\n",
+              "      <td>7.6</td>\n",
+              "      <td>3.0</td>\n",
               "      <td>0.0</td>\n",
-              "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>1.0</td>\n",
+              "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>340</th>\n",
-              "      <td>5.5</td>\n",
-              "      <td>2.5</td>\n",
+              "      <th>64</th>\n",
+              "      <td>5.8</td>\n",
+              "      <td>2.6</td>\n",
               "      <td>0.0</td>\n",
-              "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>1.0</td>\n",
+              "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>220</th>\n",
-              "      <td>5.6</td>\n",
-              "      <td>2.7</td>\n",
+              "      <th>309</th>\n",
+              "      <td>5.0</td>\n",
+              "      <td>3.4</td>\n",
               "      <td>0.0</td>\n",
-              "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>1.0</td>\n",
+              "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>250</th>\n",
+              "      <th>7</th>\n",
               "      <td>6.7</td>\n",
-              "      <td>3.1</td>\n",
+              "      <td>3.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
-              "      <td>1.0</td>\n",
+              "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>142</th>\n",
-              "      <td>5.5</td>\n",
-              "      <td>2.3</td>\n",
+              "      <th>182</th>\n",
+              "      <td>6.1</td>\n",
+              "      <td>2.8</td>\n",
               "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>0.0</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>401</th>\n",
-              "      <td>5.1</td>\n",
-              "      <td>3.8</td>\n",
-              "      <td>0.0</td>\n",
+              "      <th>49</th>\n",
+              "      <td>4.7</td>\n",
+              "      <td>3.2</td>\n",
               "      <td>0.0</td>\n",
               "      <td>1.0</td>\n",
               "      <td>0.0</td>\n",
+              "      <td>0.0</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -498,16 +498,16 @@
             ],
             "text/plain": [
               "      X1   X2   Y0   Y1   Y2   Y'\n",
-              "89   4.8  3.0  0.0  1.0  0.0  0.0\n",
-              "361  6.4  2.7  0.0  1.0  0.0  0.0\n",
-              "363  6.1  2.6  1.0  0.0  0.0  0.0\n",
-              "75   6.0  2.9  1.0  0.0  0.0  0.0\n",
-              "76   6.6  2.9  0.0  1.0  0.0  1.0\n",
-              "340  5.5  2.5  0.0  1.0  0.0  1.0\n",
-              "220  5.6  2.7  0.0  1.0  0.0  1.0\n",
-              "250  6.7  3.1  0.0  1.0  0.0  1.0\n",
-              "142  5.5  2.3  1.0  0.0  0.0  0.0\n",
-              "401  5.1  3.8  0.0  0.0  1.0  0.0"
+              "414  6.7  3.3  0.0  0.0  1.0  1.0\n",
+              "125  5.5  2.5  0.0  0.0  1.0  0.0\n",
+              "394  6.7  3.1  0.0  0.0  1.0  0.0\n",
+              "411  6.0  2.2  1.0  0.0  0.0  0.0\n",
+              "95   7.6  3.0  0.0  0.0  1.0  1.0\n",
+              "64   5.8  2.6  0.0  0.0  1.0  0.0\n",
+              "309  5.0  3.4  0.0  0.0  1.0  0.0\n",
+              "7    6.7  3.0  0.0  1.0  0.0  0.0\n",
+              "182  6.1  2.8  1.0  0.0  0.0  0.0\n",
+              "49   4.7  3.2  0.0  1.0  0.0  0.0"
             ]
           },
           "execution_count": 7,
@@ -623,27 +623,27 @@
               "      <th>0</th>\n",
               "      <td>5.1</td>\n",
               "      <td>3.5</td>\n",
-              "      <td>1.192797</td>\n",
-              "      <td>0.067656</td>\n",
-              "      <td>0.137434</td>\n",
+              "      <td>1.107461</td>\n",
+              "      <td>0.166893</td>\n",
+              "      <td>0.018765</td>\n",
               "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>1</th>\n",
               "      <td>5.1</td>\n",
               "      <td>3.5</td>\n",
-              "      <td>0.025577</td>\n",
-              "      <td>1.100564</td>\n",
-              "      <td>0.091577</td>\n",
+              "      <td>0.162464</td>\n",
+              "      <td>1.187359</td>\n",
+              "      <td>0.187721</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>2</th>\n",
               "      <td>5.1</td>\n",
               "      <td>3.5</td>\n",
-              "      <td>0.089813</td>\n",
-              "      <td>0.161362</td>\n",
-              "      <td>1.114748</td>\n",
+              "      <td>0.086876</td>\n",
+              "      <td>0.178472</td>\n",
+              "      <td>1.179201</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
@@ -652,9 +652,9 @@
             ],
             "text/plain": [
               "    X1   X2        Y0        Y1        Y2   Y'\n",
-              "0  5.1  3.5  1.192797  0.067656  0.137434  1.0\n",
-              "1  5.1  3.5  0.025577  1.100564  0.091577  0.0\n",
-              "2  5.1  3.5  0.089813  0.161362  1.114748  0.0"
+              "0  5.1  3.5  1.107461  0.166893  0.018765  1.0\n",
+              "1  5.1  3.5  0.162464  1.187359  0.187721  0.0\n",
+              "2  5.1  3.5  0.086876  0.178472  1.179201  0.0"
             ]
           },
           "execution_count": 10,
@@ -732,93 +732,93 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>386</th>\n",
-              "      <td>5.1</td>\n",
-              "      <td>3.8</td>\n",
-              "      <td>0.011358</td>\n",
-              "      <td>1.042194</td>\n",
-              "      <td>0.194025</td>\n",
-              "      <td>0.0</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>291</th>\n",
-              "      <td>6.3</td>\n",
-              "      <td>3.3</td>\n",
-              "      <td>1.075981</td>\n",
-              "      <td>0.065165</td>\n",
-              "      <td>0.060387</td>\n",
-              "      <td>0.0</td>\n",
+              "      <th>295</th>\n",
+              "      <td>5.5</td>\n",
+              "      <td>2.6</td>\n",
+              "      <td>0.197643</td>\n",
+              "      <td>1.199976</td>\n",
+              "      <td>0.180766</td>\n",
+              "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>40</th>\n",
-              "      <td>4.3</td>\n",
-              "      <td>3.0</td>\n",
-              "      <td>0.159590</td>\n",
-              "      <td>0.052912</td>\n",
-              "      <td>1.183372</td>\n",
+              "      <th>46</th>\n",
+              "      <td>5.2</td>\n",
+              "      <td>3.4</td>\n",
+              "      <td>0.178395</td>\n",
+              "      <td>0.190600</td>\n",
+              "      <td>1.159765</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>3</th>\n",
-              "      <td>4.6</td>\n",
+              "      <th>187</th>\n",
+              "      <td>6.7</td>\n",
               "      <td>3.1</td>\n",
-              "      <td>1.047817</td>\n",
-              "      <td>0.180400</td>\n",
-              "      <td>0.110310</td>\n",
+              "      <td>0.188947</td>\n",
+              "      <td>1.093288</td>\n",
+              "      <td>0.139723</td>\n",
               "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>141</th>\n",
-              "      <td>5.0</td>\n",
-              "      <td>3.4</td>\n",
-              "      <td>0.122966</td>\n",
-              "      <td>1.140903</td>\n",
-              "      <td>0.105534</td>\n",
-              "      <td>0.0</td>\n",
+              "      <th>210</th>\n",
+              "      <td>6.9</td>\n",
+              "      <td>3.1</td>\n",
+              "      <td>0.095428</td>\n",
+              "      <td>0.182643</td>\n",
+              "      <td>1.037533</td>\n",
+              "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>179</th>\n",
-              "      <td>5.1</td>\n",
+              "      <th>29</th>\n",
+              "      <td>5.5</td>\n",
               "      <td>3.5</td>\n",
-              "      <td>0.139589</td>\n",
-              "      <td>1.163171</td>\n",
-              "      <td>0.007990</td>\n",
-              "      <td>0.0</td>\n",
+              "      <td>1.131419</td>\n",
+              "      <td>0.077241</td>\n",
+              "      <td>0.177483</td>\n",
+              "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>430</th>\n",
-              "      <td>5.6</td>\n",
-              "      <td>3.0</td>\n",
-              "      <td>0.064294</td>\n",
-              "      <td>1.183321</td>\n",
-              "      <td>0.152484</td>\n",
+              "      <th>315</th>\n",
+              "      <td>6.4</td>\n",
+              "      <td>3.2</td>\n",
+              "      <td>0.099738</td>\n",
+              "      <td>0.197291</td>\n",
+              "      <td>1.035431</td>\n",
               "      <td>1.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>55</th>\n",
-              "      <td>6.6</td>\n",
-              "      <td>2.9</td>\n",
-              "      <td>0.061399</td>\n",
-              "      <td>0.020338</td>\n",
-              "      <td>1.139617</td>\n",
+              "      <th>152</th>\n",
+              "      <td>5.8</td>\n",
+              "      <td>2.7</td>\n",
+              "      <td>0.069061</td>\n",
+              "      <td>0.045325</td>\n",
+              "      <td>1.061221</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>427</th>\n",
-              "      <td>6.1</td>\n",
+              "      <th>168</th>\n",
+              "      <td>6.5</td>\n",
               "      <td>2.8</td>\n",
-              "      <td>0.083090</td>\n",
-              "      <td>0.108569</td>\n",
-              "      <td>1.065685</td>\n",
+              "      <td>0.093164</td>\n",
+              "      <td>1.177413</td>\n",
+              "      <td>0.095890</td>\n",
+              "      <td>1.0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>348</th>\n",
+              "      <td>6.9</td>\n",
+              "      <td>3.1</td>\n",
+              "      <td>1.094184</td>\n",
+              "      <td>0.196944</td>\n",
+              "      <td>0.083975</td>\n",
               "      <td>0.0</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>189</th>\n",
-              "      <td>6.0</td>\n",
-              "      <td>2.2</td>\n",
-              "      <td>0.011013</td>\n",
-              "      <td>1.022085</td>\n",
-              "      <td>0.199416</td>\n",
+              "      <th>261</th>\n",
+              "      <td>6.3</td>\n",
+              "      <td>2.8</td>\n",
+              "      <td>0.197558</td>\n",
+              "      <td>0.080273</td>\n",
+              "      <td>1.009379</td>\n",
               "      <td>1.0</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
@@ -827,16 +827,16 @@
             ],
             "text/plain": [
               "      X1   X2        Y0        Y1        Y2   Y'\n",
-              "386  5.1  3.8  0.011358  1.042194  0.194025  0.0\n",
-              "291  6.3  3.3  1.075981  0.065165  0.060387  0.0\n",
-              "40   4.3  3.0  0.159590  0.052912  1.183372  0.0\n",
-              "3    4.6  3.1  1.047817  0.180400  0.110310  1.0\n",
-              "141  5.0  3.4  0.122966  1.140903  0.105534  0.0\n",
-              "179  5.1  3.5  0.139589  1.163171  0.007990  0.0\n",
-              "430  5.6  3.0  0.064294  1.183321  0.152484  1.0\n",
-              "55   6.6  2.9  0.061399  0.020338  1.139617  0.0\n",
-              "427  6.1  2.8  0.083090  0.108569  1.065685  0.0\n",
-              "189  6.0  2.2  0.011013  1.022085  0.199416  1.0"
+              "295  5.5  2.6  0.197643  1.199976  0.180766  1.0\n",
+              "46   5.2  3.4  0.178395  0.190600  1.159765  0.0\n",
+              "187  6.7  3.1  0.188947  1.093288  0.139723  1.0\n",
+              "210  6.9  3.1  0.095428  0.182643  1.037533  1.0\n",
+              "29   5.5  3.5  1.131419  0.077241  0.177483  1.0\n",
+              "315  6.4  3.2  0.099738  0.197291  1.035431  1.0\n",
+              "152  5.8  2.7  0.069061  0.045325  1.061221  0.0\n",
+              "168  6.5  2.8  0.093164  1.177413  0.095890  1.0\n",
+              "348  6.9  3.1  1.094184  0.196944  0.083975  0.0\n",
+              "261  6.3  2.8  0.197558  0.080273  1.009379  1.0"
             ]
           },
           "execution_count": 11,
@@ -890,7 +890,7 @@
           "data": {
             "text/plain": [
               "array([[299,   1],\n",
-              "       [ 12, 138]], dtype=int64)"
+              "       [ 10, 140]], dtype=int64)"
             ]
           },
           "execution_count": 13,
@@ -924,16 +924,6 @@
       "execution_count": 13,
       "metadata": {},
       "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n",
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n"
-          ]
-        },
         {
           "data": {
             "text/html": [
@@ -962,19 +952,19 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>11</th>\n",
+              "      <th>10</th>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>AdaBoostClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>4</th>\n",
+              "      <th>3</th>\n",
               "      <td>0.048889</td>\n",
               "      <td>0.000000</td>\n",
               "      <td>DecisionTreeClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>5</th>\n",
+              "      <th>4</th>\n",
               "      <td>0.048889</td>\n",
               "      <td>0.000000</td>\n",
               "      <td>ExtraTreeClassifier</td>\n",
@@ -992,27 +982,21 @@
               "      <td>GaussianNB</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>2</th>\n",
+              "      <th>1</th>\n",
               "      <td>0.104444</td>\n",
-              "      <td>0.051111</td>\n",
+              "      <td>0.044444</td>\n",
               "      <td>GradientBoostingClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>9</th>\n",
               "      <td>0.104444</td>\n",
-              "      <td>0.097778</td>\n",
+              "      <td>0.091111</td>\n",
               "      <td>KNeighborsClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>0</th>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
-              "      <td>LinearSVC</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
               "      <td>LogisticRegression</td>\n",
               "    </tr>\n",
               "    <tr>\n",
@@ -1022,16 +1006,16 @@
               "      <td>MLPClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>3</th>\n",
-              "      <td>0.051111</td>\n",
-              "      <td>0.004444</td>\n",
+              "      <th>2</th>\n",
+              "      <td>0.053333</td>\n",
+              "      <td>0.002222</td>\n",
               "      <td>RandomForestClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>10</th>\n",
-              "      <td>0.200000</td>\n",
-              "      <td>0.197778</td>\n",
-              "      <td>SVC</td>\n",
+              "      <th>5</th>\n",
+              "      <td>0.333333</td>\n",
+              "      <td>0.053333</td>\n",
+              "      <td>XGBClassifier</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -1039,18 +1023,17 @@
             ],
             "text/plain": [
               "        err1      err2                       model\n",
-              "11  0.333333  0.333333          AdaBoostClassifier\n",
-              "4   0.048889  0.000000      DecisionTreeClassifier\n",
-              "5   0.048889  0.000000         ExtraTreeClassifier\n",
+              "10  0.333333  0.333333          AdaBoostClassifier\n",
+              "3   0.048889  0.000000      DecisionTreeClassifier\n",
+              "4   0.048889  0.000000         ExtraTreeClassifier\n",
               "6   0.048889  0.000000        ExtraTreesClassifier\n",
               "8   0.333333  0.333333                  GaussianNB\n",
-              "2   0.104444  0.051111  GradientBoostingClassifier\n",
-              "9   0.104444  0.097778        KNeighborsClassifier\n",
-              "0   0.333333  0.333333                   LinearSVC\n",
-              "1   0.333333  0.333333          LogisticRegression\n",
+              "1   0.104444  0.044444  GradientBoostingClassifier\n",
+              "9   0.104444  0.091111        KNeighborsClassifier\n",
+              "0   0.333333  0.333333          LogisticRegression\n",
               "7   0.333333  0.333333               MLPClassifier\n",
-              "3   0.051111  0.004444      RandomForestClassifier\n",
-              "10  0.200000  0.197778                         SVC"
+              "2   0.053333  0.002222      RandomForestClassifier\n",
+              "5   0.333333  0.053333               XGBClassifier"
             ]
           },
           "execution_count": 14,
@@ -1089,18 +1072,17 @@
         "from sklearn.neural_network import MLPClassifier\n",
         "from sklearn.naive_bayes import GaussianNB\n",
         "from sklearn.neighbors import KNeighborsClassifier, RadiusNeighborsClassifier\n",
-        "from sklearn.svm import SVC, LinearSVC\n",
+        "from xgboost import XGBClassifier\n",
         "\n",
-        "models = [LinearSVC,\n",
-        "          (LogisticRegression, dict(multi_class=\"ovr\", solver=\"liblinear\")),\n",
+        "models = [(LogisticRegression, dict(multi_class=\"ovr\", solver=\"liblinear\")),\n",
         "          GradientBoostingClassifier,\n",
         "          (RandomForestClassifier, dict(n_estimators=20)),\n",
         "          DecisionTreeClassifier,\n",
         "          ExtraTreeClassifier,\n",
+        "          XGBClassifier,\n",
         "          (ExtraTreesClassifier, dict(n_estimators=20)),\n",
         "          (MLPClassifier, dict(activation=\"logistic\")),\n",
         "          GaussianNB, KNeighborsClassifier, \n",
-        "          (SVC, dict(gamma=\"scale\")),\n",
         "          (AdaBoostClassifier, dict(base_estimator=LogisticRegression(multi_class=\"ovr\", solver=\"liblinear\"), \n",
         "                                    algorithm=\"SAMME\"))]\n",
         "\n",
@@ -1161,7 +1143,7 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>11</th>\n",
+              "      <th>10</th>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>AdaBoostClassifier</td>\n",
@@ -1170,7 +1152,7 @@
               "      <td>AdaBoostClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>4</th>\n",
+              "      <th>3</th>\n",
               "      <td>0.048889</td>\n",
               "      <td>0.000000</td>\n",
               "      <td>DecisionTreeClassifier</td>\n",
@@ -1179,7 +1161,7 @@
               "      <td>DecisionTreeClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>5</th>\n",
+              "      <th>4</th>\n",
               "      <td>0.048889</td>\n",
               "      <td>0.000000</td>\n",
               "      <td>ExtraTreeClassifier</td>\n",
@@ -1206,36 +1188,27 @@
               "      <td>GaussianNB</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>2</th>\n",
+              "      <th>1</th>\n",
               "      <td>0.104444</td>\n",
-              "      <td>0.051111</td>\n",
+              "      <td>0.044444</td>\n",
               "      <td>GradientBoostingClassifier</td>\n",
               "      <td>0.333333</td>\n",
-              "      <td>0.246667</td>\n",
+              "      <td>0.224444</td>\n",
               "      <td>GradientBoostingClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>9</th>\n",
               "      <td>0.104444</td>\n",
-              "      <td>0.097778</td>\n",
+              "      <td>0.091111</td>\n",
               "      <td>KNeighborsClassifier</td>\n",
               "      <td>0.335556</td>\n",
-              "      <td>0.326667</td>\n",
+              "      <td>0.340000</td>\n",
               "      <td>KNeighborsClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>0</th>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
-              "      <td>LinearSVC</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>LinearSVC</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
               "      <td>LogisticRegression</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -1251,22 +1224,22 @@
               "      <td>MLPClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>3</th>\n",
-              "      <td>0.051111</td>\n",
-              "      <td>0.004444</td>\n",
+              "      <th>2</th>\n",
+              "      <td>0.053333</td>\n",
+              "      <td>0.002222</td>\n",
               "      <td>RandomForestClassifier</td>\n",
-              "      <td>0.340000</td>\n",
-              "      <td>0.042222</td>\n",
+              "      <td>0.333333</td>\n",
+              "      <td>0.024444</td>\n",
               "      <td>RandomForestClassifier</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>10</th>\n",
-              "      <td>0.200000</td>\n",
-              "      <td>0.197778</td>\n",
-              "      <td>SVC</td>\n",
+              "      <th>5</th>\n",
               "      <td>0.333333</td>\n",
+              "      <td>0.053333</td>\n",
+              "      <td>XGBClassifier</td>\n",
               "      <td>0.333333</td>\n",
-              "      <td>SVC</td>\n",
+              "      <td>0.315556</td>\n",
+              "      <td>XGBClassifier</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -1274,32 +1247,30 @@
             ],
             "text/plain": [
               "        err1      err2                       model   errACP1   errACP2  \\\n",
-              "11  0.333333  0.333333          AdaBoostClassifier  0.333333  0.333333   \n",
-              "4   0.048889  0.000000      DecisionTreeClassifier  0.333333  0.000000   \n",
-              "5   0.048889  0.000000         ExtraTreeClassifier  0.333333  0.000000   \n",
+              "10  0.333333  0.333333          AdaBoostClassifier  0.333333  0.333333   \n",
+              "3   0.048889  0.000000      DecisionTreeClassifier  0.333333  0.000000   \n",
+              "4   0.048889  0.000000         ExtraTreeClassifier  0.333333  0.000000   \n",
               "6   0.048889  0.000000        ExtraTreesClassifier  0.333333  0.000000   \n",
               "8   0.333333  0.333333                  GaussianNB  0.333333  0.333333   \n",
-              "2   0.104444  0.051111  GradientBoostingClassifier  0.333333  0.246667   \n",
-              "9   0.104444  0.097778        KNeighborsClassifier  0.335556  0.326667   \n",
-              "0   0.333333  0.333333                   LinearSVC  0.333333  0.333333   \n",
-              "1   0.333333  0.333333          LogisticRegression  0.333333  0.333333   \n",
+              "1   0.104444  0.044444  GradientBoostingClassifier  0.333333  0.224444   \n",
+              "9   0.104444  0.091111        KNeighborsClassifier  0.335556  0.340000   \n",
+              "0   0.333333  0.333333          LogisticRegression  0.333333  0.333333   \n",
               "7   0.333333  0.333333               MLPClassifier  0.333333  0.333333   \n",
-              "3   0.051111  0.004444      RandomForestClassifier  0.340000  0.042222   \n",
-              "10  0.200000  0.197778                         SVC  0.333333  0.333333   \n",
+              "2   0.053333  0.002222      RandomForestClassifier  0.333333  0.024444   \n",
+              "5   0.333333  0.053333               XGBClassifier  0.333333  0.315556   \n",
               "\n",
               "                      modelACP  \n",
-              "11          AdaBoostClassifier  \n",
-              "4       DecisionTreeClassifier  \n",
-              "5          ExtraTreeClassifier  \n",
+              "10          AdaBoostClassifier  \n",
+              "3       DecisionTreeClassifier  \n",
+              "4          ExtraTreeClassifier  \n",
               "6         ExtraTreesClassifier  \n",
               "8                   GaussianNB  \n",
-              "2   GradientBoostingClassifier  \n",
+              "1   GradientBoostingClassifier  \n",
               "9         KNeighborsClassifier  \n",
-              "0                    LinearSVC  \n",
-              "1           LogisticRegression  \n",
+              "0           LogisticRegression  \n",
               "7                MLPClassifier  \n",
-              "3       RandomForestClassifier  \n",
-              "10                         SVC  "
+              "2       RandomForestClassifier  \n",
+              "5                XGBClassifier  "
             ]
           },
           "execution_count": 15,
@@ -1352,16 +1323,6 @@
       "execution_count": 15,
       "metadata": {},
       "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n",
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n"
-          ]
-        },
         {
           "data": {
             "text/html": [
@@ -1394,7 +1355,7 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>11</th>\n",
+              "      <th>10</th>\n",
               "      <td>AdaBoostClassifier</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -1404,34 +1365,34 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>4</th>\n",
+              "      <th>3</th>\n",
               "      <td>DecisionTreeClassifier</td>\n",
               "      <td>0.026667</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.546667</td>\n",
-              "      <td>0.220000</td>\n",
-              "      <td>0.293333</td>\n",
-              "      <td>0.573333</td>\n",
+              "      <td>0.226667</td>\n",
+              "      <td>0.206667</td>\n",
+              "      <td>0.273333</td>\n",
+              "      <td>0.313333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>5</th>\n",
+              "      <th>4</th>\n",
               "      <td>ExtraTreeClassifier</td>\n",
               "      <td>0.026667</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.206667</td>\n",
+              "      <td>0.253333</td>\n",
+              "      <td>0.213333</td>\n",
+              "      <td>0.253333</td>\n",
               "      <td>0.273333</td>\n",
-              "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>6</th>\n",
               "      <td>ExtraTreesClassifier</td>\n",
               "      <td>0.026667</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.126667</td>\n",
-              "      <td>0.180000</td>\n",
-              "      <td>0.166667</td>\n",
-              "      <td>0.153333</td>\n",
+              "      <td>0.140000</td>\n",
+              "      <td>0.200000</td>\n",
+              "      <td>0.213333</td>\n",
+              "      <td>0.220000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>8</th>\n",
@@ -1444,37 +1405,27 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>2</th>\n",
+              "      <th>1</th>\n",
               "      <td>GradientBoostingClassifier</td>\n",
               "      <td>0.080000</td>\n",
-              "      <td>0.016667</td>\n",
-              "      <td>0.183333</td>\n",
-              "      <td>0.166667</td>\n",
-              "      <td>0.280000</td>\n",
-              "      <td>0.226667</td>\n",
+              "      <td>0.013333</td>\n",
+              "      <td>0.176667</td>\n",
+              "      <td>0.186667</td>\n",
+              "      <td>0.246667</td>\n",
+              "      <td>0.240000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>9</th>\n",
               "      <td>KNeighborsClassifier</td>\n",
-              "      <td>0.116667</td>\n",
-              "      <td>0.090000</td>\n",
-              "      <td>0.113333</td>\n",
-              "      <td>0.133333</td>\n",
+              "      <td>0.070000</td>\n",
+              "      <td>0.076667</td>\n",
+              "      <td>0.073333</td>\n",
+              "      <td>0.160000</td>\n",
+              "      <td>0.160000</td>\n",
               "      <td>0.166667</td>\n",
-              "      <td>0.153333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>0</th>\n",
-              "      <td>LinearSVC</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
               "      <td>LogisticRegression</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -1494,24 +1445,24 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>3</th>\n",
+              "      <th>2</th>\n",
               "      <td>RandomForestClassifier</td>\n",
               "      <td>0.026667</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.193333</td>\n",
-              "      <td>0.180000</td>\n",
+              "      <td>0.156667</td>\n",
+              "      <td>0.206667</td>\n",
+              "      <td>0.266667</td>\n",
               "      <td>0.213333</td>\n",
-              "      <td>0.173333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>10</th>\n",
-              "      <td>SVC</td>\n",
-              "      <td>0.253333</td>\n",
-              "      <td>0.236667</td>\n",
-              "      <td>0.243333</td>\n",
-              "      <td>0.253333</td>\n",
-              "      <td>0.246667</td>\n",
-              "      <td>0.246667</td>\n",
+              "      <th>5</th>\n",
+              "      <td>XGBClassifier</td>\n",
+              "      <td>0.106667</td>\n",
+              "      <td>0.036667</td>\n",
+              "      <td>0.333333</td>\n",
+              "      <td>0.193333</td>\n",
+              "      <td>0.280000</td>\n",
+              "      <td>0.346667</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -1519,32 +1470,30 @@
             ],
             "text/plain": [
               "                       modelTT  err_train  err2_train  err2b_train_clean  \\\n",
-              "11          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
-              "4       DecisionTreeClassifier   0.026667    0.000000           0.546667   \n",
-              "5          ExtraTreeClassifier   0.026667    0.000000           0.333333   \n",
-              "6         ExtraTreesClassifier   0.026667    0.000000           0.126667   \n",
+              "10          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
+              "3       DecisionTreeClassifier   0.026667    0.000000           0.226667   \n",
+              "4          ExtraTreeClassifier   0.026667    0.000000           0.253333   \n",
+              "6         ExtraTreesClassifier   0.026667    0.000000           0.140000   \n",
               "8                   GaussianNB   0.333333    0.333333           0.333333   \n",
-              "2   GradientBoostingClassifier   0.080000    0.016667           0.183333   \n",
-              "9         KNeighborsClassifier   0.116667    0.090000           0.113333   \n",
-              "0                    LinearSVC   0.333333    0.333333           0.333333   \n",
-              "1           LogisticRegression   0.333333    0.333333           0.333333   \n",
+              "1   GradientBoostingClassifier   0.080000    0.013333           0.176667   \n",
+              "9         KNeighborsClassifier   0.070000    0.076667           0.073333   \n",
+              "0           LogisticRegression   0.333333    0.333333           0.333333   \n",
               "7                MLPClassifier   0.333333    0.333333           0.333333   \n",
-              "3       RandomForestClassifier   0.026667    0.000000           0.193333   \n",
-              "10                         SVC   0.253333    0.236667           0.243333   \n",
+              "2       RandomForestClassifier   0.026667    0.000000           0.156667   \n",
+              "5                XGBClassifier   0.106667    0.036667           0.333333   \n",
               "\n",
               "    err_test  err2_test  err2b_test_clean  \n",
-              "11  0.333333   0.333333          0.333333  \n",
-              "4   0.220000   0.293333          0.573333  \n",
-              "5   0.206667   0.273333          0.333333  \n",
-              "6   0.180000   0.166667          0.153333  \n",
+              "10  0.333333   0.333333          0.333333  \n",
+              "3   0.206667   0.273333          0.313333  \n",
+              "4   0.213333   0.253333          0.273333  \n",
+              "6   0.200000   0.213333          0.220000  \n",
               "8   0.333333   0.333333          0.333333  \n",
-              "2   0.166667   0.280000          0.226667  \n",
-              "9   0.133333   0.166667          0.153333  \n",
+              "1   0.186667   0.246667          0.240000  \n",
+              "9   0.160000   0.160000          0.166667  \n",
               "0   0.333333   0.333333          0.333333  \n",
-              "1   0.333333   0.333333          0.333333  \n",
               "7   0.333333   0.333333          0.333333  \n",
-              "3   0.180000   0.213333          0.173333  \n",
-              "10  0.253333   0.246667          0.246667  "
+              "2   0.206667   0.266667          0.213333  \n",
+              "5   0.193333   0.280000          0.346667  "
             ]
           },
           "execution_count": 16,
@@ -1629,16 +1578,6 @@
       "execution_count": 16,
       "metadata": {},
       "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n",
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n"
-          ]
-        },
         {
           "data": {
             "text/html": [
@@ -1671,7 +1610,7 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>11</th>\n",
+              "      <th>10</th>\n",
               "      <td>AdaBoostClassifier</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -1681,34 +1620,34 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>4</th>\n",
+              "      <th>3</th>\n",
               "      <td>DecisionTreeClassifier</td>\n",
-              "      <td>0.046667</td>\n",
+              "      <td>0.033333</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.200000</td>\n",
-              "      <td>0.160000</td>\n",
-              "      <td>0.260000</td>\n",
-              "      <td>0.226667</td>\n",
+              "      <td>0.143333</td>\n",
+              "      <td>0.193333</td>\n",
+              "      <td>0.273333</td>\n",
+              "      <td>0.206667</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>5</th>\n",
+              "      <th>4</th>\n",
               "      <td>ExtraTreeClassifier</td>\n",
-              "      <td>0.046667</td>\n",
+              "      <td>0.033333</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.160000</td>\n",
+              "      <td>0.143333</td>\n",
               "      <td>0.226667</td>\n",
-              "      <td>0.273333</td>\n",
-              "      <td>0.160000</td>\n",
+              "      <td>0.233333</td>\n",
+              "      <td>0.180000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>6</th>\n",
               "      <td>ExtraTreesClassifier</td>\n",
-              "      <td>0.046667</td>\n",
+              "      <td>0.033333</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.156667</td>\n",
-              "      <td>0.180000</td>\n",
+              "      <td>0.123333</td>\n",
+              "      <td>0.200000</td>\n",
+              "      <td>0.213333</td>\n",
               "      <td>0.193333</td>\n",
-              "      <td>0.180000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>8</th>\n",
@@ -1721,37 +1660,27 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>2</th>\n",
+              "      <th>1</th>\n",
               "      <td>GradientBoostingClassifier</td>\n",
-              "      <td>0.110000</td>\n",
-              "      <td>0.016667</td>\n",
-              "      <td>0.160000</td>\n",
-              "      <td>0.133333</td>\n",
-              "      <td>0.220000</td>\n",
+              "      <td>0.083333</td>\n",
+              "      <td>0.013333</td>\n",
+              "      <td>0.203333</td>\n",
               "      <td>0.193333</td>\n",
+              "      <td>0.226667</td>\n",
+              "      <td>0.280000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>9</th>\n",
               "      <td>KNeighborsClassifier</td>\n",
-              "      <td>0.096667</td>\n",
-              "      <td>0.103333</td>\n",
-              "      <td>0.090000</td>\n",
-              "      <td>0.146667</td>\n",
-              "      <td>0.153333</td>\n",
-              "      <td>0.146667</td>\n",
+              "      <td>0.106667</td>\n",
+              "      <td>0.106667</td>\n",
+              "      <td>0.100000</td>\n",
+              "      <td>0.180000</td>\n",
+              "      <td>0.180000</td>\n",
+              "      <td>0.193333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>0</th>\n",
-              "      <td>LinearSVC</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
               "      <td>LogisticRegression</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -1771,24 +1700,24 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>3</th>\n",
+              "      <th>2</th>\n",
               "      <td>RandomForestClassifier</td>\n",
-              "      <td>0.050000</td>\n",
+              "      <td>0.040000</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.163333</td>\n",
-              "      <td>0.180000</td>\n",
-              "      <td>0.140000</td>\n",
-              "      <td>0.160000</td>\n",
+              "      <td>0.176667</td>\n",
+              "      <td>0.206667</td>\n",
+              "      <td>0.240000</td>\n",
+              "      <td>0.253333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>10</th>\n",
-              "      <td>SVC</td>\n",
-              "      <td>0.260000</td>\n",
-              "      <td>0.256667</td>\n",
-              "      <td>0.256667</td>\n",
-              "      <td>0.240000</td>\n",
+              "      <th>5</th>\n",
+              "      <td>XGBClassifier</td>\n",
+              "      <td>0.080000</td>\n",
+              "      <td>0.063333</td>\n",
+              "      <td>0.170000</td>\n",
+              "      <td>0.186667</td>\n",
+              "      <td>0.220000</td>\n",
               "      <td>0.240000</td>\n",
-              "      <td>0.246667</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -1796,32 +1725,30 @@
             ],
             "text/plain": [
               "                       modelTT  err_train  err2_train  err2b_train_clean  \\\n",
-              "11          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
-              "4       DecisionTreeClassifier   0.046667    0.000000           0.200000   \n",
-              "5          ExtraTreeClassifier   0.046667    0.000000           0.160000   \n",
-              "6         ExtraTreesClassifier   0.046667    0.000000           0.156667   \n",
+              "10          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
+              "3       DecisionTreeClassifier   0.033333    0.000000           0.143333   \n",
+              "4          ExtraTreeClassifier   0.033333    0.000000           0.143333   \n",
+              "6         ExtraTreesClassifier   0.033333    0.000000           0.123333   \n",
               "8                   GaussianNB   0.333333    0.333333           0.333333   \n",
-              "2   GradientBoostingClassifier   0.110000    0.016667           0.160000   \n",
-              "9         KNeighborsClassifier   0.096667    0.103333           0.090000   \n",
-              "0                    LinearSVC   0.333333    0.333333           0.333333   \n",
-              "1           LogisticRegression   0.333333    0.333333           0.333333   \n",
+              "1   GradientBoostingClassifier   0.083333    0.013333           0.203333   \n",
+              "9         KNeighborsClassifier   0.106667    0.106667           0.100000   \n",
+              "0           LogisticRegression   0.333333    0.333333           0.333333   \n",
               "7                MLPClassifier   0.333333    0.333333           0.333333   \n",
-              "3       RandomForestClassifier   0.050000    0.000000           0.163333   \n",
-              "10                         SVC   0.260000    0.256667           0.256667   \n",
+              "2       RandomForestClassifier   0.040000    0.000000           0.176667   \n",
+              "5                XGBClassifier   0.080000    0.063333           0.170000   \n",
               "\n",
               "    err_test  err2_test  err2b_test_clean  \n",
-              "11  0.333333   0.333333          0.333333  \n",
-              "4   0.160000   0.260000          0.226667  \n",
-              "5   0.226667   0.273333          0.160000  \n",
-              "6   0.180000   0.193333          0.180000  \n",
+              "10  0.333333   0.333333          0.333333  \n",
+              "3   0.193333   0.273333          0.206667  \n",
+              "4   0.226667   0.233333          0.180000  \n",
+              "6   0.200000   0.213333          0.193333  \n",
               "8   0.333333   0.333333          0.333333  \n",
-              "2   0.133333   0.220000          0.193333  \n",
-              "9   0.146667   0.153333          0.146667  \n",
+              "1   0.193333   0.226667          0.280000  \n",
+              "9   0.180000   0.180000          0.193333  \n",
               "0   0.333333   0.333333          0.333333  \n",
-              "1   0.333333   0.333333          0.333333  \n",
               "7   0.333333   0.333333          0.333333  \n",
-              "3   0.180000   0.140000          0.160000  \n",
-              "10  0.240000   0.240000          0.246667  "
+              "2   0.206667   0.240000          0.253333  \n",
+              "5   0.186667   0.220000          0.240000  "
             ]
           },
           "execution_count": 17,
@@ -1864,16 +1791,6 @@
       "execution_count": 17,
       "metadata": {},
       "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n",
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n"
-          ]
-        },
         {
           "data": {
             "text/html": [
@@ -1906,7 +1823,7 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>11</th>\n",
+              "      <th>10</th>\n",
               "      <td>AdaBoostClassifier</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -1916,34 +1833,34 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>4</th>\n",
+              "      <th>3</th>\n",
               "      <td>DecisionTreeClassifier</td>\n",
-              "      <td>0.033333</td>\n",
+              "      <td>0.040000</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.170000</td>\n",
-              "      <td>0.206667</td>\n",
-              "      <td>0.229333</td>\n",
-              "      <td>0.226667</td>\n",
+              "      <td>0.120000</td>\n",
+              "      <td>0.180000</td>\n",
+              "      <td>0.209333</td>\n",
+              "      <td>0.240000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>5</th>\n",
+              "      <th>4</th>\n",
               "      <td>ExtraTreeClassifier</td>\n",
-              "      <td>0.033333</td>\n",
+              "      <td>0.040000</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.140000</td>\n",
-              "      <td>0.233333</td>\n",
-              "      <td>0.233333</td>\n",
+              "      <td>0.073333</td>\n",
+              "      <td>0.213333</td>\n",
+              "      <td>0.232000</td>\n",
               "      <td>0.220000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>6</th>\n",
               "      <td>ExtraTreesClassifier</td>\n",
-              "      <td>0.033333</td>\n",
+              "      <td>0.040000</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.060000</td>\n",
-              "      <td>0.200000</td>\n",
-              "      <td>0.202667</td>\n",
-              "      <td>0.186667</td>\n",
+              "      <td>0.066667</td>\n",
+              "      <td>0.213333</td>\n",
+              "      <td>0.168000</td>\n",
+              "      <td>0.160000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>8</th>\n",
@@ -1956,37 +1873,27 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>2</th>\n",
+              "      <th>1</th>\n",
               "      <td>GradientBoostingClassifier</td>\n",
-              "      <td>0.096667</td>\n",
-              "      <td>0.080000</td>\n",
-              "      <td>0.150000</td>\n",
+              "      <td>0.086667</td>\n",
+              "      <td>0.087333</td>\n",
+              "      <td>0.166667</td>\n",
+              "      <td>0.173333</td>\n",
+              "      <td>0.192000</td>\n",
               "      <td>0.186667</td>\n",
-              "      <td>0.208000</td>\n",
-              "      <td>0.220000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>9</th>\n",
               "      <td>KNeighborsClassifier</td>\n",
-              "      <td>0.100000</td>\n",
-              "      <td>0.079333</td>\n",
-              "      <td>0.093333</td>\n",
-              "      <td>0.186667</td>\n",
-              "      <td>0.188000</td>\n",
-              "      <td>0.226667</td>\n",
+              "      <td>0.110000</td>\n",
+              "      <td>0.094667</td>\n",
+              "      <td>0.106667</td>\n",
+              "      <td>0.113333</td>\n",
+              "      <td>0.158667</td>\n",
+              "      <td>0.153333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>0</th>\n",
-              "      <td>LinearSVC</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
               "      <td>LogisticRegression</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -2006,23 +1913,23 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>3</th>\n",
+              "      <th>2</th>\n",
               "      <td>RandomForestClassifier</td>\n",
-              "      <td>0.036667</td>\n",
-              "      <td>0.000000</td>\n",
-              "      <td>0.076667</td>\n",
-              "      <td>0.193333</td>\n",
-              "      <td>0.196000</td>\n",
-              "      <td>0.140000</td>\n",
+              "      <td>0.046667</td>\n",
+              "      <td>0.000667</td>\n",
+              "      <td>0.090000</td>\n",
+              "      <td>0.160000</td>\n",
+              "      <td>0.188000</td>\n",
+              "      <td>0.226667</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>10</th>\n",
-              "      <td>SVC</td>\n",
-              "      <td>0.203333</td>\n",
-              "      <td>0.181333</td>\n",
+              "      <th>5</th>\n",
+              "      <td>XGBClassifier</td>\n",
+              "      <td>0.123333</td>\n",
+              "      <td>0.108667</td>\n",
               "      <td>0.173333</td>\n",
-              "      <td>0.226667</td>\n",
-              "      <td>0.216000</td>\n",
+              "      <td>0.153333</td>\n",
+              "      <td>0.204000</td>\n",
               "      <td>0.193333</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
@@ -2031,32 +1938,30 @@
             ],
             "text/plain": [
               "                       modelTT  err_train  err2_train  err2b_train_clean  \\\n",
-              "11          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
-              "4       DecisionTreeClassifier   0.033333    0.000000           0.170000   \n",
-              "5          ExtraTreeClassifier   0.033333    0.000000           0.140000   \n",
-              "6         ExtraTreesClassifier   0.033333    0.000000           0.060000   \n",
+              "10          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
+              "3       DecisionTreeClassifier   0.040000    0.000000           0.120000   \n",
+              "4          ExtraTreeClassifier   0.040000    0.000000           0.073333   \n",
+              "6         ExtraTreesClassifier   0.040000    0.000000           0.066667   \n",
               "8                   GaussianNB   0.333333    0.333333           0.333333   \n",
-              "2   GradientBoostingClassifier   0.096667    0.080000           0.150000   \n",
-              "9         KNeighborsClassifier   0.100000    0.079333           0.093333   \n",
-              "0                    LinearSVC   0.333333    0.333333           0.333333   \n",
-              "1           LogisticRegression   0.333333    0.333333           0.333333   \n",
+              "1   GradientBoostingClassifier   0.086667    0.087333           0.166667   \n",
+              "9         KNeighborsClassifier   0.110000    0.094667           0.106667   \n",
+              "0           LogisticRegression   0.333333    0.333333           0.333333   \n",
               "7                MLPClassifier   0.333333    0.333333           0.333333   \n",
-              "3       RandomForestClassifier   0.036667    0.000000           0.076667   \n",
-              "10                         SVC   0.203333    0.181333           0.173333   \n",
+              "2       RandomForestClassifier   0.046667    0.000667           0.090000   \n",
+              "5                XGBClassifier   0.123333    0.108667           0.173333   \n",
               "\n",
               "    err_test  err2_test  err2b_test_clean  \n",
-              "11  0.333333   0.333333          0.333333  \n",
-              "4   0.206667   0.229333          0.226667  \n",
-              "5   0.233333   0.233333          0.220000  \n",
-              "6   0.200000   0.202667          0.186667  \n",
+              "10  0.333333   0.333333          0.333333  \n",
+              "3   0.180000   0.209333          0.240000  \n",
+              "4   0.213333   0.232000          0.220000  \n",
+              "6   0.213333   0.168000          0.160000  \n",
               "8   0.333333   0.333333          0.333333  \n",
-              "2   0.186667   0.208000          0.220000  \n",
-              "9   0.186667   0.188000          0.226667  \n",
+              "1   0.173333   0.192000          0.186667  \n",
+              "9   0.113333   0.158667          0.153333  \n",
               "0   0.333333   0.333333          0.333333  \n",
-              "1   0.333333   0.333333          0.333333  \n",
               "7   0.333333   0.333333          0.333333  \n",
-              "3   0.193333   0.196000          0.140000  \n",
-              "10  0.226667   0.216000          0.193333  "
+              "2   0.160000   0.188000          0.226667  \n",
+              "5   0.153333   0.204000          0.193333  "
             ]
           },
           "execution_count": 18,
@@ -2101,16 +2006,6 @@
       "execution_count": 18,
       "metadata": {},
       "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n",
-            "c:\\python370_x64\\lib\\site-packages\\sklearn\\svm\\base.py:922: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-            "  \"the number of iterations.\", ConvergenceWarning)\n"
-          ]
-        },
         {
           "data": {
             "text/html": [
@@ -2143,7 +2038,7 @@
               "  </thead>\n",
               "  <tbody>\n",
               "    <tr>\n",
-              "      <th>11</th>\n",
+              "      <th>10</th>\n",
               "      <td>AdaBoostClassifier</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -2153,34 +2048,34 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>4</th>\n",
+              "      <th>3</th>\n",
               "      <td>DecisionTreeClassifier</td>\n",
               "      <td>0.033333</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.133333</td>\n",
-              "      <td>0.220000</td>\n",
-              "      <td>0.313333</td>\n",
-              "      <td>0.246667</td>\n",
+              "      <td>0.143333</td>\n",
+              "      <td>0.200000</td>\n",
+              "      <td>0.233333</td>\n",
+              "      <td>0.193333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>5</th>\n",
+              "      <th>4</th>\n",
               "      <td>ExtraTreeClassifier</td>\n",
               "      <td>0.033333</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.180000</td>\n",
-              "      <td>0.226667</td>\n",
-              "      <td>0.286667</td>\n",
-              "      <td>0.200000</td>\n",
+              "      <td>0.246667</td>\n",
+              "      <td>0.233333</td>\n",
+              "      <td>0.320000</td>\n",
+              "      <td>0.300000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>6</th>\n",
               "      <td>ExtraTreesClassifier</td>\n",
               "      <td>0.033333</td>\n",
               "      <td>0.000000</td>\n",
-              "      <td>0.126667</td>\n",
+              "      <td>0.143333</td>\n",
               "      <td>0.206667</td>\n",
-              "      <td>0.160000</td>\n",
-              "      <td>0.193333</td>\n",
+              "      <td>0.220000</td>\n",
+              "      <td>0.180000</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>8</th>\n",
@@ -2193,37 +2088,27 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>2</th>\n",
+              "      <th>1</th>\n",
               "      <td>GradientBoostingClassifier</td>\n",
-              "      <td>0.063333</td>\n",
-              "      <td>0.006667</td>\n",
-              "      <td>0.136667</td>\n",
-              "      <td>0.200000</td>\n",
-              "      <td>0.246667</td>\n",
-              "      <td>0.213333</td>\n",
+              "      <td>0.090000</td>\n",
+              "      <td>0.013333</td>\n",
+              "      <td>0.133333</td>\n",
+              "      <td>0.220000</td>\n",
+              "      <td>0.206667</td>\n",
+              "      <td>0.186667</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>9</th>\n",
               "      <td>KNeighborsClassifier</td>\n",
-              "      <td>0.100000</td>\n",
-              "      <td>0.120000</td>\n",
-              "      <td>0.120000</td>\n",
-              "      <td>0.200000</td>\n",
-              "      <td>0.146667</td>\n",
-              "      <td>0.146667</td>\n",
+              "      <td>0.103333</td>\n",
+              "      <td>0.110000</td>\n",
+              "      <td>0.123333</td>\n",
+              "      <td>0.206667</td>\n",
+              "      <td>0.180000</td>\n",
+              "      <td>0.186667</td>\n",
               "    </tr>\n",
               "    <tr>\n",
               "      <th>0</th>\n",
-              "      <td>LinearSVC</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "      <td>0.333333</td>\n",
-              "    </tr>\n",
-              "    <tr>\n",
-              "      <th>1</th>\n",
               "      <td>LogisticRegression</td>\n",
               "      <td>0.333333</td>\n",
               "      <td>0.333333</td>\n",
@@ -2243,24 +2128,24 @@
               "      <td>0.333333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>3</th>\n",
+              "      <th>2</th>\n",
               "      <td>RandomForestClassifier</td>\n",
-              "      <td>0.033333</td>\n",
-              "      <td>0.003333</td>\n",
-              "      <td>0.160000</td>\n",
-              "      <td>0.193333</td>\n",
-              "      <td>0.293333</td>\n",
-              "      <td>0.206667</td>\n",
+              "      <td>0.040000</td>\n",
+              "      <td>0.000000</td>\n",
+              "      <td>0.146667</td>\n",
+              "      <td>0.180000</td>\n",
+              "      <td>0.266667</td>\n",
+              "      <td>0.173333</td>\n",
               "    </tr>\n",
               "    <tr>\n",
-              "      <th>10</th>\n",
-              "      <td>SVC</td>\n",
-              "      <td>0.223333</td>\n",
-              "      <td>0.290000</td>\n",
-              "      <td>0.296667</td>\n",
+              "      <th>5</th>\n",
+              "      <td>XGBClassifier</td>\n",
+              "      <td>0.100000</td>\n",
+              "      <td>0.033333</td>\n",
+              "      <td>0.210000</td>\n",
+              "      <td>0.206667</td>\n",
+              "      <td>0.240000</td>\n",
               "      <td>0.246667</td>\n",
-              "      <td>0.300000</td>\n",
-              "      <td>0.293333</td>\n",
               "    </tr>\n",
               "  </tbody>\n",
               "</table>\n",
@@ -2268,32 +2153,30 @@
             ],
             "text/plain": [
               "                       modelTT  err_train  err2_train  err2b_train_clean  \\\n",
-              "11          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
-              "4       DecisionTreeClassifier   0.033333    0.000000           0.133333   \n",
-              "5          ExtraTreeClassifier   0.033333    0.000000           0.180000   \n",
-              "6         ExtraTreesClassifier   0.033333    0.000000           0.126667   \n",
+              "10          AdaBoostClassifier   0.333333    0.333333           0.333333   \n",
+              "3       DecisionTreeClassifier   0.033333    0.000000           0.143333   \n",
+              "4          ExtraTreeClassifier   0.033333    0.000000           0.246667   \n",
+              "6         ExtraTreesClassifier   0.033333    0.000000           0.143333   \n",
               "8                   GaussianNB   0.333333    0.333333           0.333333   \n",
-              "2   GradientBoostingClassifier   0.063333    0.006667           0.136667   \n",
-              "9         KNeighborsClassifier   0.100000    0.120000           0.120000   \n",
-              "0                    LinearSVC   0.333333    0.333333           0.333333   \n",
-              "1           LogisticRegression   0.333333    0.333333           0.333333   \n",
+              "1   GradientBoostingClassifier   0.090000    0.013333           0.133333   \n",
+              "9         KNeighborsClassifier   0.103333    0.110000           0.123333   \n",
+              "0           LogisticRegression   0.333333    0.333333           0.333333   \n",
               "7                MLPClassifier   0.333333    0.333333           0.333333   \n",
-              "3       RandomForestClassifier   0.033333    0.003333           0.160000   \n",
-              "10                         SVC   0.223333    0.290000           0.296667   \n",
+              "2       RandomForestClassifier   0.040000    0.000000           0.146667   \n",
+              "5                XGBClassifier   0.100000    0.033333           0.210000   \n",
               "\n",
               "    err_test  err2_test  err2b_test_clean  \n",
-              "11  0.333333   0.333333          0.333333  \n",
-              "4   0.220000   0.313333          0.246667  \n",
-              "5   0.226667   0.286667          0.200000  \n",
-              "6   0.206667   0.160000          0.193333  \n",
+              "10  0.333333   0.333333          0.333333  \n",
+              "3   0.200000   0.233333          0.193333  \n",
+              "4   0.233333   0.320000          0.300000  \n",
+              "6   0.206667   0.220000          0.180000  \n",
               "8   0.333333   0.333333          0.333333  \n",
-              "2   0.200000   0.246667          0.213333  \n",
-              "9   0.200000   0.146667          0.146667  \n",
+              "1   0.220000   0.206667          0.186667  \n",
+              "9   0.206667   0.180000          0.186667  \n",
               "0   0.333333   0.333333          0.333333  \n",
-              "1   0.333333   0.333333          0.333333  \n",
               "7   0.333333   0.333333          0.333333  \n",
-              "3   0.193333   0.293333          0.206667  \n",
-              "10  0.246667   0.300000          0.293333  "
+              "2   0.180000   0.266667          0.173333  \n",
+              "5   0.206667   0.240000          0.246667  "
             ]
           },
           "execution_count": 19,
diff --git a/appveyor.yml b/appveyor.yml
index ff5b5c7..b85c5d0 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -21,6 +21,7 @@ install:
   - "%PYTHON%\\Scripts\\pymy_install3 --set=pyensae"
   - "%PYTHON%\\Scripts\\pymy_install3 pycrypto pyzmq statsmodels minepy opencv_python"
   - "%PYTHON%\\Scripts\\pymy_install3 --task=tool --source=zip graphviz"
+  - conda install -c conda-forge xgboost
   - "%PYTHON%\\Scripts\\pip install -r requirements_conda.txt"
   - "%PYTHON%\\Scripts\\pip install -r requirements.txt"
   - "set PATH=%PATH%;C:\\projects\\mlstatpy\\build\\update_modules\\Graphviz\\bin"
diff --git a/requirements_conda.txt b/requirements_conda.txt
index 98dc5d4..e444064 100644
--- a/requirements_conda.txt
+++ b/requirements_conda.txt
@@ -23,3 +23,4 @@ setuptools
 Sphinx
 statsmodels
 xarray
+xgboost