No info rate

docs/sources/CHANGELOG.md

@@ -27,7 +27,8 @@ The CHANGELOG for the current development version is available at
 
 ##### Bug Fixes
 
-- Fix axis DeprecationWarning in matplotlib v3.1.0 and newer. ([#673](https://github.com/rasbt/mlxtend/pull/673))
+- Fixes axis DeprecationWarning in matplotlib v3.1.0 and newer. ([#673](https://github.com/rasbt/mlxtend/pull/673))
+- Fixes an issue with using `meshgrid` in `no_information_rate` function used by the `bootstrap_point632_score` function for the .632+ estimate. ([#688](https://github.com/rasbt/mlxtend/pull/688))
 
 ### Version 0.17.2 (02-24-2020)
 

docs/sources/user_guide/data/loadlocal_mnist.ipynb

@@ -101,29 +101,34 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from mlxtend.data import loadlocal_mnist"
+    "from mlxtend.data import loadlocal_mnist\n",
+    "import platform"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
+    "if not platform.system() == 'Windows':\n",
+    "    X, y = loadlocal_mnist(\n",
+    "            images_path='train-images-idx3-ubyte', \n",
+    "            labels_path='train-labels-idx1-ubyte')\n",
     "\n",
-    "\n",
-    "X, y = loadlocal_mnist(\n",
-    "        images_path='/Users/Sebastian/Desktop/train-images-idx3-ubyte', \n",
-    "        labels_path='/Users/Sebastian/Desktop/train-labels-idx1-ubyte')\n"
+    "else:\n",
+    "    X, y = loadlocal_mnist(\n",
+    "            images_path='train-images.idx3-ubyte', \n",
+    "            labels_path='train-labels.idx1-ubyte')"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -186,7 +191,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -216,15 +221,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 5,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "np.savetxt(fname='/Users/Sebastian/Desktop/images.csv', \n",
+    "np.savetxt(fname='images.csv', \n",
     "           X=X, delimiter=',', fmt='%d')\n",
-    "np.savetxt(fname='/Users/Sebastian/Desktop/labels.csv', \n",
+    "np.savetxt(fname='labels.csv', \n",
     "           X=y, delimiter=',', fmt='%d')"
    ]
   },
@@ -237,7 +240,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -318,5 +321,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

docs/sources/user_guide/evaluate/bootstrap_point632_score.ipynb

@@ -121,8 +121,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Accuracy: 94.52%\n",
-      "95% Confidence interval: [88.88, 98.28]\n"
+      "Accuracy: 94.36%\n",
+      "95% Confidence interval: [88.46, 98.31]\n"
      ]
     }
    ],
@@ -165,8 +165,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Accuracy: 96.58%\n",
-      "95% Confidence interval: [92.37, 98.97]\n"
+      "Accuracy: 96.57%\n",
+      "95% Confidence interval: [92.37, 98.95]\n"
      ]
     }
    ],
@@ -209,8 +209,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Accuracy: 96.40%\n",
-      "95% Confidence interval: [92.34, 99.00]\n"
+      "Accuracy: 96.28%\n",
+      "95% Confidence interval: [92.10, 98.90]\n"
      ]
     }
    ],
@@ -383,7 +383,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.7"
+   "version": "3.7.1"
   },
   "toc": {
    "nav_menu": {},
@@ -399,5 +399,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
 }

mlxtend/evaluate/bootstrap_point632.py

@@ -9,6 +9,7 @@
 import numpy as np
 from .bootstrap_outofbag import BootstrapOutOfBag
 from sklearn.base import clone
+from itertools import product
 
 
 def _check_arrays(X, y=None):
@@ -29,7 +30,7 @@ def _check_arrays(X, y=None):
 
 
 def no_information_rate(targets, predictions, loss_fn):
-    combinations = np.array(np.meshgrid(targets, predictions)).reshape(-1, 2)
+    combinations = np.array(list(product(targets, predictions)))
     return loss_fn(combinations[:, 0], combinations[:, 1])
 
 
@@ -164,18 +165,21 @@ def bootstrap_point632_score(estimator, X, y, n_splits=200,
             acc = test_acc
 
         else:
-            train_acc = scoring_func(y[train], cloned_est.predict(X[train]))
+            test_err = 1 - test_acc
+            train_err = 1 - scoring_func(y[train],
+                                         cloned_est.predict(X[train]))
             if method == '.632+':
-                gamma = no_information_rate(y,
-                                            cloned_est.predict(X),
-                                            scoring_func)
-                R = (-(test_acc - train_acc)) / (gamma - (1 - test_acc))
-                weight = 0.632 / (1-0.368 * R)
+                gamma = 1 - (no_information_rate(
+                    y,
+                    cloned_est.predict(X),
+                    scoring_func))
+                R = (test_err - train_err) / (gamma - train_err)
+                weight = 0.632 / (1 - 0.368*R)
 
             else:
                 weight = 0.632
 
-            acc = weight*test_acc + (1. - weight)*train_acc
+            acc = 1 - (weight*test_err + (1. - weight)*train_err)
 
         scores[cnt] = acc
         cnt += 1

mlxtend/evaluate/tests/test_bootstrap_point632.py

@@ -52,14 +52,14 @@ def test_632plus():
                                       method='.632+')
     acc = np.mean(scores)
     assert len(scores == 200)
-    assert np.round(acc, 5) == 0.96528, np.round(acc, 5)
+    assert np.round(acc, 5) == 0.9649, np.round(acc, 5)
 
     tree2 = DecisionTreeClassifier(random_state=123, max_depth=1)
     scores = bootstrap_point632_score(tree2, X, y, random_seed=123,
                                       method='.632+')
     acc = np.mean(scores)
     assert len(scores == 200)
-    assert np.round(acc, 5) == 0.65034, np.round(acc, 5)
+    assert np.round(acc, 5) == 0.64831, np.round(acc, 5)
 
 
 def test_custom_accuracy():