part_1_mnist.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "part_1_mnist.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QFUu7sgPx7ak"
      },
      "source": [
        "# Part 1 - MNIST\n",
        "\n",
        "<table align=\"left\"><td>\n",
        "<a target=\"_blank\" href=\"https://colab.research.google.com/github/thomasjpfan/skorch_talk/blob/master/part_1_mnist.ipynb\">\n",
        "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>  \n",
        "</td><td>\n",
        "<a target=\"_blank\" href=\"https://github.com/thomasjpfan/skorch_talk/blob/master/part_1_mnist.ipynb\"><img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a></td></table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "V8ZzeHxUx7an"
      },
      "source": [
        "**Note**: If you are running this in [a colab notebook](https://colab.research.google.com/github/thomasjpfan/skorch_talk/blob/master/part_1_mnist.ipynb), we recommend you enable a free GPU by going:\n",
        "\n",
        "> **Runtime**   →   **Change runtime type**   →   **Hardware Accelerator: GPU**\n",
        "\n",
        "If you are running in colab, you should install the dependencies and download the dataset by running the following cell:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GlTaAqGMx7ap",
        "outputId": "a815fdb1-3008-40f6-bcac-e0559752aac2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165
        }
      },
      "source": [
        "! [ ! -z \"$COLAB_GPU\" ] && pip install skorch"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: skorch in /usr/local/lib/python3.6/dist-packages (0.6.0)\n",
            "Requirement already satisfied: tqdm>=4.14.0 in /usr/local/lib/python3.6/dist-packages (from skorch) (4.28.1)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from skorch) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from skorch) (1.16.4)\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.6/dist-packages (from skorch) (0.8.3)\n",
            "Requirement already satisfied: scikit-learn>=0.19.1 in /usr/local/lib/python3.6/dist-packages (from skorch) (0.21.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.19.1->skorch) (0.13.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "GuRaoHKLx7av",
        "toc-hr-collapsed": false
      },
      "source": [
        "## Explore data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "DdDrR3qVx7aw",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from pathlib import Path\n",
        "from urllib import request"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "c_FZ5LVIx7a1",
        "outputId": "2d1d2cc6-f59c-424f-8fca-265aa127cebf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "def download_mnist(dataset_dir=Path('data')):\n",
        "    dataset_dir.mkdir(exist_ok=True)\n",
        "    mnist_data = dataset_dir / 'mnist.npz'\n",
        "    url = 'https://github.com/thomasjpfan/skorch_talk/blob/master/data/mnist.npz?raw=true'\n",
        "    if not mnist_data.exists():\n",
        "        print(\"Starting to download data...\")\n",
        "        data = request.urlopen(url, timeout=15).read()\n",
        "        with open(mnist_data, 'wb') as f:\n",
        "            f.write(data)\n",
        "    print(\"Data has been downloaded to {}.\".format(dataset_dir))\n",
        "    \n",
        "download_mnist()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Starting to download data...\n",
            "Data has been downloaded to data.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9xqMJ7Ilx7a5",
        "colab": {}
      },
      "source": [
        "mnist = np.load('data/mnist.npz', allow_pickle=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "J1oObFLOx7bC",
        "outputId": "8d00a952-cfca-477c-fd8b-487976a370bd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "X = mnist['data'].astype('float32')\n",
        "y = mnist['target'].astype('int64')\n",
        "\n",
        "print(X.shape, y.shape)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(70000, 784) (70000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MCuJPK7Vx7bF",
        "outputId": "ddb1154e-8c92-4b25-edca-ef5737bcd970",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "source": [
        "fig, axes = plt.subplots(2, 5, figsize=(10, 4))\n",
        "fig.patch.set_alpha(0)\n",
        "Path('images').mkdir(exist_ok=True)\n",
        "\n",
        "for idx, ax in enumerate(axes.flatten()):\n",
        "    ax.imshow(X[idx].reshape(28, 28), cmap='gray')\n",
        "    ax.set_title(y[idx], fontsize=20)\n",
        "    ax.set_xticks([])\n",
        "    ax.set_yticks([])\n",
        "fig.tight_layout()\n",
        "fig.savefig('images/mnist_example.png')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAArYAAAEbCAYAAAAvacdBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XvczVX+9/HPQkmEHEaqQQdKNZEo\nyU0HVEKnEca5gw4T1R0/HVSaEh00SecUg+aWSaimJiYKnW7V6H5IiqZIkYucj2Hdf+zt2/ezuLZ9\n7Wvv/d177dfz8ehxrfe19rX3Z+o72/K9PnstY60VAAAAIN+ViboAAAAAIB1Y2AIAAMALLGwBAADg\nBRa2AAAA8AILWwAAAHiBhS0AAAC8wMIWAAAAXii4ha0x5ntjjC3mn1VR14fcZIw52hjzkjHmJ2PM\njvh19Lgx5vCoa0P+MMb0CL3fXBN1Pcg9xpg/GmNGG2PmGmM2xq+ViVHXhdxmYq41xnxijNlsjNli\njPnUGHO9Maag1nrloi4gIhtE5PH9fH9ztgtB7jPGHCciH4rI70RkuogsFpEzRORmEbnQGHO2tXZt\nhCUiDxhjfi8iT0rsfaZSxOUgdw0RkUYSu05WiMiJ0ZaDPDFRRP4kIqtF5P+IyFYRaSsiz4hICxHp\nFV1p2VWoC9v11tqhUReBvPG0xBa1A6y1o/d+0xjzmIjcKiLDROT6iGpDHjDGGBEZKyJrReQ1ERkY\nbUXIYbdKbEG7VERai8jsaMtBrjPGXCaxRe13InKGtXZN/PsHi8gUEelpjJlmrX0twjKzpqBuTwMl\nFb9b205EvheRp5zpe0Vki8TeNCpmuTTklwEicp6I9JXYNQPsl7V2trV2ieW8eyTvsvjXkXsXtSIi\n1tqdInJ3PN6U9aoiUqgL2/LxXrc7jTE3G2PONcaUjboo5KRz419nWGv3hCestZtE5AMROVREmme7\nMOQHY0xDERkhIqOstXOirgeAd46If/3vfub2fu9/xe/geq9QWxGOEJEJzve+M8b0tda+H0VByFkn\nxL9+U8z8Eond0W0gIu9mpSLkDWNMOYm91ywXkTsjLgeAn/bepT1mP3PHxr+Wi48XZ6WiCBXiHdux\nInK+xBa3FUXkDyLynIjUE5G3jTGNoisNOahK/OuGYub3fr9qFmpB/rlHRE4TkT7W2m1RFwPAS/+M\nf/3fxphqe79pjDlIRO4LPa4gdvEpuDu21tr7nG8tFJHrjTGbReQ2ERkqv/WrAEBKjDFnSuwu7Uhr\n7UdR1wPAW5NEpKeIXCAii4wx00Vku4i0EZHaEvuNUR0R2VPsM3ikEO/YFufZ+NdWkVaBXLP3jmyV\nYub3fn99FmpBnoi3IIyXWAvL3Qd4OACkzFq7W0Q6isjtIlIkIr3j/yyR2FZfm+IPXR1JgVlWcHds\nEyiKf+XT7Qj7Ov61QTHz9eNfi+vBRWGqJL9dM9tju33t4wVjzAsS+1DZLVmrDIB3rLW/ishD8X8C\nxphDJPbn1Bpr7XdR1JZtLGx/s/dT7fv7VCEK1949JNsZY8qEd0YwxhwmImdLbCPsj6MoDjlrh4i8\nWMxcE4n13c6T2F+caFMAkCldReRgiR3aUBAKamEb33ZnubV2i/P9ehI7EUgkdnoHICIi1tpvjTEz\nJLbzwZ9FZHRo+j6J3eF/zr2mUNjiHxTb75G5xpihElvY/s1aOyabdQHwkzGmsrV2o/O9xiLyiIis\nk9iWgwWhoBa2ItJFRG4zxswRkWUS6zs5TkQuFpFDROQtEXk0uvKQo26U2JG6TxhjzheRr0TkTInt\ncfuNiNwVYW0APGKMuVRELo3HvfuTnmWMGRcfr7HWcnIdXDONMdsk9oH4TSLSUGJrm20i0tFa+1OU\nxWVToS1sZ0tsX9LTJPYr5IoS+9DPPIntNTmB017git+1bSoifxGRC0WkvYisFJFRInKftXZdlPUB\n8EpjiX3wJ+xY+W0/0mXCkczY16sSazvoISIVRORHEXleRIZba1dEWVi2GdZxAAAA8AHbfQEAAMAL\nLGwBAADgBRa2AAAA8AILWwAAAHiBhS0AAAC8UKLtvowxbKGQH9ZYa2tGXcReXDf5wVq733Nfo8A1\nkzd4r0EquG6QiqSuG+7Y+mlZ1AUAKAi81yAVXDdIRVLXDQtbAAAAeIGFLQAAALzAwhYAAABeYGEL\nAAAAL7CwBQAAgBdY2AIAAMALLGwBAADgBRa2AAAA8AILWwAAAHiBhS0AAAC8wMIWAAAAXmBhCwAA\nAC+wsAUAAIAXykVdAOCD008/XeWbbropGPfq1UvNjR8/XuXRo0er/Pnnn6e5OgAACgN3bAEAAOAF\nFrYAAADwAgtbAAAAeMFYa5N/sDHJPzhHlC1bVuUqVaqU6OfDvZKHHnqomjvhhBNU/vOf/6zyo48+\nGoy7deum5rZv367yiBEjVL7vvvtKVKfjM2tt09I8QTrl43VzII0bN1Z51qxZKleuXDnp59qwYYPK\n1atXT72wUrDWmkheeD98vGYy6fzzzw/GL7/8sppr3bq1yl9//XU6X5r3mhw2ZMgQld0/V8qU+e3e\n1jnnnKPm3n///YzVJVw3SE1S1w13bAEAAOAFFrYAAADwAgtbAAAAeCEv9rGtU6eOygcffLDKLVq0\nULlly5bBuGrVqmruiiuuSFtdK1asUPmJJ55Q+bLLLgvGmzZtUnNffPGFyhnuZ0IpnXHGGSpPmTJF\nZbd3O9y77v6337lzp8puT23z5s2Dsbunrfuz2FerVq1Udv/9Tp06NZvlZEWzZs2C8fz58yOsBFHq\n06ePyoMHD1Z5z549xf5sST5vA+Qy7tgCAADACyxsAQAA4IWcbEU40FZKJd2yK13cX+O4W6ls3rxZ\n5fC2OytXrlRz69atUznNW/AgBeHt3Jo0aaLmJk6cqHLt2rWTft4lS5ao/PDDD6s8adIklT/44INg\n7F5jw4cPT/p1C5W7bVH9+vVV9qEVIbxNk4jIMcccE4zr1q2r5ozJmV3ckGHuf/tDDjkkokqQaWee\neabKPXr0UNnd5u/kk08u9rkGDhyo8k8//aRyuL1TRP95+Mknnxy42Czjji0AAAC8wMIWAAAAXmBh\nCwAAAC/kZI/t8uXLVV67dq3K6eyxdftD1q9fr/K5554bjN2tliZMmJC2OhC95557Lhi7RyCXhtuv\nW6lSJZXdrd7CPaKnnnpq2uooFL169VL5o48+iqiSzHF7vK+99tpg7PaDL168OCs1IRpt2rQJxv37\n90/4WPda6NChQzD++eef01sY0q5Lly7BeNSoUWquRo0aKru99e+9957KNWvWDMaPPPJIwtd1nyv8\ns127dk34s1Hgji0AAAC8wMIWAAAAXmBhCwAAAC/kZI/tL7/8ovKgQYNUDvcFiYj85z//Udk92jZs\nwYIFKrdt21blLVu2qBze++3mm28u9nmRf04//XSVL7744mB8oL0/3b7YN954Q+VHH300GLt7ArrX\nq7un8XnnnZd0HdiXu8erj8aMGVPsnLtvMvzi7ik6duzYYHygz5+4vZTLli1LX2EotXLl9JKsadOm\nKr/wwgvBOLzvuojInDlzVL7//vtVnjdvnsrly5cPxpMnT1Zz7dq1S1jnp59+mnA+av7/CQAAAICC\nwMIWAAAAXmBhCwAAAC/kZI+ta9q0aSrPmjVL5U2bNqncqFGjYHz11VeruXDvo8i+PbWuL7/8Mhj3\n69fvwMUiZzVu3FjlmTNnqly5cuVgbK1Vc2+//bbK7j637rncQ4YMCcZuP2RRUZHKX3zxhcp79uwJ\nxuG+X5F998T9/PPPpdC5e/3WqlUrokqyJ1EvpXtdwy+9e/dW+cgjjyz2se7epePHj89ESUiTHj16\nqJyol979/3l4j1sRkY0bNyZ8rfDjD9RTu2LFCpX/9re/JXx81LhjCwAAAC+wsAUAAIAXWNgCAADA\nC3nRY+s6UO/Ihg0bip0Ln6kuIvLKK6+oHO5vRH5r0KCByu5+yG6f4po1a4LxypUr1ZzbU7R582aV\n//nPfybMqapQoYLKt912m8rdu3dPy+vks/bt26vs/jvzgds3fMwxxxT72B9//DHT5SCLatSoofJV\nV12lcvjPrPXr16u5Bx54IHOFodTcvWbvvPNOld3Pejz99NPBOPw5DpEDr4tcd911V9KPHTBggMru\n50RyDXdsAQAA4AUWtgAAAPBCXrYiHMjQoUODsXtsqrstU5s2bVSeMWNGxupCZoWPCBTZd2s391fW\n7jZxvXr1CsbukYG58uvtOnXqRF1CzjnhhBMSzoe37MtX7rXstiZ88803wdi9rpFf6tWrp/KUKVOS\n/tnRo0erPHv27HSUhDS55557VHZbD3bu3KnyO++8o/LgwYOD8bZt2xK+1iGHHKKyu6VX+M8S9+h2\nt4Vl+vTpCV8r13DHFgAAAF5gYQsAAAAvsLAFAACAF7zssQ0fk+tu7+UeQfrCCy+o7PYkhXstn3rq\nKTXnbsWBaJ122mkquz21rksuuUTl999/P+01IXrz58+PuoT9Ch/hLCJy4YUXBmP3aM0DHXkZ3jbI\n3fIJ+SV8HYjse2S069133w3Go0aNykhNSF3VqlWD8Y033qjm3DWE21N76aWXJv06xx9/vMovv/yy\nyu7njcJeffVVlR9++OGkXzcXcccWAAAAXmBhCwAAAC+wsAUAAIAXvOyxDfv2229V7tOnj8pjx45V\nuWfPnsXmihUrqrnx48er7B7Diux67LHHVHb35nN7aHO1p7ZMmd/+vskRz6VXrVq1lH+2UaNGKrvX\nVHgf7KOPPlrNHXzwwSq7xx+H/zuL6H0pP/nkEzW3Y8cOlcuV02/dn3322T61Iz+4fZQjRoxI+Ph5\n8+ap3Lt372Cc6Dh5RCP8PuAej+xyj6793e9+p3Lfvn2DcadOndTcKaeconKlSpVUdvt5w3nixIlq\nLvw5pXzEHVsAAAB4gYUtAAAAvMDCFgAAAF7wvsfWNXXqVJWXLFmistunef755wfjBx98UM3VrVtX\n5WHDhqn8448/plwnktOhQ4dg3LhxYzXn9hS9/vrrWamptMJ9te7/hgULFmS7nJznnpnu/jt79tln\nVXbPZ0/E3UPU7bHdtWtXMN66dauaW7RokcovvfSSyuE9skV0z/fPP/+s5lasWKFyhQoVVF68ePE+\ntSN31atXLxhPmTKlRD/73//+V2X3WkFu2blzZzAuKipSczVr1lT5u+++U7kke+X/9NNPKm/cuFHl\n2rVrq7xmzZpg/MYbbyT9OvmAO7YAAADwAgtbAAAAeIGFLQAAALxQcD22roULF6p85ZVXqtyxY8dg\n7O55e91116lcv359ldu2bZuOEpFAuNfQ3Td09erVKr/yyitZqelAypcvr/LQoUOLfeysWbNUvuOO\nOzJRUl5zz19ftmyZyi1atEj5uZcvX67ytGnTVP7qq6+C8ccff5zy67j69eunstuL5/ZZIr8MHjw4\nGJd0r+oD7XOL3LJ+/fpg7O5Z/Oabb6rs7rnt7sM/ffr0YDxu3Dg198svv6g8adIkld0eW3feJ9yx\nBQAAgBdY2AIAAMALBd+K4Ar/2kBEZMKECcF4zJgxas491rJVq1Yqn3POOcH4vffeS0+BSJp7DGlU\nRx67rQdDhgxRedCgQSqHt3YaOXKkmtu8eXOaq/PPQw89FHUJpRbeZnB/SrpFFKLlbkXYrl27pH82\n/OtnEZGvv/46LTUh+9yjst0Wo9Jw1x+tW7dW2W158bmdiTu2AAAA8AILWwAAAHiBhS0AAAC8UPA9\ntu6RmX/84x9VbtasWTB2e2pd7hGac+bMKWV1KI0oj9AN99S5PbRdunRR2e2hu+KKKzJXGLzgHg2O\n3DZjxgyVDz/88GIf624b16dPn0yUBM+4x2y7PbXu8bxs9wUAAADkOBa2AAAA8AILWwAAAHjB+x7b\nE044QeWbbrpJ5csvv1zlI444Iunn3r17t8ruPqklPSoRJWeM2e9YZN/jC2+++eaM1XHrrbeqfPfd\ndwfjKlWqqLmXX35Z5V69emWsLgDRq169usqJ/mx4+umnVWbvaiTjnXfeibqEnMEdWwAAAHiBhS0A\nAAC8wMIWAAAAXvCix9bti+3WrVswdntq69Wrl/LrfPrppyoPGzZM5Sj3TS1U4b353H363OviiSee\nUPmll15See3atcG4efPmaq5nz54qN2rUSOWjjz5a5eXLlwdjt/fJ7aEDDsTtH2/QoIHK7t6niNbY\nsWNVLlMm+XtIH374YbrLQQG44IILoi4hZ3DHFgAAAF5gYQsAAAAv5EUrQq1atVQ+6aSTVH7yySdV\nPvHEE1N+rU8++UTlRx55JBi7R5+ynVduK1u2rMo33nijyu7RtRs3bgzG9evXL9Frub8+nD17djC+\n5557SvRcgMttsynJr7aReeEjtEVE2rRpo7L7Z8XOnTuD8VNPPaXmfv755zRXh0Jw7LHHRl1CzuDd\nEQAAAF5gYQsAAAAvsLAFAACAF3Kmx7ZatWrB+LnnnlNzbv9SaXpJ3F7IkSNHquxuzbRt27aUXwuZ\n99FHHwXj+fPnq7lmzZol/Fl3OzC3lzssvBWYiMikSZNUzuRxvYDrrLPOUnncuHHRFAIREalatarK\nBzqa/ccffwzGAwcOzEhNKCxz585V2e3DL6TPBHHHFgAAAF5gYQsAAAAvsLAFAACAF7LWY3vmmWeq\nPGjQIJXPOOOMYHzUUUeV6rW2bt0ajN1jVB988EGVt2zZUqrXQrRWrFgRjC+//HI1d91116k8ZMiQ\npJ931KhRKj/zzDMqL126NOnnAkrLPVIXAMIWLlyo8pIlS1R2P5t03HHHBeOioqLMFRYB7tgCAADA\nCyxsAQAA4AUWtgAAAPBC1npsL7vssoQ5kUWLFqn85ptvqrxr1y6Vw3vTrl+/PunXQX5buXKlykOH\nDk2YgVz19ttvq9y5c+eIKkEyFi9erLK7X3rLli2zWQ6wz+eJxowZo/KwYcOCcf/+/dWcu+bKN9yx\nBQAAgBdY2AIAAMALLGwBAADgBWOtTf7BxiT/YETpM2tt06iL2IvrJj9Ya3Nms1SumbzBew1SwXWT\nYZUrV1Z58uTJKrdp0yYYv/baa2qub9++KufQfv9JXTfcsQUAAIAXWNgCAADAC7Qi+Ilf86DEaEVA\nCnivQSq4brLMbU0Ib/d1ww03qLlTTz1V5Rza/otWBAAAABQOFrYAAADwAgtbAAAAeIEeWz/Rv4QS\no8cWKeC9BqngukEq6LEFAABA4WBhCwAAAC+wsAUAAIAXypXw8WtEZFkmCkFa1Y26AAfXTe7jmkEq\nuG6QCq4bpCKp66ZEHx4DAAAAchWtCAAAAPACC1sAAAB4gYUtAAAAvMDCFgAAAF5gYQsAAAAvsLAF\nAACAF1jYAgAAwAssbAEAAOAFFrYAAADwAgtbAAAAeIGFLQAAALxQUAtbY0x1Y8w1xpipxpilxpht\nxpgNxph5xpirjTEF9e8DyTHG/NEYM9oYM9cYs9EYY40xE6OuC7nPGPOQMeZdY8wP8febX4wx/zHG\n3GuMqR51fcg9vN8gXYwxPeLXjzXGXBN1PdlSLuoCsqyziDwjIitFZLaILBeRWiJyuYiMEZGLjDGd\nrbU2uhKRg4aISCMR2SwiK0TkxGjLQR65VUQ+F5GZIrJaRCqKSHMRGSoi/Ywxza21P0RXHnIQ7zco\nNWPM70XkSYldR5UiLierCm1h+42IdBKRf1pr9+z9pjHmThH5vyJyhcQWuVOiKQ856laJ/QGzVERa\nS+wvRUAyKltrt7vfNMYME5E7ReQOEbkx61Uhl/F+g1IxxhgRGSsia0XkNREZGG1F2VVQv3q31s6y\n1r4RXtTGv79KRJ6Nx3OyXhhymrV2trV2CXfyUVL7W9TGTY5/rZ+tWpAfeL9BGgwQkfNEpK+IbIm4\nlqwrqIXtAfwa/7or0ioAFIKO8a//L9IqAHjFGNNQREaIyChr7Zyo64lCobUi7JcxppyI9IrHf0VZ\nCwD/GGMGSqzPrYqINBWRlhJb1I6Isi4A/oivZSZI7PNDd0ZcTmRY2MaMEJFTROQta+07URcDwDsD\nJfZB1b3+JSJ9rLVFEdUDwD/3iMhpItLSWrst6mKiUvCtCMaYASJym4gsFpGeEZcDwEPW2iOstUZE\njpDYB1SPFZH/GGOaRFsZAB8YY86U2F3akdbaj6KuJ0oFvbA1xtwkIqNEZJGInGut/SXikgB4zFr7\ns7V2qoi0E5HqIjI+4pIA5Ll4C8J4ie38dHfE5USuYBe2xphbRGS0iCyU2KJ2VcQlASgQ1tplEvsL\n9cnGmBpR1wMgr1USkQYi0lBEtocOZbAicm/8MS/Ev/d4ZFVmSUH22BpjBkusr3aBiLS11q6JuCQA\nhefI+NfdkVYBIN/tEJEXi5lrIrG+23ki8rWIeN+mUHALW2PM3SLyFxH5TETa0X4AIBOMMQ1E5Gdr\n7Qbn+2VE5H4R+Z2IfGitXRdFfQD8EP+g2H6PzDXGDJXYwvZv1tox2awrKgW1sDXG9JbYona3iMwV\nkQGxAzqU762147JcGnKYMeZSEbk0Ho+Ifz3LGDMuPl5jrS2ok12QlPYiMtwYM09EvpPYKUC1JHaa\n1LEiskpEro2uPOQi3m+A0imoha2IHBP/WlZEbinmMe+LyLisVIN80VhEejvfOzb+j4jIMimwIwuR\nlH+LyPES27P2NBGpKrFTgL6R2F6TT/AbI+wH7zdAKRhO7QMAAIAPCnZXBAAAAPiFhS0AAAC8wMIW\nAAAAXmBhCwAAAC+UaFeE+CkWyH1rrLU1oy5iL66b/GCt3Wfvu6hwzeQN3muQCq4bpCKp64Y7tn5a\nFnUBAAoC7zVIBdcNUpHUdcPCFgAAAF5gYQsAAAAvsLAFAACAF1jYAgAAwAssbAEAAOAFFrYAAADw\nAgtbAAAAeIGFLQAAALzAwhYAAABeYGELAAAAL7CwBQAAgBdY2AIAAMALLGwBAADghXJRFwDkolGj\nRqk8YMAAlRcuXKhyhw4dVF62bFlmCgMAIEe9++67KhtjVD7vvPMyXgN3bAEAAOAFFrYAAADwAgtb\nAAAAeIEeW8dhhx2mcqVKlYLxxRdfrOZq1qyp8mOPPabyjh070lwdMqlevXrBuEePHmpuz549Kjds\n2FDlE088UWV6bAtDgwYNgvFBBx2k5lq1aqXy008/rbJ7TZXG9OnTg3HXrl3V3M6dO9P2Okg/97pp\n0aKFyg8++KDKZ599dsZrApL117/+VWX3+h0/fnw2yxER7tgCAADAEyxsAQAA4AUWtgAAAPBCwfXY\nhvsoRUQGDx6s8llnnaXyKaeckvRz165dW2V371PktqKiomA8Z84cNdepU6dsl4MccPLJJ6vcp08f\nlTt37hyMy5TR9wmOPPJIld2eWmttGiqMCV+fzz77rJq75ZZbVN64cWPaXhelV6VKFZVnz56t8qpV\nq1Q+4ogjEs4DmTZixIhgfP3116u5X3/9VWV3X9ts4I4tAAAAvMDCFgAAAF7wshUhvPWS+2u47t27\nq1yhQgWV3ePffvjhh2C8adMmNedu+XTllVeqHN7eZ/HixQcqGxHbsmVLMGa7LoiIDB8+XOX27dtH\nVEnyevXqpfKLL76o8gcffJDNclBKbusBrQiIWvPmzYOxu13dvHnzVJ48eXJWagrjji0AAAC8wMIW\nAAAAXmBhCwAAAC/kZY+tuz3KQw89pHKXLl2CsXtE7oEsWbJE5QsuuCAYu70kbt9sjRo1EmbktqpV\nqwbjRo0aRVgJcsXMmTNVTtRju3r1apXd3lZ3O7BER+q6x1K2bt06YZ3wl/u5D0Bk3yO777rrLpW7\ndeum8i+//JLya7nPFd4G9dtvv1VzAwcOTPl10oU7tgAAAPACC1sAAAB4gYUtAAAAvJCXPbaXXXaZ\nytdcc03Kz+X2h7Rt21bl8D62xx9/fMqvg9x36KGHBuM6deqU6GebNWumcrj/mj1x89czzzyj8rRp\n04p9rHuUZGn2F61cubLKCxcuVNk9rjfMrfHTTz9NuQ5Ezz16+ZBDDomoEuSS559/XuX69eurfNJJ\nJ6ns7i9bEnfeeafK1atXD8bXXnutmvviiy9Sfp104Y4tAAAAvMDCFgAAAF5gYQsAAAAv5GWPbefO\nnZN+7Pfff6/y/PnzVR48eLDK4Z5aV8OGDZN+XeSfn376KRiPGzdOzQ0dOjThz7rz69evD8ZPPvlk\naUtDRHbt2qVyoveHdArvny0icvjhhyf9sytWrFB5x44daakJuaFp06Yqf/zxxxFVgiht3bpV5XT2\nYjdu3FjlunXrqhzegzsXe765YwsAAAAvsLAFAACAF1jYAgAAwAt52WPr7pvWr18/lWfMmBGMly5d\nqubc89xLolatWin/LPLL/fffr/KBemyB0uratWswdt/jKlSokPTz3HPPPWmrCZnn9nFv2LBB5SpV\nqqh83HHHZbwm5Kbwn0t/+MMf1NxXX32lckn2k61YsaLK7mePwnu8i+i+7ldffTXp18kW7tgCAADA\nCyxsAQAA4IW8bEUIb8skkr1fE5911llZeR3knjJl9N8Bw9udAMno3r27yrfffrvK4SO7DzrooBI9\n94IFC4Kxe7Qvclt4a0ARkblz56rcoUOHbJaDHPL73/9e5XCLktvCctNNN6lcVFSU9Os89thjKrtb\nqrprrrPPPjvp544Cd2wBAADgBRa2AAAA8AILWwAAAHghL3tsS2PAgAEqu9tcJOJur+H68MMPVf7o\no4+SLww5ze2pdY8vhJ/q1auncs+ePVVu06ZN0s/VsmVLlUtyDW3cuFFltz/3rbfeCsbbtm1L+nkB\n5I5TTjlF5alTp6pco0aNYDx69Gg19/7775fotQYOHBiM+/Tpk/Cxw4YNK9FzR407tgAAAPACC1sA\nAAB4gYUtAAAAvOBFj6173NtJJ50UjO+991411759+4TPVZL9St293fr27avy7t27E74WgNzi9ri9\n/vrrKtepUyeb5QTcvU2ff/75SOpA9KpXrx51CUhRuXJ6ydWjRw+VX3zxRZUTrUfcffXvuOMOld29\naatVq6ZyeK9aY4yaGz9+vMrPPfec5BPu2AIAAMALLGwBAADgBRa2AAAA8EJe9Ni656afdtppKk+Z\nMkXl2rVrB2N3T0e3L9bda/a2U1pkAAAHgklEQVTCCy9U2e3fDXP7ZS6//HKVR40aFYx37txZ7PMA\nyE1u75mbS6Ik/fuuDh06qHzRRRep/Pbbb6dcF/JLp06doi4BKeratavKY8aMUdnd29p9j1i6dGkw\nbtq0qZpz8yWXXKLyUUcdpXJ4nVRUVKTmrrrqqn1qzyfcsQUAAIAXWNgCAADACyxsAQAA4IWc7LE9\n+OCDVXb7Xl977bWEP3/fffcF41mzZqm5Dz74QGV3bzf38e6+lmE1a9ZUefjw4SovX748GE+bNk3N\n7dixo9jnRe4paX9kq1atgvGTTz6ZkZqQfgsXLlT5nHPOUdndd/Kdd94Jxtu3by/Va1999dXBuH//\n/qV6LuSv2bNnq+z2VyO/dOnSJRiPHTtWzf36668qr1+/XuU//elPKq9bty4Yjxw5Us21bt1aZbfn\n1v18QLift0aNGmruhx9+UNl9H/z2228ll3HHFgAAAF5gYQsAAAAvGHd7iYQPNib5B5dQeEuvv/zl\nL2pu0KBBCX/W3eqmZ8+ewdi9te+2D7z11lsqN2nSROXwNl0PP/ywmnPbFNztNcL+/e9/q/zQQw+p\nHP4Vw/4sWLAg4bzjM2tt0wM/LDsyed1ki3s8ckn+f3PqqaeqvGjRorTUlG7W2tT3skozH66ZkqpS\npUowXrt2bcLHduzYUeUIt/vivSbNrrjiCpX/8Y9/qOxuYRk+Qn7ZsmWZKyy9Cua6Cbc31q1bV809\n8MADKrutComE/7uL7HvsrXvkbqJWBNff//53lXv16pV0XRmW1HXDHVsAAAB4gYUtAAAAvMDCFgAA\nAF6IbLuvsmXLqnz//fcH44EDB6q5LVu2qHz77berPGnSJJXDfbXulhfu1kvu8bxLlixR+YYbbgjG\n7jYslStXVrlFixYqd+/ePRi7xyDOnDlTEnG32zjmmGMSPh6Z9eyzz6p83XXXJf2z/fr1U/mWW25J\nS03wywUXXBB1CcgBu3btSjjv9kqWL18+k+WglKZPnx6M3a1K3T/nS8LdoivR1qQiIt26dVPZ3dow\nbMWKFSnXlQu4YwsAAAAvsLAFAACAF1jYAgAAwAuR9di6fYfhvtqtW7eqObefccaMGSo3b95c5b59\n+wbjiy66SM1VqFBBZXfPXHcfuUQ9MBs3blT5X//6V7HZ7W9xj8pz3XrrrQnnkV2LFy+OugSkSXjP\n7Hbt2qk590htd8/QdAq/T4mIjBo1KmOvhfwR7skU2fe958QTT1Q53LN/4403Zq4wpCSd/78O73Xd\nuXNnNed+5sc99nby5MlpqyPXcccWAAAAXmBhCwAAAC+wsAUAAIAXTEnOvE/necorV65UuWbNmsF4\nx44das7tMapYsaLKxx9/fNKvO3ToUJWHDx+u8u7du5N+rhxWMOdwR+Wbb75R+bjjjiv2sWXK6L8/\nuter2wsVFWutOfCjsiOd10zLli1Vvuuuu4Jx27Zt1Zy7X3Rp9pmsVq2ayu3bt1d59OjRKh922GHF\nPpfb6+vui+3usZ1FvNdk2OOPP66y25tdq1atYLx9+/as1JQGXDcpuOOOO4JxeO9/EZGioiKVmzVr\npnK+700bl9R1wx1bAAAAeIGFLQAAALwQ2XZfq1atUjnciuAeEdioUaOEz/XWW2+pPGfOnGA8bdo0\nNff999+r7EnrAbLsyy+/VPnYY48t9rF79uzJdDlIwD1GO9HRk//zP/+j8qZNm1J+XbfNoUmTJion\nagN77733VH7mmWdUjrD1ABFzr5udO3dGVAkyrW7duipfc801wdi9Dp5//nmVPWk9SAl3bAEAAOAF\nFrYAAADwAgtbAAAAeCGyHttWrVqpfOmllwZjtxdt9erVKr/00ksqr1u3TmV6jpBpbj9Tx44dI6oE\n6XTDDTdk7bXc97U33ngjGN98881qLo+2cUKGuUenXnLJJcF46tSp2S4HGTRz5kyVwz23EydOVHP3\n3ntvVmrKB9yxBQAAgBdY2AIAAMALLGwBAADghch6bN39ISdMmLDfMZCLFi1apPJXX32lcsOGDbNZ\nDhLo06ePyv379w/GvXv3TtvruEcjb926VeW5c+eq7PZpL1y4MG21wB9XXnmlyu6R8+57D/wxduxY\nlcPH6E6fPj3b5eQN7tgCAADACyxsAQAA4AUWtgAAAPCCSXRe+T4PNib5ByNKn1lrm0ZdxF5cN/nB\nWmuirmGvTF4z5cuXD8Zu/+0DDzyg8uGHH67ytGnTVA7vM+n2vK1atao0ZeYL3msybNKkSSq7/fud\nOnUKxsuWLctKTWnAdYNUJHXdcMcWAAAAXmBhCwAAAC+wsAUAAIAX6LH1E/1LKLFC6bFFWvFeg1Rw\n3SAV9NgCAACgcLCwBQAAgBdY2AIAAMALLGwBAADgBRa2AAAA8AILWwAAAHiBhS0AAAC8wMIWAAAA\nXmBhCwAAAC+wsAUAAIAXypXw8WtEZFkmCkFa1Y26AAfXTe7jmkEquG6QCq4bpCKp68ZYyxHJAAAA\nyH+0IgAAAMALLGwBAADgBRa2AAAA8AILWwAAAHiBhS0AAAC8wMIWAAAAXmBhCwAAAC+wsAUAAIAX\nWNgCAADAC/8frCG0mMU5LWwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "sXQ58BrIx7bH"
      },
      "source": [
        "## Simple Neutral Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "yyC3iLu6x7bI",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class SimpleFeedforward(nn.Module):\n",
        "    def __init__(self, dropout=0.5):\n",
        "        super().__init__()\n",
        "        self.module = nn.Sequential(\n",
        "            nn.Linear(784, 98),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Dropout(dropout),\n",
        "            nn.Linear(98, 10)\n",
        "        )\n",
        "    def forward(self, X):\n",
        "        return self.module(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CdYsykSix7bJ",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_scaled = X / X.max()\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X_scaled, y, test_size=0.25, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "aemaHabqx7bK",
        "colab": {}
      },
      "source": [
        "from skorch import NeuralNet\n",
        "\n",
        "net = NeuralNet(\n",
        "    SimpleFeedforward,\n",
        "    criterion=nn.CrossEntropyLoss,\n",
        "    max_epochs=10,\n",
        "    lr=0.3,\n",
        "    device='cuda', # comment out to run on cpu\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IT1kjOIex7bL",
        "outputId": "054b98a6-71e1-457f-eacd-5f968f21eb3d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "_ = net.fit(X_train, y_train)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  epoch    train_loss    valid_loss     dur\n",
            "-------  ------------  ------------  ------\n",
            "      1        \u001b[36m0.5725\u001b[0m        \u001b[32m0.3651\u001b[0m  0.9327\n",
            "      2        \u001b[36m0.3142\u001b[0m        \u001b[32m0.2735\u001b[0m  0.8396\n",
            "      3        \u001b[36m0.2653\u001b[0m        \u001b[32m0.1859\u001b[0m  0.8433\n",
            "      4        \u001b[36m0.2362\u001b[0m        \u001b[32m0.1615\u001b[0m  0.8500\n",
            "      5        \u001b[36m0.2164\u001b[0m        \u001b[32m0.1434\u001b[0m  0.8448\n",
            "      6        \u001b[36m0.2020\u001b[0m        0.1577  0.8568\n",
            "      7        \u001b[36m0.1958\u001b[0m        \u001b[32m0.1304\u001b[0m  0.8429\n",
            "      8        \u001b[36m0.1850\u001b[0m        0.1321  0.8449\n",
            "      9        \u001b[36m0.1755\u001b[0m        \u001b[32m0.1226\u001b[0m  0.8510\n",
            "     10        \u001b[36m0.1718\u001b[0m        0.1304  0.8497\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "n6tvumQfx7bM",
        "outputId": "1d0a614d-0214-4ce7-809c-7190561ec7be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "net.set_params(max_epochs=5)\n",
        "_ = net.partial_fit(X_train, y_train)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "     11        \u001b[36m0.1648\u001b[0m        \u001b[32m0.1129\u001b[0m  0.8646\n",
            "     12        \u001b[36m0.1597\u001b[0m        \u001b[32m0.1100\u001b[0m  0.8572\n",
            "     13        \u001b[36m0.1521\u001b[0m        0.1139  0.8666\n",
            "     14        \u001b[36m0.1495\u001b[0m        0.1120  0.8500\n",
            "     15        \u001b[36m0.1451\u001b[0m        0.1105  0.8549\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "hekEXe_yx7bO"
      },
      "source": [
        "## History"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kn6Jyd6Cx7bO",
        "outputId": "9ee96198-8966-4b55-bc08-48942ffa8dea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(net.history)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_wc2Fa1Jx7bQ",
        "outputId": "e47788de-642e-4276-f149-98e9226d5e90",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "net.history[-1, 'valid_loss']"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.11050002356937953"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "K6H9Bl2-x7bS",
        "outputId": "f719af39-5c36-4280-adf3-42f8443e55d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "net.history[-2:, 'train_loss']"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.14953009933517092, 0.14514406461942764]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "_Z7jxthpx7bU"
      },
      "source": [
        "## Callbacks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "LEIY-j-Tx7bV"
      },
      "source": [
        "### EpochScoring"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lM-TfcFhx7bV",
        "outputId": "8952a2aa-5aa3-4f45-86ef-a3da1b8fc3de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "y_pred = net.predict(X_test)\n",
        "print(y_pred.shape)\n",
        "print(y_pred[0, :])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(17500, 10)\n",
            "[-8.3347     0.7850033  6.4455976  6.909385  -9.811217   5.438596\n",
            " -9.500775  -2.8927622 15.760244  -4.2489367]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "stNbGk22x7bX",
        "outputId": "1f67b5be-5eff-4824-ae43-45a30913b6ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def accuracy_argmax(y_true, y_pred):\n",
        "    return np.mean(y_true == np.argmax(y_pred, -1))\n",
        "\n",
        "print('test accuracy:', accuracy_argmax(y_test, y_pred))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test accuracy: 0.9653142857142857\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JA_SLcSmx7bY",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import make_scorer\n",
        "from skorch.callbacks import EpochScoring\n",
        "\n",
        "accuracy_argmax_scorer = make_scorer(accuracy_argmax)\n",
        "\n",
        "epoch_acc = EpochScoring(\n",
        "    accuracy_argmax_scorer,\n",
        "    name='valid_acc',\n",
        "    lower_is_better=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SvSl8PiHx7bZ",
        "colab": {}
      },
      "source": [
        "net = NeuralNet(\n",
        "    SimpleFeedforward,\n",
        "    criterion=nn.CrossEntropyLoss,\n",
        "    max_epochs=10,\n",
        "    lr=0.3,\n",
        "    device='cuda',\n",
        "    callbacks=[epoch_acc]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "agGbjo8vx7ba",
        "outputId": "30171c10-1c04-44a3-f8f4-2e5c3c3f3fcb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "_ = net.fit(X_train, y_train)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  epoch    train_loss    valid_acc    valid_loss     dur\n",
            "-------  ------------  -----------  ------------  ------\n",
            "      1        \u001b[36m0.5739\u001b[0m       \u001b[32m0.9031\u001b[0m        \u001b[35m0.3159\u001b[0m  0.8901\n",
            "      2        \u001b[36m0.3232\u001b[0m       \u001b[32m0.9273\u001b[0m        \u001b[35m0.2321\u001b[0m  0.8563\n",
            "      3        \u001b[36m0.2691\u001b[0m       0.9260        \u001b[35m0.2317\u001b[0m  0.8619\n",
            "      4        \u001b[36m0.2414\u001b[0m       \u001b[32m0.9484\u001b[0m        \u001b[35m0.1716\u001b[0m  0.8594\n",
            "      5        \u001b[36m0.2192\u001b[0m       \u001b[32m0.9562\u001b[0m        \u001b[35m0.1459\u001b[0m  0.8606\n",
            "      6        \u001b[36m0.2056\u001b[0m       0.9534        0.1505  0.8599\n",
            "      7        \u001b[36m0.1948\u001b[0m       \u001b[32m0.9588\u001b[0m        \u001b[35m0.1361\u001b[0m  0.8599\n",
            "      8        \u001b[36m0.1862\u001b[0m       0.9570        0.1389  0.8536\n",
            "      9        \u001b[36m0.1770\u001b[0m       \u001b[32m0.9615\u001b[0m        \u001b[35m0.1261\u001b[0m  0.8498\n",
            "     10        \u001b[36m0.1761\u001b[0m       \u001b[32m0.9630\u001b[0m        \u001b[35m0.1229\u001b[0m  0.8587\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "h2aItvhnx7bb"
      },
      "source": [
        "## Integration with sklearn"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0GF8gij-x7bc",
        "colab": {}
      },
      "source": [
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.preprocessing import MinMaxScaler"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bmQbp65Hx7bc",
        "colab": {}
      },
      "source": [
        "pipe = Pipeline([\n",
        "    ('min_max', MinMaxScaler()),\n",
        "    ('net', net)\n",
        "])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qlLFKQnPx7bd",
        "outputId": "76becf80-ae4c-42fc-db61-b0ec85ea6f78",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.25, random_state=42)\n",
        "_ = pipe.fit(X_train, y_train)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Re-initializing module.\n",
            "Re-initializing optimizer.\n",
            "  epoch    train_loss    valid_acc    valid_loss     dur\n",
            "-------  ------------  -----------  ------------  ------\n",
            "      1        \u001b[36m0.5760\u001b[0m       \u001b[32m0.9024\u001b[0m        \u001b[35m0.3209\u001b[0m  0.8915\n",
            "      2        \u001b[36m0.3174\u001b[0m       \u001b[32m0.9290\u001b[0m        \u001b[35m0.2390\u001b[0m  0.8827\n",
            "      3        \u001b[36m0.2667\u001b[0m       \u001b[32m0.9440\u001b[0m        \u001b[35m0.1848\u001b[0m  0.8939\n",
            "      4        \u001b[36m0.2375\u001b[0m       \u001b[32m0.9451\u001b[0m        \u001b[35m0.1787\u001b[0m  0.8833\n",
            "      5        \u001b[36m0.2195\u001b[0m       \u001b[32m0.9532\u001b[0m        \u001b[35m0.1536\u001b[0m  0.8874\n",
            "      6        \u001b[36m0.2052\u001b[0m       0.9508        0.1560  0.8902\n",
            "      7        \u001b[36m0.1967\u001b[0m       \u001b[32m0.9550\u001b[0m        \u001b[35m0.1448\u001b[0m  0.8973\n",
            "      8        \u001b[36m0.1833\u001b[0m       \u001b[32m0.9626\u001b[0m        \u001b[35m0.1245\u001b[0m  0.8824\n",
            "      9        \u001b[36m0.1763\u001b[0m       0.9610        0.1295  0.8813\n",
            "     10        \u001b[36m0.1698\u001b[0m       \u001b[32m0.9641\u001b[0m        \u001b[35m0.1190\u001b[0m  0.8838\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bV1zyKX6x7be",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "\n",
        "pipe.set_params(net__verbose=0)\n",
        "param_grid = {\n",
        "    'net__module__dropout': [0.2, 0.5, 0.8]\n",
        "}\n",
        "\n",
        "gs = GridSearchCV(pipe, param_grid, \n",
        "                  scoring=accuracy_argmax_scorer, \n",
        "                  cv=3, verbose=3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "scpe5_-tx7bf",
        "outputId": "9d0de8dd-b6c2-4c90-9365-21d203a91cd3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        }
      },
      "source": [
        "_ = gs.fit(X, y)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n",
            "[CV] net__module__dropout=0.2 ........................................\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[CV] ............ net__module__dropout=0.2, score=0.966, total=   9.9s\n",
            "[CV] net__module__dropout=0.2 ........................................\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    9.9s remaining:    0.0s\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[CV] ............ net__module__dropout=0.2, score=0.963, total=   9.6s\n",
            "[CV] net__module__dropout=0.2 ........................................\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   19.6s remaining:    0.0s\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[CV] ............ net__module__dropout=0.2, score=0.968, total=   9.6s\n",
            "[CV] net__module__dropout=0.5 ........................................\n",
            "[CV] ............ net__module__dropout=0.5, score=0.958, total=   9.6s\n",
            "[CV] net__module__dropout=0.5 ........................................\n",
            "[CV] ............ net__module__dropout=0.5, score=0.958, total=   9.7s\n",
            "[CV] net__module__dropout=0.5 ........................................\n",
            "[CV] ............ net__module__dropout=0.5, score=0.964, total=   9.7s\n",
            "[CV] net__module__dropout=0.8 ........................................\n",
            "[CV] ............ net__module__dropout=0.8, score=0.937, total=   9.5s\n",
            "[CV] net__module__dropout=0.8 ........................................\n",
            "[CV] ............ net__module__dropout=0.8, score=0.932, total=   9.5s\n",
            "[CV] net__module__dropout=0.8 ........................................\n",
            "[CV] ............ net__module__dropout=0.8, score=0.944, total=   9.5s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:  1.4min finished\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Gp8MJQWdx7bh",
        "outputId": "50b38492-d6c3-46f3-d055-74f52d4522f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(\"best score:\", gs.best_score_)\n",
        "print(\"best_params\", gs.best_params_)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "best score: 0.9658857142857142\n",
            "best_params {'net__module__dropout': 0.2}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IsWQgHQHhpEi",
        "colab_type": "text"
      },
      "source": [
        "## Conform to scikit-learn Classifer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XY_xubxih6WK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.base import ClassifierMixin\n",
        "from sklearn.utils.extmath import softmax\n",
        "from skorch import NeuralNet\n",
        "\n",
        "class MyNNClassifer(ClassifierMixin, NeuralNet):\n",
        "    def fit(self, X, y, **fit_params):\n",
        "        super().fit(X, y, **fit_params)\n",
        "        self.classes_ = np.unique(y)\n",
        "  \n",
        "    def predict_proba(self, X):\n",
        "        y_pred = super().predict_proba(X)\n",
        "        return softmax(y_pred)\n",
        "      \n",
        "    def predict(self, X):\n",
        "        y_pred = super().predict(X)\n",
        "        return np.argmax(y_pred, axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iq-Pwx7upcIT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from skorch.callbacks import EpochScoring\n",
        "epoch_acc = EpochScoring(\n",
        "    'accuracy',\n",
        "    name='valid_acc',\n",
        "    lower_is_better=False)\n",
        "\n",
        "net = MyNNClassifer(\n",
        "    SimpleFeedforward,\n",
        "    criterion=nn.CrossEntropyLoss,\n",
        "    max_epochs=10,\n",
        "    lr=0.3,\n",
        "    device='cuda', # comment out to run on cpu\n",
        "    callbacks=[epoch_acc]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N9lsD3gXpr4u",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "d636e879-53ef-48a0-8550-3310870b9af6"
      },
      "source": [
        "_ = net.fit(X_train, y_train)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  epoch    train_loss    valid_acc    valid_loss     dur\n",
            "-------  ------------  -----------  ------------  ------\n",
            "      1        \u001b[36m0.5763\u001b[0m       \u001b[32m0.8983\u001b[0m        \u001b[35m0.3319\u001b[0m  1.1257\n",
            "      2        \u001b[36m0.3164\u001b[0m       \u001b[32m0.9327\u001b[0m        \u001b[35m0.2233\u001b[0m  1.0862\n",
            "      3        \u001b[36m0.2666\u001b[0m       0.9319        \u001b[35m0.2189\u001b[0m  1.0448\n",
            "      4        \u001b[36m0.2388\u001b[0m       \u001b[32m0.9508\u001b[0m        \u001b[35m0.1626\u001b[0m  1.0473\n",
            "      5        \u001b[36m0.2149\u001b[0m       0.9431        0.1800  1.0277\n",
            "      6        \u001b[36m0.2037\u001b[0m       \u001b[32m0.9561\u001b[0m        \u001b[35m0.1465\u001b[0m  1.0530\n",
            "      7        \u001b[36m0.1917\u001b[0m       \u001b[32m0.9591\u001b[0m        \u001b[35m0.1343\u001b[0m  1.0409\n",
            "      8        \u001b[36m0.1815\u001b[0m       \u001b[32m0.9613\u001b[0m        \u001b[35m0.1277\u001b[0m  1.0335\n",
            "      9        \u001b[36m0.1763\u001b[0m       \u001b[32m0.9631\u001b[0m        \u001b[35m0.1221\u001b[0m  1.0309\n",
            "     10        \u001b[36m0.1709\u001b[0m       0.9621        0.1273  1.0402\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdG5gLSzrKWN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "431abdfd-8879-4fdb-b91d-1984e893ca4c"
      },
      "source": [
        "net.classes_"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CGFoPsttp-xq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_test_pred = net.predict(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zHmFma11qCWA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4d441188-0ccd-49e4-bdd3-d4404d2dfae2"
      },
      "source": [
        "y_test_pred[0: 2]"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([8, 4])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1vVuHrnqIHi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_test_pred_proba = net.predict_proba(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_I8qwga9qVbN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "610274da-3abb-4ecb-8f75-4fe211b895a4"
      },
      "source": [
        "y_test_pred_proba[0:2, :]"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[2.71880793e-07, 3.25494000e-07, 1.12523943e-04, 3.65644810e-03,\n",
              "        1.11876930e-09, 2.35797212e-04, 1.20907435e-08, 7.18731812e-08,\n",
              "        9.95992780e-01, 1.83940176e-06],\n",
              "       [9.76298122e-12, 2.07509760e-08, 8.03121181e-11, 5.19260990e-09,\n",
              "        9.99871612e-01, 2.94302449e-09, 2.53785742e-10, 1.11806148e-04,\n",
              "        6.08241280e-09, 1.65573420e-05]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    }
  ]
}