notebooks/Highest Quality Examples.ipynb

{
 "metadata": {
  "name": "Highest Quality Examples"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from itertools import islice\n",
      "import logging\n",
      "import json\n",
      "\n",
      "from bubbly.model import Model\n",
      "from bubbly.dr1 import LocationGenerator, highest_quality_on_params\n",
      "from bubbly.extractors import RingWaveletCompressionExtractor, enhance_contrast\n",
      "from bubbly.util import summary\n",
      "\n",
      "from bubbly.util import rfp_curve\n",
      "import brewer2mpl\n",
      "\n",
      "\n",
      "logging.getLogger('bubbly').addHandler(logging.StreamHandler())\n",
      "\n",
      "%pylab"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Welcome to pylab, a matplotlib-based Python environment [backend: module://IPython.kernel.zmq.pylab.backend_inline].\n",
        "For more information, type 'help(pylab)'.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "/Users/beaumont/Library/Python/2.7/lib/python/site-packages/scikits/__init__.py:1: UserWarning: Module argparse was already imported from /opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/argparse.pyc, but /Users/beaumont/Library/Python/2.7/lib/python/site-packages is being added to sys.path\n",
        "  __import__('pkg_resources').declare_namespace(__name__)\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Instantiate Model"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ex = RingWaveletCompressionExtractor()\n",
      "\n",
      "l = LocationGenerator()\n",
      "l.positive_generator = highest_quality_on_params\n",
      "\n",
      "model = Model(ex, l, \n",
      "              weak_learner_params=dict(verbose=1, max_depth=1, n_estimators=200, subsample=.4), \n",
      "              cascade_params=dict(verbose=1, max_layers=1))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### First fit"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model.fit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Locally scanning for 462 false positives\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for false_positives: 0.72 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for _make_xy: 38.11 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Fitting\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for fit: 76.20 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Cascade round 1. False pos rate: 0.000000e+00. Recall: 9.935065e-01\n",
        "WARNING: Could not reduce false positive enough after 1 layers. False positive rate: 0.000000e+00. Recall: 9.935065e-01 [bubbly.cascade]\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Performance on training data"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "x, y = model._make_xy(model.training_data[0]['pos'], model.training_data[0]['neg'])\n",
      "summary(model.estimator, x, y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for _make_xy: 19.61 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "False Positive: 0.000\n",
        "Recall:         0.987\n",
        "AUC:            0.994\n",
        "Accuracy:       0.997\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Performance on test data"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cv_locator = LocationGenerator(1)\n",
      "cv_locator.positive_generator = highest_quality_on_params\n",
      "on2 = cv_locator.positives()\n",
      "\n",
      "off2 = list(islice(cv_locator.negatives_iterator(), 10000))\n",
      "x2, y2 = model._make_xy(on2, off2)\n",
      "summary(model.estimator, x2, y2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "WARNING: RuntimeWarning: invalid value encountered in divide [bubbly.util]\n",
        "WARNING: RuntimeWarning: invalid value encountered in divide [bubbly.util]\n",
        "WARNING"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for _make_xy: 200.90 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ": Non-finite values in feature vectors. Fixing [bubbly.model]\n",
        "False Positive: 0.002"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Recall:         0.764\n",
        "AUC:            0.881"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Accuracy:       0.995\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "colors = brewer2mpl.get_map('Purples', 'sequential', 7).mpl_colors[::-1]\n",
      "\n",
      "for i, y2p in enumerate(model.estimator.staged_decision_function(x2)):\n",
      "    rfp_curve(y2p, y2, label = 'CV %i' % i, color = colors[i])\n",
      "\n",
      "yp = model.estimator.decision_function(x)\n",
      "rfp_curve(yp, y, color='red', label='Training Data')\n",
      "\n",
      "plt.ylim(0, .01)\n",
      "plt.legend(loc='upper left')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "<matplotlib.legend.Legend at 0x112dd3090>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEUCAYAAAAFnmACAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlYlOX+P/D3ICj7JoLGoKADggtgTZKWW6YIGolbkCUp\nGZpLp8U07XyPWXq09ZdZgeWSZR5bvgkpcjR3SyAStMQUCRVGAZMdlxG4f3/4dQxhmOFxmIHh/bou\nrotn5r7v+Tw39rx71pEJIQSIiIiaycLUBRARUdvEACEiIkkYIEREJAkDhIiIJGGAEBGRJAwQIiKS\nhAFCRuft7Q1bW1s4ODiga9eueOqpp1BRUSFprIULF8LNzQ1ubm5YtGhRk20/++wz+Pr6wsHBAWFh\nYbh48aLmvZqaGsybNw/dunVD586dERERgQsXLgAAzp8/DwcHh3o/FhYWeP/99/Wqcfjw4bCxsanX\nPy0tDQBgYWEBe3t7ODg4QC6X46WXXkJdXV2z5qCkpASRkZGwt7eHt7c3tmzZ0mT7999/H926dYOT\nkxNiY2OhVqv1HmvPnj3w9/eHnZ0dHn74YZw/f77e+835e5AZEERG5u3tLfbs2SOEEKKwsFAEBQWJ\nBQsWNHuc+Ph40bt3b6FSqYRKpRJ9+vQR8fHxjbbdt2+fcHd3F9nZ2UKtVovZs2eLYcOGad7/4IMP\nRFBQkCguLhbXrl0T06ZNExMmTGh0rLy8PNGhQwdx7tw5veocPny4WLduXaPvyWQykZubK4QQ4o8/\n/hBdu3bVug7aREVFiaioKFFdXS0OHz4snJycxIkTJxptm5KSIjw8PER2drYoLS0Vw4cPF4sWLdJr\nrEuXLgknJyfx7bffiuvXr4sFCxaIBx54QNO3OX8PMg8MEDK6vweIEEIsWLBAhIeHN3ucQYMGiU8/\n/VSzvH79+nobtL976aWXxJw5czTLFy5cEDKZTPz5559CCCGeffZZ8corr2je3759u+jdu3ejYy1d\nulQ8/PDDetepb4AIIcTkyZPFvHnz9B67qqpKdOzYUeTk5GhemzZtWr1Q+Lvo6GixZMkSzfLevXtF\n165d9RorISFBPPjgg5r3qqurhY2NjTh16pQQonl/DzIPPIRFJiH+7wEIBQUFSElJQUhIiOa9lStX\nwsXFpdEfV1dXTbvs7GwEBQVplgMDA3HixIlGP08mk2k+E4DmMNHvv/8OABg9ejR27tyJixcv4sqV\nK9i8eTPCw8MbrXvTpk2IiYmRtL5NvZednY1Dhw5hwIABAIBx48ZpnYeIiAgAwOnTp2FpaQmFQqEZ\nLygoSOs8NDZnRUVFKC0t1TnWiRMn6vW1tbWFQqHQvN+cvweZB0tTF0DtjxAC48ePh0wmQ1VVFR57\n7DG89tprmvcXLVqk1/HzqqoqODk5aZYdHR1RVVXVaNsxY8YgOjoas2bNgkKhwLJlyyCTyXDlyhUA\nwIQJE5CUlARPT0906NABgYGB+OijjxqMc/jwYRQXF2PSpEnNWt/58+fj5ZdfBgD06tULGRkZmvfv\nvfdedOjQAa6urpg5cyaefvppAMD27dv1mgNHR8d6rzk4OKCyslJr+zvnDAAqKyt1jlVVVQV3d/d6\n7zs6OtZ7X9+/B5kH7oGQ0clkMiQmJqKiogL79+/H3r17621Q9WVvb1/v5Ht5eTns7e0bbTty5Egs\nXboUEydOhI+PD3x8fDQnrgFgwYIFqKysRElJCaqrqxEZGYmwsLAG43z++eeYNGkSbG1t9a5TJpPh\nww8/RGlpKUpLSxusa2ZmJkpKSnDmzBlNsOnrzjkAbs6Dg4ODXu3Ly8sB3AwKbWPdChUHB4cmP6s5\nfw8yDwwQMqmhQ4di3rx5WLhwoea1FStWNLjq6dbP3/8PuW/fvsjKytIsHzt2DP369dP6Wc899xxO\nnz6NwsJCTJgwATU1NZr2KSkpmD59OpydndGxY0fMnTsX6enpKCkp0fS/evUqvv3222YfvpIqLCxM\n6zyMHTsWAODn54eamhqcOXNG06+peWhszjw8PODi4qJ1rL59+2r6Hjt2TPNedXU1cnNz673fnL8H\nmQETnn+hdurOk+iXLl0Stra2IjU1tVnjxMfHi4CAAKFSqURBQYHo06ePSEhIaLTttWvXxG+//Sbq\n6urEuXPnxLBhw+qdTI6OjhYTJ04U5eXlQq1Wi+XLlwu5XF5vjM2bNwsfH58GY+fl5QmZTKb1qqzh\nw4eLzz77rNH37jyJLkVUVJSIjo4W1dXV4tChQ8LJyUlkZ2c32jYlJUV07dpVZGdni5KSEjFs2DDx\n6quv6jXWrauwvvvuO3H16lWxYMECMWjQIE3f5vw9yDwwQMjo7gwQIYSYPXu2iIyMbPZYr7zyinB1\ndRWurq5i4cKF9d7r27ev+Oqrr4QQQpSWlorAwEBhZ2cnunbtKhYvXizq6uo0bQsLC8XkyZOFm5ub\ncHZ2FkOGDBG//PJLvfFCQ0PF//zP/zSo4eDBg8LHx0fU1NQ0WmNTV2FZWFjcdYCUlJSI8ePHCzs7\nO9GjRw+xZcsWzXvnzp0T9vb2Ij8/X/Pae++9Jzw8PISjo6OYMWOGUKvVeo0lhBA//vij8Pf3FzY2\nNmLEiBENQrOpvweZH5kQ/D4QoruxfPlyuLu7Y+bMmaYuhcioWuQcSEpKCvz9/eHr64tVq1Y12mb+\n/Pnw9fVFUFAQMjMzNa/PmDEDHh4e6N+/f732JSUlGDVqFPz8/DB69GiUlZW1ROlEzbZkyRKGB7VL\nBg+Q2tpazJ07FykpKcjOzsaWLVtw8uTJem2Sk5Nx5swZ5OTkYO3atZg9e7bmvenTpyMlJaXBuCtX\nrsSoUaNw+vRpjBw5EitXrjR06URE1AwGD5D09HQoFAp4e3vDysoKUVFRSExMrNcmKSlJcyVLSEgI\nysrKUFhYCAAYMmQIXFxcGoz79z4xMTHYtm2boUsnIqJmMPiNhCqVCl5eXppluVyueXBcU21UKhW6\ndu2qddyioiJ4eHgAADw8PFBUVNSgTXOunyciotuknA43+B6IvhvxO4ttzsZfJpNpbS9uXlnW7n/i\n4uJMXkNr+eFccC5MMRernt2Gfd/+bvL11OdHKoMHiKenJ/Lz8zXL+fn5mrt9tbUpKCiAp6dnk+N6\neHhoDnNdvHixwSMViIjIuAweIEqlEjk5OTh79izUajW2bt2qefDbLREREdi0aRMAIDU1Fc7OzprD\nU9pERETg888/B3DzcRLjx483dOlERNQMBg8QS0tLrFmzBqGhoejTpw8ef/xxBAQEICEhAQkJCQCA\n8PBw9OzZEwqFAnFxcfj44481/aOjozF48GCcPn0aXl5e2LBhA4CbD9jbvXs3/Pz8sHfvXn5ZjQ5K\npdLUJbQanIvbOBe3cS7unlndSHjnI7vbs6ysLAQHB5u6jFaBc3Eb5+K2lp6Lt+ISMXC0AsMn9m2x\nzzAUqdvOdvE4d1dXV5SWlpq6DGphLi4u9R5+SEQtq10ESGlpKfdM2gFexk1kXHycOxERScIAISIi\nSRggREQkCQOEiIgkYYAQEZEkDJBW4quvvoJSqYSDgwPuuecehIeH46effsJ//vMf+Pj4NGhfU1MD\nd3d3JCcnax2vR48esLe3R2RkJC9jJiKDY4C0Au+99x5eeOEFvPbaayguLkZ+fj7mzJmDH374AZGR\nkSgrK8OBAwfq9UlJSUGHDh0wZsyYBuOdOHECs2bNwubNm1FUVARbW1s899xzxlodImon2sV9IK1Z\neXk5/vWvf2Hjxo31nu81duxYjB07FgAwZcoUbNq0CcOGDdO8v2nTJjzxxBOwsGj4/wCbN29GREQE\nHnroIQDAG2+8gYCAAFRXV8POzq6F14iI2gsGCICxXf5tkHF2XHq12X2OHDmCa9euITIyUmubmJgY\nhIWF4aOPPoK1tTXKy8uxfft2pKamNto+OzsbDz74oGa5Z8+e6NSpE06fPo0BAwY0u0YiosYwQCBt\nw28oly9fhpubW6N7ErcMHjwYHh4e+P777xEdHY2vv/4avXv3RmBgYKPtq6qq4OTkVO81R0dHVFZW\nGrR2ImrfeA7ExDp37oy//voLdXV1TbabNm2a5hH4X3zxBaZNm6a1rb29PcrLy+u9Vl5eDgcHh7sv\nmIjo/zBATGzQoEHo1KkTvv/++ybbPfnkk9izZw+OHDmCtLQ0TJ06VWvbvn374tixY5rl3NxcqNVq\n+Pn5GaxuIiIGiIk5OTlh2bJlmDNnDhITE3HlyhXcuHEDO3fuxMKFCzXtvL298dBDDyE6OhqjR49u\n8hsZp06dih9++AGHDx9GdXU1/vnPf2LixIk8gU5EBsUAaQVefPFFvPfee3jzzTfh7u6O7t274+OP\nP25wYj0mJgb5+flNHr4CgD59+iA+Ph5Tp06Fh4cHrl69Wu9Lu4iIDKFdfKEUv2iqfdD2d+aXKN3G\nubjN0HPx3ZpUFBdUaJaP7v0TUxcO4RdKERFR0777KA3j4+6HjX1HAICXb2cMGNbwKRLmhAFCRGQg\no6cGwblL+znXyHMgREQkCQOEiIgkYYAQEZEkDBAiIpKEAUJERJIwQIiISBIGiBkJDw/HF198YfC2\nRESNYYCYmL29PRwcHODg4AALCwvY2tpqlrds2dKssZKTk/HUU08ZvG1z7N+/HxYWFpp18PLywuOP\nP46MjAy9x1i6dGmL1EZEhsUAMbGqqipUVlaisrISPXr0wPbt2zXL0dHRmnY1NTUmrLJ5PD09NeuQ\nmpoKf39/DBkyBHv37jV1aURkQAyQVmr//v2Qy+V466230K1bN8TGxqKsrAzjxo2Du7s7XF1d8eij\nj0KlUmn6DB8+HOvWrQMAbNy4EQ899BAWLFgAV1dX9OzZEykpKZLa5uXlYejQoXB0dMSoUaMwZ84c\nvfcQPD098frrr+OZZ56p93Th559/Ht27d4eTkxOUSiUOHz4M4OZ3vf/73//G1q1b4eDgoPkGxQ0b\nNqBPnz5wdHREr169sHbtWokzS0SGwgABAJnMMD8GVlRUhNLSUpw/fx4JCQmoq6tDbGwszp8/j/Pn\nz8PGxgZz587922rIIPtbHenp6fD398fly5fxyiuvIDY2VlLbJ554Ag888ABKSkqwdOlSfPnll/X6\n6iMyMhJHjx7F1atXAQADBw7EsWPHUFpaiieeeAKTJ0+GWq3GmDFjsHjxYkRFRaGyshKZmZkAAA8P\nD+zYsQMVFRXYsGEDXnjhBc17RGQaDBAAEMIwPwZmYWGB119/HVZWVrC2toarqysiIyNhbW0Ne3t7\nLF68GAcOHNDav0ePHoiNjYVMJsO0adNw8eJFFBcXN6vt+fPnkZGRgWXLlsHS0hIPPvggIiIimv3k\nznvuuQdCCJSVlQG4+Z0lLi4usLCwwIsvvojr16/j1KlTAAAhRIPxw8PD4eNz88F0Q4cOxejRo3Ho\n0KFm1UBEhsUAacW6dOmCjh07apavXLmCuLg4eHt7w8nJCcOGDUN5ebnWjXnXrl01v9va2gK4ec6l\nOW0vXLgAV1dXWFtba9738vJq9rqoVCrIZDI4OzsDAN555x306dMHzs7OcHFxQXl5Of766y+t/Xfu\n3IkHHngAnTt3houLC5KTk3H58uVm10FEhsMAacXuPEz07rvv4vTp00hPT0d5eTkOHDjQ6P+tG1K3\nbt1QUlKiOfQEAOfPn2/2ON9//z3uu+8+2NjY4NChQ3j77bfxzTffoKysDKWlpXByctKsx53rff36\ndUycOBGvvPIKiouLUVpaivDwcH7HC5GJMUDakKqqKtjY2MDJyQklJSV4/fXXW/wze/ToAaVSiaVL\nl+LGjRs4cuQItm/frtc5ECEEVCoVXn/9daxbtw4rVqwAAFRWVsLS0hJubm5Qq9VYtmwZKipufxFP\n165dcfbsWU1AqNVqqNVquLm5wcLCAjt37sSuXbtaZoWJSG8MkFbszo30P/7xD1y9ehVubm4YPHgw\nwsLCtG7I7zxJ3th4+rbdvHkzjhw5gs6dO+Of//wnHn/88XqH1u7sd+HCBc19IAMHDsSJEydw4MAB\nPPLIIwCAMWPGYMyYMfDz84O3tzdsbGzQvXt3zRiTJ08GAHTu3BlKpRIODg5YvXo1pkyZAldXV2zZ\nsgWPPfZYo59PREYkWsDOnTtF7969hUKhECtXrmy0zbx584RCoRCBgYHi6NGjOvumpaWJ+++/XwQH\nBwulUinS09MbjKltdVpoNdutKVOmiKVLl5q6jAa0/Z0zMzONXEnrxbm4zdBzEe3//0RpcZVBxzQW\nqdtIg29Za2pqRK9evUReXp5Qq9UiKChIZGdn12uzY8cOERYWJoQQIjU1VYSEhOjsO2zYMJGSkiKE\nECI5OVkMHz684cowQFrEL7/8Is6cOSNqa2tFcnKysLa2FllZWaYuqwEGiG6ci9sYILdJ3UYa/Ctt\n09PToVAo4O3tDQCIiopCYmIiAgICNG2SkpIQExMDAAgJCUFZWRkKCwuRl5entW+3bt1QXl4OACgr\nK4Onp6ehSyctCgsLMWHCBFy+fBleXl6Ij49HUFCQqcsiIhMzeICoVKp6l3nK5XKkpaXpbKNSqXDh\nwgWtfVeuXImHHnoIL7/8Murq6nDkyJFGP3/WrFma35VKJZRKpUHWqz0bN24cxo0bZ+oy9JKVldXg\ntaqqqkZfb484F7cZei7GvdALZ/L+gKWqg8HGbCkZGRnNej6dNgYPEH3vUBbNvAQzNjYWq1evRmRk\nJL755hvMmDEDu3fvbtAuPj6+WeOSeQkODm7wWlZWVqOvt0eci9sMPRdvRR/AxwdHwLmLncHGbCnB\nwcF45plnNMsJCQmSxjH4VVienp7Iz8/XLOfn50MulzfZpqCgAHK5vMm+6enpiIyMBABMmjQJ6enp\nhi6diIiaweABolQqkZOTg7Nnz0KtVmPr1q2IiIio1yYiIgKbNm0CAKSmpsLZ2RkeHh5N9lUoFJrH\nduzduxd+fn561+Ti4qK5VJU/5vvj4uJioH/FRKQPgx/CsrS0xJo1axAaGora2lrExsYiICBAs4sU\nFxeH8PBwJCcnQ6FQwM7ODhs2bGiyLwCsXbsWc+bMwfXr12FjY9Osp7GWlJQYejVbPR6qIKKWJhPN\nPRnRislkMj7e4v8wQG7jXNzGubjN0HPxRMAH+PjgM23iHMidpG47eSc6ERFJwgAhIiJJGCBERCSJ\nwU+iExG1J7/uycWVKjXU12pMXYrRMUCIiCS6Vq3G0qnfYPDY3hg81g92Tta6O5kRBggRkURCAB2t\nLfHqukhTl2ISPAdCRESSMECIiEgSBggREUnCACEiIkkYIEREJAkDhIiIJGGAEBGRJAwQIiKShAFC\nRESSMECIiEgSBggREUnCACEiIkkYIEREJAkDhIiIJGGAEBGRJAwQIiKShAFCRESS8BsJiYiaqa5O\n4NSvF3Cl4pqpSzEpBggRUTPln/4LSyZtgSKwK0JCfU1djskwQIiImqmuVqCbtzPe+uFJU5diUjwH\nQkREkjBAiIhIEgYIERFJoleAnD17Fj/++CMA4MqVK6ioqGjRooiIqPXTGSBr167F5MmTERcXBwAo\nKChAZGRkixdGREStm84A+eijj3D48GE4OjoCAPz8/FBcXNzihRERUeumM0A6deqETp06aZZramog\nk8latCgiImr9dAbIsGHDsHz5cly5cgW7d+/G5MmT8eijjxqjNiIiasV0BsiqVavQpUsX9O/fHwkJ\nCQgPD8ebb75pjNqIiKgV03kn+rZt2xATE4Nnn33WGPUQEVEboXMPJCkpCb6+vnjqqaewfft21NTU\n6Bw0JSUF/v7+8PX1xapVqxptM3/+fPj6+iIoKAiZmZl69f3www8REBCAfv36YeHChTrrICKilqMz\nQDZu3IgzZ85g0qRJ2LJlC3r27InY2Fit7WtrazF37lykpKQgOzsbW7ZswcmTJ+u1SU5OxpkzZ5CT\nk4O1a9di9uzZOvvu27cPSUlJOH78OH7//Xe8/PLLd7PeRER0l/S6kbBjx44ICwtDVFQU7rvvPmzb\ntk1r2/T0dCgUCnh7e8PKygpRUVFITEys1yYpKQkxMTEAgJCQEJSVlaGwsLDJvp988gleffVVWFlZ\nAQC6dOkiaYWJiMgwdJ4DSU5Oxtdff419+/Zh+PDhmDlzJr755hut7VUqFby8vDTLcrkcaWlpOtuo\nVCpcuHBBa9+cnBwcPHgQixcvhrW1Nd555x0olcoGnz9r1izN70qlstE27UFVVRWysrJMXUarwLm4\njXNx293MxdVqNYY97dlm5zIjIwMZGRl3PY7OANm0aROioqIQHx8Pa2trnQPqe4+IEEKvdrfU1NSg\ntLQUqamp+OWXXzBlyhT8+eefDdrFx8c3a1xzlZWVheDgYFOX0SpwLm7jXNx2N3ORd6IYX248jinT\nwwxclXEEBwfjmWee0SwnJCRIGkdngPznP/9p1oCenp7Iz8/XLOfn50MulzfZpqCgAHK5HDdu3NDa\nVy6XY8KECQCA+++/HxYWFrh8+TI6d+7crPqIiMgwtJ4DefDBBwEA9vb2cHBwqPdz67EmjVEqlcjJ\nycHZs2ehVquxdetWRERE1GsTERGBTZs2AQBSU1Ph7OwMDw+PJvuOHz8ee/fuBQCcPn0aarWa4UFE\nZEJa90B++uknADePEzZrQEtLrFmzBqGhoaitrUVsbCwCAgI0u0hxcXEIDw9HcnIyFAoF7OzssGHD\nhib7AsCMGTMwY8YM9O/fHx07dtQEEBERmYbOQ1hPPfUUvvjiC52v/V1YWBjCwuofG7z1NN9b1qxZ\no3dfALCysmryM4mIyLh0Xsb7+++/11uuqanBr7/+2mIFERFR26A1QFasWAEHBwf89ttv9c5/uLu7\nNzinQUTUHpz8RYXo3v8Przz6JTrZWJm6HJPTeghr8eLFWLx4MRYtWoSVK1casyYiolap7FI1/AZ0\nw0sfR8DalgGiNUD++OMP+Pv7Y/LkyTh69GiD9++9994WLYyIyFg+fHEnSi9V62xXWlSFzvc4wNHV\nxghVtX5aA+Tdd9/Fp59+ipdeeqnRmwP37dvXooURERnLj1t/w4JPImBppfvpTl5+bkaoqG3QGiCf\nfvopAGD//v3GqoWIyGRCQhWw6qTzwlT6G51x+80336CiogIA8MYbb2DChAmNHtIiIqL2RWeALFu2\nDI6Ojjh8+DD27NmDGTNm1HtgIRERtU86A6RDhw4AgO3bt2PmzJkYN24cbty40eKFERFR66YzQDw9\nPfHss89i69atGDt2LK5du4a6ujpj1EZERK2YzgD5+uuvERoail27dsHZ2RmlpaV4++23jVEbERG1\nYjoDxM7ODr169UJKSgrWrFmD4uJijB492hi1ERFRK6YzQD744AM8+eSTuHTpEoqKivDkk09i9erV\nxqiNiIhaMZ0XPX/22WdIS0uDnZ0dAGDRokV44IEHMH/+/BYvjoiIWi+97pqxsLBo9HciorZK1Als\nfusQ1NdrUFfDC4Ok0Bkg06dPR0hICCZMmAAhBLZt24YZM2YYozYiohZTc6MW33+Sjsf/MRizVo6G\nZccOpi6pzdEZIC+++CKGDx+OQ4cOQSaTYePGjRgwYIAxaiMialG29h0x+flBpi6jzdIaIKmpqYiL\ni8OZM2cQGBiIdevWoU+fPsasjYiIWjGtJzTmzJmDd955B5cvX8aLL76IF154wZh1ERFRK6c1QOrq\n6jBq1ChYW1tj8uTJKC4uNmZdRETUymk9hFVeXo7//d//hRCiwbJMJsOECROMViQREbU+WgNk6NCh\n+OGHH7QuM0CIiNo3rQGyceNGI5ZBRERtDb9+i4jalbJL1fhy1SF0v5+bv7vF28qJqF0pyC1B1sGz\nsHeyxvz3w01dTpvGCCaidsfV3Q6uXe0RHNzL1KW0aTr3QKqrq/HGG29g5syZAICcnBxs3769xQsj\nIqLWTWeATJ8+HR07dsTPP/8MALjnnnuwZMmSFi+MiIhaN50Bkpubi4ULF6Jjx44AoHmsOxERtW86\nA6RTp064evWqZjk3NxedOnVq0aKIiKj103kSfenSpRgzZgwKCgrwxBNP4KeffuI9IkTU5mTuz8M7\nc35AjboWvsHdTF2OWdAZIKNHj8a9996L1NRUAMDq1avh5ubW4oURERnS5cIq9HvAC7NWjIKtYyec\nPHXC1CW1eToPYR0+fBjW1tYYN24cSktLsWLFCpw7d84YtRERGVQnG0u4eNijk42VqUsxCzoDZPbs\n2bC1tcWxY8fw3nvvoVevXpg2bZoxaiMiolZMZ4BYWlrCwsIC27Ztw5w5czBnzhxUVlYaozYiortW\ncOYyvv8kHb/u/dPUpZgdnedAHBwcsGLFCnz55Zc4dOgQamtrcePGDWPURkR01/Z/l43M/X/CX+mJ\ne0f0NHU5ZkXnHsjWrVvRqVMnrF+/Hl27doVKpcLLL7/cZJ+UlBT4+/vD19cXq1atarTN/Pnz4evr\ni6CgIGRmZurd991334WFhQVKSkp0lU5EBAC47+GemPnGI7jvYQaIIencA+nWrRteeuklzXL37t0R\nExOjtX1tbS3mzp2LH3/8EZ6enrj//vsRERGBgIAATZvk5GScOXMGOTk5SEtLw+zZs5Gamqqzb35+\nPnbv3o0ePXrczToTEZEBaN0Dsbe3h4ODQ6M/jo6OWgdMT0+HQqGAt7c3rKysEBUVhcTExHptkpKS\nNCEUEhKCsrIyFBYW6uz74osv4q233rrbdSYiIgPQugdSVVUlaUCVSgUvLy/NslwuR1pams42KpUK\nFy5c0No3MTERcrkcgYGBTX7+rFmzNL8rlUoolUpJ69HWVVVVISsry9RltAqci9va41x0CagFgAbr\n3R7n4paMjAxkZGTc9Th6P869uLgY165d0yx379690XYymUyv8W5917o+rl69ihUrVmD37t06+8fH\nx+s9rjnLyspCcHCwqctoFTgXt7XHufjyv4dgYQGERtRf7/Y4F7cEBwfjmWee0SwnJCRIGkfnSfSk\npCT4+vrCx8cHw4YNg7e3N8LCwrS29/T0RH5+vmY5Pz8fcrm8yTYFBQWQy+Va++bm5uLs2bMICgqC\nj48PCgo1OU+BAAAQSUlEQVQKcN9996G4uLhZK0tERIajM0Bee+01HDlyBH5+fsjLy8OePXsQEhKi\ntb1SqUROTg7Onj0LtVqNrVu3IiIiol6biIgIbNq0CQCQmpoKZ2dneHh4aO3br18/FBUVIS8vD3l5\neZDL5Th69Cjc3d3vcvWJiEgqnYewrKys4Obmhrq6OtTW1mLEiBF4/vnntQ9oaYk1a9YgNDQUtbW1\niI2NRUBAgGYXKS4uDuHh4UhOToZCoYCdnR02bNjQZN876XuYjIiIWo7OAHFxcUFlZSWGDBmCqVOn\nwt3dHfb29k32CQsLa3CYKy4urt7ymjVr9O57pz//5B2lRESmpvUQ1vnz5wHcvPrJ1tYW77//PsaM\nGQOFQoEffvjBaAUSEVHrpHUP5LHHHkNmZibs7OwwceJEfPfdd3j66aeNWBoREbVmOk+iAzxkRERE\nDekVIERERHfSegjr+PHjcHBwAHDzRr5bvwM3r4KqqKho+eqIiKjV0hogtbW1xqyDiIjaGB7CIiIi\nSRggREQkCQOEiIgkYYAQEZEkDBAiIpKEAUJEZunalRsoOl+G6vJruhuTJHp/oRQRUVuSsHg3Unee\nhrVdRzy1aKipyzFLDBAiMks31DWY+cYjeHhKP1OXYrZ4CIuIiCRhgBARkSQ8hEVEZmVbwi/44xcV\n/vhVhXtH9DR1OWaNAUJEZuVw0kncO6InHny0NwYM9zF1OWaNAUJEZid4iDf6hMhNXYbZ4zkQIiKS\nhAFCRESSMECIiEgSBggREUnCACEiIkkYIEREJAkv4yUis/DRK//FhT9LcO7kX5BZyExdTrvAPRAi\nMgtHdpzC6KlBeO3zCfAb0M3U5bQL3AMhIrPRf3B3uHrYm7qMdoN7IEREJAkDhIiIJGGAEBGRJAwQ\nIiKShAFCRESSMECIiEgSBggREUnSIgGSkpICf39/+Pr6YtWqVY22mT9/Pnx9fREUFITMzEydfRcs\nWICAgAAEBQVhwoQJKC8vb4nSiYhITwYPkNraWsydOxcpKSnIzs7Gli1bcPLkyXptkpOTcebMGeTk\n5GDt2rWYPXu2zr6jR4/GiRMncOzYMfj5+eHf//63oUsnIqJmMHiApKenQ6FQwNvbG1ZWVoiKikJi\nYmK9NklJSYiJiQEAhISEoKysDIWFhU32HTVqFCwsLDR9CgoKDF06ERE1g8EfZaJSqeDl5aVZlsvl\nSEtL09lGpVLhwoULOvsCwPr16xEdHd3o58+aNUvzu1KphFKplLwubVlVVRWysrJMXUarwLm4zZzn\nYuzzPZF77hTOX+ygV3tzngtdMjIykJGRcdfjGDxAZDL9noIphJA0/vLly9GxY0c88cQTjb4fHx8v\naVxzk5WVheDgYFOX0SpwLm4z57l4Z+pBrN47XO9nYZnzXOgSHByMZ555RrOckJAgaRyDH8Ly9PRE\nfn6+Zjk/Px9yubzJNgUFBZDL5Tr7bty4EcnJydi8ebOhyyaiNioleTemjI3FUdV2xD45BynJu01d\nUrth8ABRKpXIycnB2bNnoVarsXXrVkRERNRrExERgU2bNgEAUlNT4ezsDA8Pjyb7pqSk4O2330Zi\nYiKsra0NXTYRtUEpybuxYsEnqE5XwMdqBNRZ/lix4BOGiJEY/BCWpaUl1qxZg9DQUNTW1iI2NhYB\nAQGaXaS4uDiEh4cjOTkZCoUCdnZ22LBhQ5N9AWDevHlQq9UYNWoUAGDQoEH4+OOPDV0+EbUh6z/6\nDxyK76/3mkPx/djw8VaMCR9loqrajxb5PpCwsDCEhYXVey0uLq7e8po1a/TuCwA5OTmGK5CIzEKt\nuvFzqTXX64xcSfvEO9GJqM3q0LHxi3YsO3HTZgycZSJqs2bMiUKl+y/1Xqvoko7pzz1uooraF36l\nLRG1WbfOc2z4eCt+O3weAQM9Me8fz/H8h5EwQIioTRsTPgpjwkfhyb6rsfqLGfxOdCPiISwiIpKE\nAUJERJIwQIiISBIGCBERScIAISIiSRggREQkCQOEiIgkYYAQEZEkvJGQiFqtnKyL+HXvn3q1vVqt\nbuFq6E4MECJqtfZ98zvOnfoLve+9R2fbKc8PglNnWyNURbcwQIioRVWWXUXe78WS+l66UIn7Rykw\nPu5+3Y3J6BggRNSikjccxc5NWejaw1lS/5593Q1cERkKA4SIWlRtrcAjj/fHk4uGmroUMjBehUVE\nRJIwQIiISBIGCBERScIAISIiSRggREQkCQOEiIgkYYAQEZEkDBAiIpKEAUJELeKGuhalRVW4WsWH\nHJor3olORC1i7ZLd2P9dNjraWGLaq7wL3RwxQIioRVy/VoO45Y/gkehAU5dCLYSHsIiISBIGCBER\nScJDWERmLjutAClfZAIAfIfZYO9nPxjlc0+kFSBwcHejfBaZBgOEyMz99vM5VJZew4OP+qODUzkC\nH/IwyucGPuSNgaMVRvksMg0GCJGB/ZGhwtmTl0xdhsbpzIvw7uOOR6L6IysrC8Gj+5u6JDITDBAi\nA/ti5UF0sLSAWzcHU5cCAHDqbIv7HvYxdRlkhhggRHepuKAc16pvaJavVavx5MKhGDCcG20ybwwQ\nM5WRkYHg4GBTl9EqtPRczBwYj67eLpDJbi5bWMjQuZXsfdyJ/y5u41zcvRa5jDclJQX+/v7w9fXF\nqlWrGm0zf/58+Pr6IigoCJmZmTr7lpSUYNSoUfDz88Po0aNRVlbWEqWbjYyMDFOX0Gq09FzU3KhD\n/E8zEf/Ts4j/6Vl8fGgmuvd2a9HPlIr/Lm5rb3NxcMcOvBYaiqXDh+O10FAc3LHjrsc0eIDU1tZi\n7ty5SElJQXZ2NrZs2YKTJ0/Wa5OcnIwzZ84gJycHa9euxezZs3X2XblyJUaNGoXTp09j5MiRWLly\npaFLJyIySwd37MB/n38eb+7ahaUHDuDNXbvw3+efv+sQMXiApKenQ6FQwNvbG1ZWVoiKikJiYmK9\nNklJSYiJiQEAhISEoKysDIWFhU32/XufmJgYbNu2zdClExGZpV2rV2N5bm6915bn5mL3hx/e1bgG\nPweiUqng5eWlWZbL5UhLS9PZRqVS4cKFC1r7FhUVwcPj5vXrHh4eKCoqavTzZbcORBMSEhJMXUKr\n0dJzYWGxuEXHNyT+u7itPc3F8sZe/O9/8eZdbDMNHiD6bsCFEHq1aWw8mUzW6Ov6jElERIZh8ENY\nnp6eyM/P1yzn5+dDLpc32aagoAByubzR1z09PQHc3OsoLCwEAFy8eBHu7u6GLp2IiJrB4AGiVCqR\nk5ODs2fPQq1WY+vWrYiIiKjXJiIiAps2bQIApKamwtnZGR4eHk32jYiIwOeffw4A+PzzzzF+/HhD\nl05ERM1g8ENYlpaWWLNmDUJDQ1FbW4vY2FgEBARojjXGxcUhPDwcycnJUCgUsLOzw4YNG5rsCwCL\nFi3ClClTsG7dOnh7e+Prr782dOlERNQcog3auXOn6N27t1AoFGLlypWNtpk3b55QKBQiMDBQHD16\n1MgVGo+uufjyyy9FYGCg6N+/vxg8eLA4duyYCao0Dn3+XQghRHp6uujQoYP47rvvjFidcekzF/v2\n7RPBwcGib9++YtiwYcYt0Ih0zcWlS5dEaGioCAoKEn379hUbNmwwfpFGMH36dOHu7i769euntU1z\nt5ttLkBqampEr169RF5enlCr1SIoKEhkZ2fXa7Njxw4RFhYmhBAiNTVVhISEmKLUFqfPXPz888+i\nrKxMCHHzP6T2PBe32o0YMUKMHTtWfPvttyaotOXpMxelpaWiT58+Ij8/XwhxcyNqjvSZi3/9619i\n0aJFQoib8+Dq6ipu3LhhinJb1MGDB8XRo0e1BoiU7Wab+0IpqfeZaLvsty3TZy4GDRoEJycnADfn\noqCgwBSltjh95gIAPvzwQ0yaNAldunQxQZXGoc9cfPXVV5g4caLmAhc3t9Z55/zd0mcuunXrhoqK\nCgBARUUFOnfuDEtL83vK05AhQ+Di4qL1fSnbzTYXINruIdHVxhw3nPrMxd+tW7cO4eHhxijN6PT9\nd5GYmKh58oG53jOkz1zk5OSgpKQEI0aMgFKpxBdffGHsMo1Cn7mYOXMmTpw4gXvuuQdBQUH44IMP\njF1mqyBlu9nmYlbqfSbmuLFozjrt27cP69evx08//dSCFZmOPnPxj3/8AytXroRMJoO4efjWCJUZ\nnz5zcePGDRw9ehR79uzBlStXMGjQIDzwwAPw9fU1QoXGo89crFixAsHBwdi/fz9yc3MxatQoHDt2\nDA4OrfOBmC2pudvNNhcgUu8zuXU/iTnRZy4A4Pjx45g5cyZSUlKa3IVty/SZi19//RVRUVEAgL/+\n+gs7d+6ElZVVg8vM2zp95sLLywtubm6wsbGBjY0Nhg4dimPHjpldgOgzFz///DOWLFkCAOjVqxd8\nfHxw6tQpKJVKo9ZqapK2mwY7Q2MkN27cED179hR5eXni+vXrOk+iHzlyxGxPHOszF+fOnRO9evUS\nR44cMVGVxqHPXPzd008/bbZXYekzFydPnhQjR44UNTU1orq6WvTr10+cOHHCRBW3HH3m4oUXXhBL\nly4VQghRWFgoPD09xeXLl01RbovLy8vT6yS6vtvNNrcHcjf3mZgbfeZi2bJlKC0t1Rz3t7KyQnp6\nuinLbhH6zEV7oc9c+Pv7Y8yYMQgMDISFhQVmzpyJPn36mLhyw9NnLhYvXozp06cjKCgIdXV1eOut\nt+Dq6mriyg0vOjoaBw4cwF9//QUvLy+8/vrruHHj5hehSd1uyoQw0wPBRETUotrcVVhERNQ6MECI\niEgSBggREUnCACEiIkkYIETN1KFDBwwYMACBgYGYMGECqqqqDDq+t7c3SkpKAAD29vYGHZvIkBgg\nRM1ka2uLzMxMHD9+HI6Ojgb/WtS/3/1rjk9QIPPBACG6C4MGDUJubi4AIDc3F2FhYVAqlRg6dChO\nnToFACgqKkJkZCSCg4MRHByM1NRUAEBkZCSUSiX69euHTz/91GTrQCRVm7uRkKi1qK2txa5duzBy\n5EgAwLPPPouEhAQoFAqkpaXhueeew549ezB//nyMGDEC33//Perq6jSHvNavXw8XFxdcvXoVAwcO\nxKRJk8z2UTNknngjIVEzWVpaon///lCpVPD29kZqaiquXLkCd3d39O7dW9NOrVbjxIkTcHd3h0ql\ngpWVVb1xli5dim3btgEAzp49i127dmHgwIHw8fHBr7/+CldXVzg4OKCystKo60ekL+6BEDWTjY0N\nMjMzcfXqVYSGhiIxMRGPPPIInJ2dkZmZ2WifO/8/bf/+/dizZw9SU1NhbW2NESNG4Nq1a8Yon8hg\neA6ESCIbGxusXr0aS5Ysgb29PXx8fPDtt98CuBkYx48fBwCMHDkSn3zyCYCbh70qKipQUVEBFxcX\nWFtb448//tCcFyFqSxggRM309yujgoODoVAo8PXXX2Pz5s1Yt24dgoOD0a9fPyQlJQEAPvjgA+zb\ntw+BgYFQKpU4efIkxowZg5qaGvTp0wevvvoqBg0apPOziFobngMhIiJJuAdCRESSMECIiEgSBggR\nEUnCACEiIkkYIEREJAkDhIiIJPn/xJP1UhnAtsYAAAAASUVORK5CYII=\n",
       "text": [
        "<matplotlib.figure.Figure at 0x115874410>"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Bootstrapping"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "off3 = model.cloud_false_positives(1000, workers=50)\n",
      "on3 = model.training_data[0]['pos']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Cloud scanning for 1000 false positives\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "To re-fetch results, use \n",
        "cloud_false_positives(jobs=range(268580, 268630)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for cloud_false_positives: 879.74 s\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model.add_layer(on3, off3)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for _make_xy: 64.94 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Performance on test data"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "colors = ['c', 'm']\n",
      "yp = model.estimator.decision_function(x)\n",
      "rfp_curve(yp, y, color='red', label='Training Data')\n",
      "\n",
      "for i, y2p in enumerate(model.estimator.staged_decision_function(x2)):\n",
      "    rfp_curve(y2p, y2, label = 'CV %i' % i, color = colors[i])\n",
      "\n",
      "\n",
      "plt.ylim(0, .01)\n",
      "plt.legend(loc='upper left')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "<matplotlib.legend.Legend at 0x113682a90>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEUCAYAAAAFnmACAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVXX+P/DXvez7KqCAooIBLmBdtyzTGhfQIbEs1ClK\nMnQ0m8ytbB6plaPT9s2YRm1MM42x5ZeQIZO5pSYQCVpgCiTKIriwXnbu/fz+sK6iXO7lejfg9Xw8\neAznnM/nc97nk3PenPM553wkQggBIiKiTpKaOgAiIuqamECIiEgnTCBERKQTJhAiItIJEwgREemE\nCYSIiHTCBEJGFxAQAHt7ezg5OcHHxwdPPPEEampqdGprxYoV8PT0hKenJ1auXKlVnbVr10IqleLg\nwYOqdVVVVYiNjYW3tze8vb2xZs2aNnUKCwsxYcIEODg4ICQkBAcOHNA6xvHjx8POzg5OTk6qn/T0\ndACAVCqFo6MjnJyc4OfnhxdffBFKpVLrtgGgoqIC0dHRcHR0REBAABITEzss/+6776J3795wcXFB\nXFwcmpubtW7rwIEDCA4OhoODAx588EFcvHhRte3QoUOYMGECXF1d0b9//04dA3VRgsjIAgICxIED\nB4QQQpSVlYmwsDCxbNmyTrezadMmcdddd4mSkhJRUlIiQkNDxaZNmzqsk5+fL4YOHSp8fX1VMQgh\nxFNPPSUee+wx0dDQIAoLC8XAgQPFtm3bVNtHjx4tXnzxRdHY2Ci+/PJL4erqKq5cuaJVnOPHjxdb\nt25td5tEIhEFBQVCCCF+/fVX4ePjo/EYbhUTEyNiYmJEXV2dOHbsmHBxcRE5OTntlk1NTRXe3t4i\nNzdXVFZWivHjx4uVK1dq1daVK1eEi4uL+OKLL0RTU5NYtmyZGD16tKpuRkaG2Llzp9iyZYsICAjo\n1DFQ18QEQkZ3cwIRQohly5aJyMjITrczZswY8eGHH6qWP/roozYntPZMmTJFpKSk3BaDp6en+PHH\nH1XL69atE/fff78QQoizZ88KGxsbIZfLVdvHjRun9Yle2wQihBAzZ84Uzz33nFbtCiGEXC4X1tbW\nIi8vT7XuySefbJMUbjZr1iyxatUq1fLBgweFj4+PVm1t3rxZjB07VrWtrq5O2NnZibNnz7bZx/79\n+5lAegjewiKTEL9/AKG4uBipqakYNWqUatv69evh5ubW7o+7u7uqXG5uLsLCwlTLw4YNQ05Ojtp9\nfv7557C1tUVERESHMQGAUqnEL7/8AgDIycnBgAED4ODgoNoeFhbW4b46alvdttzcXBw9ehTDhw8H\nAEybNk1tP0RFRQEAzp07B0tLSwQGBmoVW3t9Vl5ejsrKSo1t5eTktKlrb2+PwMBAVT9Rz2Np6gCo\n5xFCYPr06ZBIJJDL5Xj44YfxyiuvqLavXLlSq/EMuVwOFxcX1bKzszPkcnm7ZWtra7Fq1Sp89913\n7W6fMmUKNmzYgO3bt6OsrAwfffQRGhoa2t3PH/sqKSnRGCNw/XgXL16MpUuXAgAGDhyIzMxM1fa7\n774bFhYWcHd3x7x58/DUU08BAPbu3auxbblcDmdn5zbrnJycUFtbq7b8rX0GXO8fTW3J5XJ4eXm1\n2d5Rn1P3xysQMjqJRIKkpCTU1NTg8OHDOHjwYJsTqrYcHR3bDL5XV1fD0dGx3bKrV6/GE088gb59\n+6rW3XxVsHHjRtja2iIoKAjR0dGYPXs2fH19290PcH3Q/daTrToSiQTvv/8+KisrUVlZeduxZmVl\noaKiAvn5+Vi7di0kEolW7aqLrbq6Gk5OTlqVr66uBnA9Uahr64/jdHJy6tS+qPtjAiGTGjduHJ57\n7jmsWLFCtW7dunVtnli6+efmk/bgwYORnZ2tWj516hSGDBnS7n4OHjyIjRs3onfv3ujduzeKiorw\n2GOP4c033wQAuLm5YefOnbh06RJ+/vlnKBQK1W21wYMH47fffmvzl/apU6cwePBgvfbFrSIiItT2\nw9SpUwEAgwYNQmtrK/Lz89vEpq4f2uszb29vuLm5qW3rj+McPHgwTp06pdpWV1eHgoICg/cDmTET\njr9QD3XrAPaVK1eEvb29SEtL61Q7mzZtEiEhIaKkpEQUFxeL0NBQsXnz5nbLXrt2TZSXl4vy8nJR\nVlYm/P39xRdffCHq6uqEEEIUFBSIq1evitbWVpGSkiI8PT1Fbm6uqv7o0aPF0qVLRUNDg+oprKtX\nrwohhDh//ryQSCTiwoUL7e57/Pjx4j//+U+7224dRNdFTEyMmDVrlqirqxNHjx4VLi4ubWK/WWpq\nqvDx8RG5ubmioqJCPPDAA+Kll17Sqq0/nsL68ssvRUNDg1i2bJkYM2aMqq5SqRQNDQ0iJSVF9OvX\nTzQ2NoqmpqY7OjYyb0wgZHS3JhAhhFiwYIGIjo7udFvLly8X7u7uwt3dXaxYsaLNtsGDB4tPP/1U\nqxg+++wz0adPH2Fvby+GDx8uvv322zblCwsLxfjx44WdnZ0IDg5uU/f7778X/fv3F62tre3uq6On\nsKRS6R0nkIqKCjF9+nTh4OAg+vXrJxITE1XbLly4IBwdHUVRUZFq3TvvvCO8vb2Fs7OzmDt3rmhu\nbtaqLSGE+O6770RwcLCws7MTEyZMaJM0Dx06JCQSiZBIJEIqlQqJRCImTJhwR8dG5k0iBOcDIboT\nb7zxBry8vDBv3jxTh0JkVAYZA0lNTUVwcDCCgoKwYcOGdsssXrwYQUFBCAsLQ1ZWlmr93Llz4e3t\njaFDh7YpX1FRgYkTJ2LQoEGYNGkSqqqqDBE6UaetWrWKyYN6JL0nEIVCgUWLFiE1NRW5ublITEzE\nmTNn2pRJSUlBfn4+8vLysGXLFixYsEC17emnn0Zqaupt7a5fvx4TJ07EuXPn8NBDD2H9+vX6Dp2I\niDpB7wkkIyMDgYGBCAgIgJWVFWJiYpCUlNSmTHJyMmJjYwEAo0aNQlVVFcrKygAA999/P9zc3G5r\n9+Y6sbGx2LNnj75DJyKiTtD7i4QlJSXw9/dXLfv5+ak+HNdRmZKSEvj4+Khtt7y8HN7e3gAAb29v\nlJeX31amM8/PExHRDboMh+v9CkTbk/itwXbm5C+RSNSWF9efLOvxP/Hx8SaPwVx+2Bfsizvti4tv\nX0TeC3mdan9WTg52lZWZ/Di1+dGV3hOIr68vioqKVMtFRUXw8/PrsExxcbHqrV91vL29Vbe5Ll26\ndNsnFYiIyLj0nkBkMhny8vJQWFiI5uZm7N69W/Xhtz9ERUVhx44dAIC0tDS4urqqbk+pExUVhY8/\n/hgA8PHHH2P69On6Dp2IiDpB7wnE0tISCQkJmDx5MkJDQ/H4448jJCQEmzdvxubNmwEAkZGRGDBg\nAAIDAxEfH48PPvhAVX/WrFm49957ce7cOfj7+2Pbtm0Arn9gb//+/Rg0aBAOHjyo9eRBPZVMJjN1\nCGaDfXED++IGbfpCfkqO4v8rRtURvjbQnm71IqFEIrmj+3ndSXZ2NsLDw00dhllgX9zAvrhBm77I\nW5SHul/q4BjuCM8ZnnAd56p1+7NzczHNwwOzNdxdMQe6njt7xOfc3d3dUVlZaeowyMDc3NxQUVFh\n6jCom+n1aC/4Lup4jLan6hEJpLKyklcmPQAf4yYyLn7OnYiIdMIEQkREOmECISIinTCBEBGRTphA\nupHIyEh88sknei9LRNQeJhATc3R0VM1zLZVKYW9vr1pOTEzsVFspKSl44okn9F62Mw4fPgypVKo6\nBn9/fzz++OPIzMzUuo3Vq1cbJDYi0i8mEBOTy+Wora1FbW0t+vXrh71796qWZ82apSrX2tpqwig7\nx9fXV3UMaWlpCA4Oxv3334+DBw+aOjQi0iMmEDN1+PBh+Pn54Z///Cd69+6NuLg4VFVVYdq0afDy\n8oK7uzv+/Oc/o6SkRFVn/Pjx2Lp1KwBg+/btuO+++7Bs2TK4u7tjwIABbSbq6kzZ8+fPY9y4cXB2\ndsbEiROxcOFCra8QfH19sWbNGjzzzDNYsWKFav3zzz+Pvn37wsXFBTKZDMeOHQNwfTbLf/zjH9i9\nezecnJwwfPhwAMC2bdsQGhoKZ2dnDBw4EFu2bNGxZ4lIX5hAAEAi0c+PnpWXl6OyshIXL17E5s2b\noVQqERcXh4sXL+LixYuws7PDokWLbjqMtp+5z8jIQHBwMK5du4bly5cjLi5Op7KzZ8/G6NGjUVFR\ngdWrV2Pnzp2dfmkvOjoaJ0+eRENDAwBg5MiROHXqFCorKzF79mzMnDkTzc3NmDJlCl5++WXExMSg\ntrZWNd2xt7c3vvnmG9TU1GDbtm144YUX2kyFTETGxwQCAELo50fPpFIp1qxZAysrK9ja2sLd3R3R\n0dGwtbWFo6MjXn75ZRw5ckRt/X79+iEuLg4SiQRPPvkkLl26hMuXL3eq7MWLF5GZmYm1a9fC0tIS\nY8eORVRUVKff7O/Tpw+EEKq57OfMmQM3NzdIpVIsWbIETU1NOHv2LAC0O0dBZGQk+vfvDwAYN24c\nJk2ahKNHj3YqBiLSLyYQM9arVy9YW1urluvr6xEfH4+AgAC4uLjggQceQHV1tdqT+c0zPNrb2wO4\nPubSmbKlpaVwd3eHra2tavvNs0lqq6SkBBKJBK6u1z9G99ZbbyE0NBSurq5wc3NDdXU1rl69qrb+\nvn37MHr0aHh4eMDNzQ0pKSm4du1ap+MgIv1hAjFjt94mevvtt3Hu3DlkZGSguroaR44cueMZxTTp\n3bs3KioqVLeeAODixYudbuerr77CPffcAzs7Oxw9ehRvvvkmPv/8c1RVVaGyshIuLi6q47j1uJua\nmvDII49g+fLluHz5MiorKxEZGcnvmxGZGBNIFyKXy2FnZwcXFxdUVFRgzZo1Bt9nv379IJPJsHr1\narS0tODEiRPYu3evVmMgQgiUlJRgzZo12Lp1K9atWwcAqK2thaWlJTw9PdHc3Iy1a9eipqZGVc/H\nxweFhYWqBNHc3Izm5mZ4enpCKpVi3759+Pbbbw1zwESkNSYQM3brSfpvf/sbGhoa4OnpiXvvvRcR\nERFqT+TtzRuva9ldu3bhxIkT8PDwwN///nc8/vjjbW6t3VqvtLRU9R7IyJEjkZOTgyNHjuBPf/oT\nAGDKlCmYMmUKBg0ahICAANjZ2aFv376qNmbOnAkA8PDwgEwmg5OTEzZu3IjHHnsM7u7uSExMxMMP\nP9zu/onIeHrEhFKcaEq/Hn/8cYSGhuLVV181dShtqPvvzEmUbmBf3KDthFL2wfZazQfy5sWLuNDU\npFr+X0UF1gQEdOsJpXgFQhplZmaioKAASqUS+/btQ3JyMuekJ7rFm0VF8LW2RrC9PYLt7fE3Pz9M\ndHMzdVgG1SMmlKI7U1ZWhhkzZuDatWvw9/fHpk2bEBYWZuqwiMxOXO/e8FJze7c7YgIhjaZNm4Zp\n06aZOgwiMjNMIEREt1DUKdByrQWttV3nG3SmwARCRHSLM0+cQfXRakjtpOj1SC9Th2O2mECIiG6h\nbFQi+ONgeER6mDoUs8ansIiISCdMIEREpBPewiIi+t2FdRdQ93Md5FlyQP8zNHQ7vAIxE59++qnq\nsx19+vRBZGQkjh8/jv/+97+qz5jfrLW1FV5eXkhJSVHbXr9+/eDo6Ijo6GhUVlYa+hCIurzyHeVw\nuscJgf8XCNdxrqYOx+wxgZiBd955By+88AJeeeUVXL58GUVFRVi4cCG+/vprREdHo6qq6rZ5P1JT\nU2FhYYEpU6bc1l5OTg7mz5+PXbt2oby8HPb29vjrX/9qrMMh6tI8pnnA63EvWDhYmDoUs8dbWCZW\nXV2NV199Fdu3b2/zeZCpU6di6tSpAIDHHnsMO3bswAMPPKDavmPHDsyePRtS6e1/A+zatQtRUVG4\n7777AACvvfYaQkJCUFdXBwcHBwMfERH1FEwgACSHD+ulHTF+fKfrnDhxAo2NjYiOjlZbJjY2FhER\nEfjXv/4FW1tbVFdXY+/evUhLS2u3fG5uLsaOHataHjBgAGxsbHDu3DnVHONERHeKCQS6nfj15dq1\na6p5LtS599574e3tja+++gqzZs3CZ599hrvuugvDhg1rt7xcLoeLi0ubdc7OzqitrdVr7ETUs3EM\nxMQ8PDxw9epVKJXKDss9+eST2LFjBwDgk08+wZNPPqm2rKOjI6qrq9usq66uhpOT050HTET0OyYQ\nExszZgxsbGzw1VdfdVjuL3/5Cw4cOIATJ04gPT0dc+bMUVt28ODBOHXqlGq5oKAAzc3NGDRokN7i\nJiJiAjExFxcXrF27FgsXLkRSUhLq6+vR0tKCffv2YcWKFapyAQEBuO+++zBr1ixMmjQJXl5eatuc\nM2cOvv76axw7dgx1dXX4+9//jkceeYQD6ESkV0wgZmDJkiV455138Prrr8PLywt9+/bFBx98cNvA\nemxsLIqKijq8fQUAoaGh2LRpE+bMmQNvb280NDTggw8+MOQhEFEPZJAEkpqaiuDgYAQFBWHDhg3t\nllm8eDGCgoIQFhaGrKwsjXUzMjIwcuRIDB8+HCNGjMCPP/5oiNBNZvbs2fjxxx8hl8tx6dIlfP31\n1xg9enSbMrGxsVAoFKo5wzsya9YsXLhwAXK5HF999RVcXflSFBHpl94TiEKhwKJFi5Camorc3Fwk\nJibizJkzbcqkpKQgPz8feXl52LJlCxYsWKCx7vLly/Haa68hKysLa9euxfLly/UdOhERdYLeE0hG\nRgYCAwMREBAAKysrxMTEICkpqU2Z5ORkxMbGAgBGjRqFqqoqlJWVdVi3d+/eqieLqqqq4OureZJ7\nIiIyHL2/B1JSUgJ/f3/Vsp+fH9LT0zWWKSkpQWlpqdq669evx3333YelS5dCqVTixIkT7e5//vz5\nqt9lMhlkMplejou6huzs7NvWyeXydtf3ROyLG9rri8YljThTcQYW2Z3/jMmrra0o/OUXlHbwTpe5\nyMzMRGZm5h23o/cEIpFo9wlLIUSn2o2Li8PGjRsRHR2Nzz//HHPnzsX+/ftvK7dp06ZOtUvdS3h4\n+G3rsrOz213fE7EvbmivLzJiMhCyJwT2wfadbm/S8eP4ZcgQeFlb6ytEgwkPD8czzzyjWt68ebNO\n7eg9Vfr6+qKoqEi1XFRUBD8/vw7LFBcXw8/Pr8O6GRkZqqeSHn30UWRkZOg7dCIi6gS9JxCZTIa8\nvDwUFhaiubkZu3fvRlRUVJsyUVFRqreq09LS4OrqCm9v7w7rBgYGqr5Ie/DgQb4UR0RkYnq/hWVp\naYmEhARMnjwZCoUCcXFxCAkJUV0ixcfHIzIyEikpKQgMDISDgwO2bdvWYV0A2LJlCxYuXIimpibY\n2dlhy5Yt+g6diIg6QSI6OxhhxiQSSbtjK+rWU/ei7r8z7/vfwL64od0xkOAMDNkzRKcxEK/jx/HL\niBFdYgzkVrqeI83/cQEiIjJLTCBmQp9T2paVlSEqKgq+vr6QSqW4ePGiMQ6BiHoYJhAzoO8pbaVS\nKSIjI/Hll18a6xCIqAfihFImZogpbb28vDB//ny0trYa/gCIerjUigrUtraiUcOcPt0REwiAw5LD\nemlnvBjf6TqGmNKWiIyjTqHAtJ9/RrSnJ6I9PeFq2bNOqT3raNXQ5cSvL4aY0paIOqe1qhWKegWU\nLZ27ihAAbKVSfD54sGECM3McAzExQ0xpS0Sdc6LfCfx0z0+AArB05d/V2mICMTFDTGlLRJ2jbFRi\ndOFojC4cDWufrvceh6kwgZiYIaa0BYDGxkY0Njbe9jtRT1d/rh7pQemoSatB2oA0pA1IA5Tg2VAH\nvFYzA0uWLIGPjw9ef/11zJkzB05OTpDJZFi1alWbcrGxsZg7dy7efPNNjW3a219/k1YikSA4OBgS\niQQKhcIg8RN1Jc3lzbB0sYRlmCVCvwsFAFjYW0BqxQzSWUwgZmL27NmYPXt2h2ViY2NVE3FpomlM\nhagnk9pJIbGTwG6AnalD6dKYcomISCdMIEREpJMecQvLzc1N65kSqetyc3MzdQhEPUqPSCAVFRWm\nDsHo+NluoraULUo0nG1A4298IlFfekQCISK68vkV5P01DzZ+NnCb6AY55KYOqctjAiGiHkG0CHg8\n7IGQj6/PcpqdnW3iiLo+DqITUbdWurkUR12O4tz8c7BwtDB1ON0Kr0CIqFtrKm2C71990XdlXyYQ\nPWMCIaJuT2orhaULT3f6xltYRESkEyYQIiLSCRMIERHphDcFiYg6SSkE0mtqUN3Dv3DNBEJE1Eln\n6uvxp1OncI+TE6I8PEwdjskwgRARdZJCCAy0s8P3w4ebOhST4hgIERHphAmEiIh0wgRCREQ60SqB\nFBYW4rvvvgMA1NfXo6amxqBBERGR+dOYQLZs2YKZM2ciPj4eAFBcXIzo6GiDB0ZEROZNYwL517/+\nhWPHjsHZ2RkAMGjQIFy+fNnggRERkXnTmEBsbGxgY2OjWm5tbeX0sEREpDmBPPDAA3jjjTdQX1+P\n/fv3Y+bMmfjzn/9sjNiIiMiMaUwgGzZsQK9evTB06FBs3rwZkZGReP31140RGxGRzpTNSjRfboay\nTmnqULotjW+i79mzB7GxsXj22WeNEQ8RkV7kPZeHy4mXIbWVYsD6AaYOp1vSeAWSnJyMoKAgPPHE\nE9i7dy9aW1s1Npqamorg4GAEBQVhw4YN7ZZZvHgxgoKCEBYWhqysLK3qvv/++wgJCcGQIUOwYsUK\njXEQUc+lrFci6F9BGHt5LHrP7W3qcLoljQlk+/btyM/Px6OPPorExEQMGDAAcXFxassrFAosWrQI\nqampyM3NRWJiIs6cOdOmTEpKCvLz85GXl4ctW7ZgwYIFGuseOnQIycnJOH36NH755RcsXbr0To6b\niIjukFYvElpbWyMiIgIxMTG45557sGfPHrVlMzIyEBgYiICAAFhZWSEmJgZJSUltyiQnJyM2NhYA\nMGrUKFRVVaGsrKzDuv/+97/x0ksvwcrKCgDQq1cvnQ6YiIj0Q+MYSEpKCj777DMcOnQI48ePx7x5\n8/D555+rLV9SUgJ/f3/Vsp+fH9LT0zWWKSkpQWlpqdq6eXl5+P777/Hyyy/D1tYWb731FmQy2W37\nnz9/vup3mUzWbpmeQC6XIzs729RhmAX2xQ09oS+aSpugqFGgdWwranvVoiy7rN1yd9IXcoUCzzU0\ndNm+zMzMRGZm5h23ozGB7NixAzExMdi0aRNsbW01NqjtOyJCCK3K/aG1tRWVlZVIS0vDjz/+iMce\newy//fbbbeU2bdrUqXa7q+zsbISHh5s6DLPAvrihJ/TFyYUn0euBXrAfZA+PER6w8rBqt9yd9MVp\nuRzvnzmD0120L8PDw/HMM8+oljdv3qxTOxoTyH//+99ONejr64uioiLVclFREfz8/DosU1xcDD8/\nP7S0tKit6+fnhxkzZgAARowYAalUimvXrsGjB0/mQkTt85jqAZexLqYOo9tTOwYyduxYAICjoyOc\nnJza/PzxWZP2yGQy5OXlobCwEM3Nzdi9ezeioqLalImKisKOHTsAAGlpaXB1dYW3t3eHdadPn46D\nBw8CAM6dO4fm5mYmDyIiE1J7BXL8+HEA1+8TdqpBS0skJCRg8uTJUCgUiIuLQ0hIiOoSKT4+HpGR\nkUhJSUFgYCAcHBywbdu2DusCwNy5czF37lwMHToU1tbWqgRERESmofEW1hNPPIFPPvlE47qbRURE\nICIios26P77m+4eEhASt6wKAlZVVh/skIiLj0vgY7y+//NJmubW1FT/99JPBAiIioq5BbQJZt24d\nnJyc8PPPP7cZ//Dy8rptTIOIqCc4UV0Nz+PHcX9WFuwtLEwdjsmpTSAvv/wyamtrsXTpUtTW1qp+\nKioqsH79emPGSERkFspbWjDSyQm/jR6Nw130EV59UjsG8uuvvyI4OBgzZ87EyZMnb9t+9913GzQw\nIiJjefbsWZQ3N2ssd6m5GX42NvCwav/dkp5GbQJ5++238eGHH+LFF19s9+XAQ4cOGTQwIiJj+bis\nDLtCQ2GlxYvQIfb2Roioa1CbQD788EMAwOHDh40VCxGRyfzZwwM2Uq0+D0i/09hbn3/+OWpqagAA\nr732GmbMmNHuLS0iIupZNCaQtWvXwtnZGceOHcOBAwcwd+7cNh8sJCKinkljArH4/VG1vXv3Yt68\neZg2bRpaWloMHhgREZk3jQnE19cXzz77LHbv3o2pU6eisbERSiXnGCYi6uk0JpDPPvsMkydPxrff\nfgtXV1dUVlbizTffNEZsRERkxjQmEAcHBwwcOBCpqalISEjA5cuXMWnSJGPERkREZkxjAnnvvffw\nl7/8BVeuXEF5eTn+8pe/YOPGjcaIjYiIzJjGr/H+5z//QXp6OhwcHAAAK1euxOjRo7F48WKDB0dE\nROZLYwIBAOlNL9dI+aINEZmh0k2laCxsRGNho1bllUJgdWEhGpVKtHZyim26TmMCefrppzFq1CjM\nmDEDQgjs2bMHc+fONUZsRERaK1xTCJ+nfeD3Nz84DHPQWL5FCLxTVISX+/VDQlAQrLX4jAm1pTGB\nLFmyBOPHj8fRo0chkUiwfft2DB8+3BixERF1iu9zvrDpbaN1eScLC6zs29eAEXVvahNIWloa4uPj\nkZ+fj2HDhmHr1q0IDQ01ZmxERGTG1A5oLFy4EG+99RauXbuGJUuW4IUXXjBmXEREZObUJhClUomJ\nEyfC1tYWM2fOxOXLl40ZFxERmTm1t7Cqq6vx//7f/4P4/emEm5clEglmzJhhtCCJiMj8qE0g48aN\nw9dff612mQmEiKhnU5tAtm/fbsQwiIioq+FbgUTUo1xubsb8c+dQ0NBg6lC6PCYQIupRztbX47vK\nSrhaWuI/d91l6nC6NK0+ZUJE1J30trZGH4kE4R4epg6lS9N4BVJXV4fXXnsN8+bNAwDk5eVh7969\nBg+MiIjMm8YE8vTTT8Pa2ho//PADAKBPnz5YtWqVwQMjItJGU2kTGvIbIFr5QURj05hACgoKsGLF\nClhbWwOA6rPuRESmpmxSIq1fGk5POQ1rH2tYOvGuvDFp7G0bGxs03PS0QkFBAWxstP9YGRGRoQil\ngMRSglFGeZnwAAAXiklEQVT5o0wdSo+kMYGsXr0aU6ZMQXFxMWbPno3jx4/zHREiMqmmkiacHH0S\nymYlpHbaPUy6v6ICT/z6K5qVSsicnADOAXLHNCaQSZMm4e6770ZaWhoAYOPGjfD09DR4YERE6rRW\ntUJqJ8U9P94Dqb12CaS0uRnjXFywMSgILhYWOPvzzwaOsvvT2PPHjh2Dra0tpk2bhsrKSqxbtw4X\nLlwwRmxERGpJrCTXxz2ctR/3sJNK4WNtDTsLCwNG1nNoTCALFiyAvb09Tp06hXfeeQcDBw7Ek08+\naYzYiIjIjGlMIJaWlpBKpdizZw8WLlyIhQsXora21hixERHdsbP19XinqAipFRWmDqXb0Xjt5+Tk\nhHXr1mHnzp04evQoFAoFWlpajBEbEdEd+7S8HN9WVmKMszMmu7ubOpxuReMVyO7du2FjY4OPPvoI\nPj4+KCkpwdKlSzusk5qaiuDgYAQFBWHDhg3tllm8eDGCgoIQFhaGrKwsreu+/fbbkEqlqOBfE0Sk\npSnu7ngnMJAJRM80XoH07t0bL774omq5b9++iI2NVVteoVBg0aJF+O677+Dr64sRI0YgKioKISEh\nqjIpKSnIz89HXl4e0tPTsWDBAqSlpWmsW1RUhP3796Nfv353csxERKQHaq9AHB0d4eTk1O6Ps7Oz\n2gYzMjIQGBiIgIAAWFlZISYmBklJSW3KJCcnq5LQqFGjUFVVhbKyMo11lyxZgn/+8593esxERKQH\naq9A5HK5Tg2WlJTA399ftezn54f09HSNZUpKSlBaWqq2blJSEvz8/DBs2LAO9z9//nzV7zKZDDKZ\nTKfj6Orkcjmys7NNHYZZYF/c0NX6ovFCIxoLG9vdZvGihVbHEt7QAIlEguyqqjbru1pf6FNmZiYy\nMzPvuB2tH6C+fPkyGhtv/Ifs27dvu+UkEolW7YlOvAXa0NCAdevWYf/+/Rrrb9q0Set2u7Ps7GyE\nh4ebOgyzwL64oav1Rf72fNj428B3se9t2yQWEkikms83X50/D6lEgukBAW3Wd7W+0Kfw8HA888wz\nquXNmzfr1I7GBJKcnIwXX3wRpaWl8PLywoULFxASEoKcnJx2y/v6+qKoqEi1XFRUBD8/vw7LFBcX\nw8/PDy0tLe3WLSgoQGFhIcLCwlTl77nnHmRkZMDLy6tzR0xEJlGbWYuCZQWdqtOQ1wD/Ff6QWnHu\nO3OkMYG88sorOHHiBCZOnIisrCwcOnQIn3zyidryMpkMeXl5KCwsRJ8+fbB7924kJia2KRMVFYWE\nhATExMQgLS0Nrq6u8Pb2hoeHR7t1Q0JCUF5erqrfv39//PTTT3DnExVEXUb9r/WQWEjQd1X7dy/U\ncZI5GSgiulMaE4iVlRU8PT2hVCqhUCgwYcIEPP/88+obtLREQkICJk+eDIVCgbi4OISEhKgukeLj\n4xEZGYmUlBQEBgbCwcEB27Zt67DurbS9TUZEplN1uAqV31WqluU/y2HtYw23CW4mjIr0SWMCcXNz\nQ21tLe6//37MmTMHXl5ecHR07LBOREQEIiIi2qyLj49vs5yQkKB13Vv99ttvmsImIhMr216G1ppW\nON19/QrCeYQzXB5wMXFUpE9qE8jFixfRt29fJCUlwdbWFu+++y527dqFmpoavPrqq8aMkYi6KM8o\nT/g85WPqMMhA1CaQhx9+GFlZWXBwcMAjjzyCL7/8Ek899ZQRQyMiInOm1WO8vGVEROooG5VQtihv\nX9/OOupeOIEwEelMtAoccz8GiUX7D7Z4z/E2ckRkTGoTyOnTp+HkdH3wq6GhQfU7cP0pqJqaGsNH\nR0RmTSgFRKvAuPpxpg6FTEBtAlEoFMaMg4iIuhi+3klERDrhGAgRaa2ppAkX1l0Afh8fFwrtv2lH\n3Q8TCBFpTX5ajqrDVfBddOPjhiETb/9aBPUMTCBEpFHz5WZU7q+EPFsO27628F1w+9dxqefhGAgR\naXTliysoXFOIptIm9Hq0l6nDITPBKxAi0kwAbhPdMOhfg0wdidbqFQpcbmlBVWsr3K2sTB1Ot8Qr\nECLqlhbn50P2009IunYNA2xtTR1Ot8QrECLqlpqUSrw7cCCe8OHHHA2FVyBEpFZJQglO+J3A+VXn\nIbXl6YLa4hUIEanVcL4BPk/7oE98H1h5cRyB2mICISKVi+svojarVrUsz5ajT3wf2PjZmDCqzvm/\n4mKcqK5GWk0NJnPaa4NiAiEilfLEcnjP9oZtwPVB514zesF1gquJo+qczy9fxmR3dzzSqxcmMYEY\nFBMIEbXhPsUdjmEdT1tt7h5yc8NYF06fa2gcFSMiIp0wgRARkU6YQIiISCdMIEREpBMOohN1c5f+\ncwl5z+UBAJQblPh+zPdqy4oWAQsHC2OFRl0cEwhRN9dc3gzf53wRsCYAp3NPY1jFMLVlJVIJpDa8\nMUHaYQIh6gEkVhJY2FlAYnH9f7ujv547h7yGBvxSV8d780bCfiaibuGrq1cR17s3vhoyBCOcnU0d\nTo/AKxCibubq11ch/0muWq76vgrOY3rGCfUBFxf0tuk6n13p6ngFQtTNFL9bjIbzDapl13Gu8Jrp\nZcKIqLviFQhRN+QT6wO3B91MHQZ1c7wCISIinTCBEBGRTphAiIhIJ0wgRESkEyYQIiLSiUESSGpq\nKoKDgxEUFIQNGza0W2bx4sUICgpCWFgYsrKyNNZdtmwZQkJCEBYWhhkzZqC6utoQoRMRkZb0nkAU\nCgUWLVqE1NRU5ObmIjExEWfOnGlTJiUlBfn5+cjLy8OWLVuwYMECjXUnTZqEnJwcnDp1CoMGDcI/\n/vEPfYdO1KUVbyxG3uI8NJxr0FyYSA/0nkAyMjIQGBiIgIAAWFlZISYmBklJSW3KJCcnIzY2FgAw\natQoVFVVoaysrMO6EydOhFQqVdUpLi7Wd+hEXVrxO8WwdLNE35f6wmmEk6nDoR5A7y8SlpSUwN/f\nX7Xs5+eH9PR0jWVKSkpQWlqqsS4AfPTRR5g1a1a7+58/f77qd5lMBplMpvOxdGVyuRzZ2dmmDsMs\n9JS+aF7ZjMrwSkhtpbhScKXdMt25L/7e0oKLOTkol2r3d3F37gtNMjMzkZmZecft6D2BSCQSrcoJ\nIXRq/4033oC1tTVmz57d7vZNmzbp1G53k52djfDwcFOHYRZ6Sl+kTU9D6OFQ2AbYqi3Tnfsi4ocf\ncHLwYK2/hdWd+0KT8PBwPPPMM6rlzZs369SO3hOIr68vioqKVMtFRUXw8/PrsExxcTH8/PzQ0tLS\nYd3t27cjJSUFBw4c0HfYRNRFfXPwIDbu2YMKuRyzEhOxLDoaUx980NRh9Qh6TyAymQx5eXkoLCxE\nnz59sHv3biQmJrYpExUVhYSEBMTExCAtLQ2urq7w9vaGh4eH2rqpqal48803ceTIEdjaqv8Li4h6\njm8OHsTziYkomDMHAHAEQPGuXQDAJGIEek8glpaWSEhIwOTJk6FQKBAXF4eQkBDVJVJ8fDwiIyOR\nkpKCwMBAODg4YNu2bR3WBYDnnnsOzc3NmDhxIgBgzJgx+OCDD/QdPhF1IRv37FEljz8UzJmD9/fs\nYQIxAoN8jTciIgIRERFt1sXHx7dZTkhI0LouAOTl5ekvQCLqFprUDJg3GjmOnopvohNRl2WjVLa7\nnje5jYMJhKgLE0Igb1EecufkovlKM6DdQ5DdxuLp0zHw9zGPPwzcuRPPRUWZKKKehRNKEXVlAij5\nVwlCdoagV3Qv2Pj3rOlc/xjneH/PHhyqqcG9Dg5YOns2xz+MhAmEqAsSrQJlH5dB2agEJID3HG9T\nh2QyUx98EFMffBC9f/gBn95zD+dENyLewiLqghoLG5H/Qj7qfqlDv1X9TB0O9VC8AiEyYwq5Ag2/\n3f5xxKaLTbD2ssagfw8yQVRE1zGBEJmxC/+4gLKtZbDysrptm/O9ziaIiOgGJhAiMyZaBPyW+KHv\n8r6mDoXoNhwDISIinTCBEBGRTngLi8iMnH/1PGozalXL9b/Ww3ehrwkjIlKPCYTIjFz7+hp8nvaB\nXaCdap3zKA6Wk3liAiEyMy5jXeB0N6ekBYDM2lqkVlRoVbZWoTBwNHQrJhAiMls7y8uRU1eHUc6a\nr8Je6tsXvaytjRAV/YEJhIgMqrKlBdlyuU51ixobMdXDA3+7ZVZTMg9MIERkUB+UlmJLaSkG2Nlp\nLtyOMAcHPUdE+sIEQkQGpRACsT4+WNu/v6lDIT3jeyBERKQTJhAiItIJb2ERmYGcmTlovNCI+l/r\nIbHsYdMKUpfFBEJkBir3V2Lwl4Nh6WYJhyEcNKaugQmEyESEQqBwdSEUdQooGhRwuscJlq78vyR1\nHRwDITKR1upWFL1VBBs/GwS9HwQLFwtTh0TUKfxzh8jIlC1KVKRUoLWyFVI7KfyX+Js6JCKdMIEQ\nGVnd6Tr8GvsrXB9yhU+sj6nDIdIZEwiRsQnALtAOQ74cYupIDKpZqURFaytqFQrYSXm3vDvif1Ui\nMojn8/NxV3o6PikvR19bW1OHQwbAKxAiMogGpRLvBQXhKR/epuuueAVCREQ6YQIhIiKd8BYWkYFc\nTb6KK19euW19y9UWo8ZxvLoaW0pLAQBT6urw7pkzRtnvsepqjHd1Ncq+yDSYQIgM5No31yCRSOA6\n4faT6M1znhva4aoqVLS24tFeveBWW4sH3dyMst8H3dwwzcPDKPsi02ACIdKztJoa/CyXw6u+Ho1D\nbFAzUdlOqTqgtM4o8fxYW4thDg6I9fFBdlkZpnBQm/SECYRIz/5+/jwsJRLMaG5BRZNAfm2tSePp\nZWWFye7uJo2BuicmEKI7dLGxEXKFQrV89XAGBidl4cdfWyH5RYJZ9zyOB6c+aMIIiQyDCaSbyszM\nRHh4uKnDMAuG7ovA9HQMtLODBEDrsZPo/+4PeKZ47vWNl4Bdz+8CALNIIvx3cQP74s4Z5DHe1NRU\nBAcHIygoCBs2bGi3zOLFixEUFISwsDBkZWVprFtRUYGJEydi0KBBmDRpEqqqqgwRereRmZlp6hDM\nhqH7okUI5I4YgdyRIzHlf7l46Y/k8bs5BXOQ/H6yQWPQFv9d3NBT+uL7b77BK5MnY/X48Xhl8mR8\n/803emtb71cgCoUCixYtwnfffQdfX1+MGDECUVFRCAkJUZVJSUlBfn4+8vLykJ6ejgULFiAtLa3D\nuuvXr8fEiROxfPlybNiwAevXr8f69ev1HT5pUJdbh+xx2RAKYepQtNY0pgnH3I4ZrP3kVuC45XEA\nQK1czXhHo8F2T6TW9998g/89/zzeKChQrVv1++/jpk694/b1nkAyMjIQGBiIgIAAAEBMTAySkpLa\nJJDk5GTExsYCAEaNGoWqqiqUlZXh/PnzausmJyfjyJEjAIDY2FiMHz+eCcQEWitaYRdoh6H7hpo6\nFK19suITjPptlMHadz92HNfGjoREIsHuR3YDh9opxE9BkQl8u3Fjm+QBAG8UFODv779vngmkpKQE\n/v435jfw8/NDenq6xjIlJSUoLS1VW7e8vBze3t4AAG9vb5SXl7e7f4mE80n/YfPmzYZrvIs91LPl\nwy0Gbd/mpt8TkHB7gf8B70neM2gM2jLov4supif0xRvtrfzf//C6Hs6Vek8g2p7AhdB8C0QI0W57\nEomk3fXatElERPqh90F0X19fFBUVqZaLiorg5+fXYZni4mL4+fm1u97X1xfA9auOsrIyAMClS5fg\n5eWl79CJiKgT9J5AZDIZ8vLyUFhYiObmZuzevRtRUVFtykRFRWHHjh0AgLS0NLi6usLb27vDulFR\nUfj4448BAB9//DGmT5+u79CJiKgT9H4Ly9LSEgkJCZg8eTIUCgXi4uIQEhKiutcYHx+PyMhIpKSk\nIDAwEA4ODti2bVuHdQFg5cqVeOyxx7B161YEBATgs88+03foRETUGaIL2rdvn7jrrrtEYGCgWL9+\nfbtlnnvuOREYGCiGDRsmTp48aeQIjUdTX+zcuVMMGzZMDB06VNx7773i1KlTJojSOLT5dyGEEBkZ\nGcLCwkJ8+eWXRozOuLTpi0OHDonw8HAxePBg8cADDxg3QCPS1BdXrlwRkydPFmFhYWLw4MFi27Zt\nxg/SCJ5++mnh5eUlhgwZorZMZ8+bXS6BtLa2ioEDB4rz58+L5uZmERYWJnJzc9uU+eabb0RERIQQ\nQoi0tDQxatQoU4RqcNr0xQ8//CCqqqqEENf/j9ST++KPchMmTBBTp04VX3zxhQkiNTxt+qKyslKE\nhoaKoqIiIcT1k2h3pE1fvPrqq2LlypVCiOv94O7uLlpaWkwRrkF9//334uTJk2oTiC7nzS43odTN\n75lYWVmp3hW5WXvvmah77Lcr06YvxowZAxcXFwDX+6K4uNgUoRqcNn0BAO+//z4effRR9OrVywRR\nGoc2ffHpp5/ikUceUT3g4unpaYpQDU6bvujduzdqamoAADU1NfDw8IClZff7ytP9998Ptw4+5a/L\nebPLJRB175BoKtMdT5za9MXNtm7disjISGOEZnTa/rtISkrCggULAHTfd4a06Yu8vDxUVFRgwoQJ\nkMlk+OSTT4wdplFo0xfz5s1DTk4O+vTpg7CwMLz3nnm8r2Nsupw3u1ya1fU9k+54sujMMR06dAgf\nffQRjh8/bsCITEebvvjb3/6G9evXQyKRQFy/fWuEyIxPm75oaWnByZMnceDAAdTX12PMmDEYPXo0\ngoKCjBCh8WjTF+vWrUN4eDgOHz6MgoICTJw4EadOnYKTk5MRIjQvnT1vdrkEout7Jn+8T9KdaNMX\nAHD69GnMmzcPqampHV7CdmXa9MVPP/2EmJgYAMDVq1exb98+WFlZ3faYeVenTV/4+/vD09MTdnZ2\nsLOzw7hx43Dq1Klul0C06YsffvgBq1atAgAMHDgQ/fv3x9mzZyGTyYwaq6npdN7U2wiNkbS0tIgB\nAwaI8+fPi6amJo2D6CdOnOi2A8fa9MWFCxfEwIEDxYkTJ0wUpXFo0xc3e+qpp7rtU1ja9MWZM2fE\nQw89JFpbW0VdXZ0YMmSIyMnJMVHEhqNNX7zwwgti9erVQgghysrKhK+vr7h27ZopwjW48+fPazWI\nru15s8tdgdzJeybdjTZ9sXbtWlRWVqru+1tZWSEjI8OUYRuENn3RU2jTF8HBwZgyZQqGDRsGqVSK\nefPmITQ01MSR6582ffHyyy/j6aefRlhYGJRKJf75z3/CvRvO4Dhr1iwcOXIEV69ehb+/P9asWYOW\nlhYAup83JUJ00xvBRERkUF3uKSwiIjIPTCBERKQTJhAiItIJEwgREemECYSokywsLDB8+HAMGzYM\nM2bMgFwu12v7AQEBqKioAAA4OjrqtW0ifWICIeoke3t7ZGVl4fTp03B2dtb7tKg3v/3bHb+gQN0H\nEwjRHRgzZgwKCgoAAAUFBYiIiIBMJsO4ceNw9uxZAEB5eTmio6MRHh6O8PBwpKWlAQCio6Mhk8kw\nZMgQfPjhhyY7BiJddbkXCYnMhUKhwLfffouHHnoIAPDss89i8+bNCAwMRHp6Ov7617/iwIEDWLx4\nMSZMmICvvvoKSqVSdcvro48+gpubGxoaGjBy5Eg8+uij3fZTM9Q98UVCok6ytLTE0KFDUVJSgoCA\nAKSlpaG+vh5eXl646667VOWam5uRk5MDLy8vlJSUwMrKqk07q1evxp49ewAAhYWF+PbbbzFy5Ej0\n798fP/30E9zd3eHk5ITa2lqjHh+RtngFQtRJdnZ2yMrKQkNDAyZPnoykpCT86U9/gqurK7Kystqt\nc+vfaYcPH8aBAweQlpYGW1tbTJgwAY2NjcYIn0hvOAZCpCM7Ozts3LgRq1atgqOjI/r3748vvvgC\nwPWEcfr0aQDAQw89hH//+98Art/2qqmpQU1NDdzc3GBra4tff/1VNS5C1JUwgRB10s1PRoWHhyMw\nMBCfffYZdu3aha1btyI8PBxDhgxBcnIyAOC9997DoUOHMGzYMMhkMpw5cwZTpkxBa2srQkND8dJL\nL2HMmDEa90VkbjgGQkREOuEVCBER6YQJhIiIdMIEQkREOmECISIinTCBEBGRTphAiIhIJ/8fT0cs\ntRRZyHYAAAAASUVORK5CYII=\n",
       "text": [
        "<matplotlib.figure.Figure at 0x112dc6790>"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model.estimator.bias_.pop()\n",
      "model.estimator.estimators_.pop()\n",
      "model.add_layer(on3, off3[:462])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for _make_xy: 33.79 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Is it posible that the first stage of fitting overfit, doomed the cascade?\n",
      "\n",
      "No, but it seems that imbalanced drastically reduces performance"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import random\n",
      "model2 = Model(ex, l, \n",
      "              weak_learner_params=dict(verbose=1, max_depth=1, n_estimators=200, subsample=.4), \n",
      "              cascade_params=dict(verbose=1, max_layers=1))\n",
      "off_all = off3 + model.training_data[0]['neg']\n",
      "\n",
      "model2.fit(on3, random.sample(off_all, len(on3)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for _make_xy: 42.55 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Fitting\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for fit: 61.20 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Cascade round 1. False pos rate: 0.000000e+00. Recall: 9.935065e-01\n"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model2.add_layer(on3, random.sample(off_all, len(on3)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Runtime for _make_xy: 37.59 s\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "colors = ['c', 'm', 'b']\n",
      "yp = model2.estimator.decision_function(x)\n",
      "rfp_curve(yp, y, color='red', label='Training Data')\n",
      "\n",
      "for i, y2p in enumerate(model2.estimator.staged_decision_function(x2)):\n",
      "    rfp_curve(y2p, y2, label = 'CV %i' % i, color = colors[i])\n",
      "\n",
      "\n",
      "plt.ylim(0, .002)\n",
      "plt.legend(loc='upper left')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 40,
       "text": [
        "<matplotlib.legend.Legend at 0x11b32e150>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEUCAYAAADnQnt7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtcVGX+B/DPDKDcbyqIgKIMBngBjETLFC1FwUgtDbSi\nJDPT7Lamm+5mpobbZr/MdaXWS5YZWquQIlneNYElwTbQRAIFFLxwF3CY4fn94TqJMsyAhxkun/fr\nNa8X55zneeZ7HvB8Pec55zwyIYQAERGRhOTGDoCIiDoeJhciIpIckwsREUmOyYWIiCTH5EJERJJj\nciEiIskxuVCb4eHhAUtLS9jY2KBnz5545plnUFFR0aK2Fi5ciO7du6N79+5YtGiR1nJ5eXmQy+Ww\nsbHRfFasWHFXOaVSCR8fH7i7uzdYn5mZieDgYNjb28Pd3R3Lly/XO8bg4GBYWFg0+O6UlBQAgFwu\nh7W1NWxsbODm5oY333wT9fX1ercNACUlJZg8eTKsra3h4eGBbdu2NVn+o48+gouLC+zs7BAdHQ2l\nUql3W/v374e3tzesrKwwZswYXLhwocF2fX8f1IEIojbCw8ND7N+/XwghRFFRkfDz8xMLFixodjvr\n168X9913nygsLBSFhYXC19dXrF+/vtGyubm5QiaTifr6+ibbXL58uRg5cqRwd3dvsD4gIEAsWbJE\n1NfXi5ycHOHi4iISEhL0ijM4OFhs2LCh0W0ymUzk5OQIIYQ4c+aM6Nmzp9Z90CYiIkJERESI69ev\ni2PHjgk7OzuRmZnZaNmkpCTh7OwssrKyRGlpqQgODhaLFi3Sq60rV64IOzs78c0334gbN26IBQsW\niGHDhmnqNuf3QR0Hkwu1GbcnFyGEWLBggQgNDW12O8OHDxefffaZZnnjxo0NDna3u5VcVCqV1vZ+\n//134ePjI/bu3Svc3NwabOvatas4ffq0Znnq1KkiJiZGrzj1TS632n3llVf0alcIIaqqqkSXLl1E\ndna2Zt2zzz7bIGHcLjIyUixevFizfODAAdGzZ0+92oqNjRUPPfSQZtv169eFhYWF+O2334QQzft9\nUMfBy2LUpoj/vTCioKAASUlJCAoK0myLiYmBg4NDox9HR0dNuaysLPj5+WmWBw8ejMzMzCa/t0+f\nPnB3d8fMmTNx7dq1BtteeeUVvP/++zA3N7+r3rhx4/D5559DpVLhzJkzOHHiBB599NFm729T27Ky\nsnD06FEEBAQAACZOnKi1H8LDwwEAZ8+ehampKRQKhaY9Pz8/rf3QWJ8VFxejtLRUZ1uZmZkN6lpa\nWkKhUGi2t+T3Qe0fkwu1GUIITJo0Cba2tujduzc8PT2xZMkSzfZFixahtLS00U9JSYmmXFVVFezs\n7DTLtra2qKqqavQ7e/TogbS0NFy4cAE///wzKisrMWPGDM32nTt3QgiBxx9/vNH6H330EeLi4mBh\nYQFfX1+88MILuP/++/Xe3/nz52sSQ2BgYIPtQ4YMgaOjI8LDwzFr1iw899xzAIDdu3dr7YeEhARN\nH9ja2jZoz8bGBpWVlY3G0lifAUBlZaXOthrbbmtr22C7vr8P6jhMjR0A0S0ymQzx8fEYM2YMjhw5\ngsceewxpaWkYOnRos9qxtrZucCNAeXk5rK2tGy1rZWWFIUOGAACcnJywdu1auLi44Pr16wCAt956\nC3v37m20bnV1NcaMGYMVK1Zg+vTpuHTpEp588kk4OTlhzpw5eu3vJ598gpkzZza6PT09Hf369dPZ\nTmPu7APgZj/Y2NjoVb68vBzAzSSira1bCcXGxqbJ72rO74M6Dp65UJs0cuRIvPLKK1i4cKFm3cqV\nKxvcWXX75/b/OQ8YMAAZGRma5VOnTmHgwIHN+v76+npkZ2fj/PnzePjhh+Hi4oInnngCly5dQq9e\nvXDhwgVkZmaisrISTz/9NORyOVxdXfHUU08hMTHx3jugCRMmTNDaD2FhYQCA/v37Q6VS4dy5c5p6\nTfVDY33m7OwMBwcHrW0NGDBAU/fUqVOabdevX0dOTk6D7ff6+6B2yJgDPkS3u3NA/8qVK8LS0lIk\nJyc3q53169cLHx8fUVhYKAoKCoSvr6+IjY1ttGxKSoo4c+aMUKvV4urVq2LatGlizJgxQgghVCqV\nKC4u1nz+/e9/i169eoni4mKhVqtFSUmJsLKyEl999ZVQq9Xi0qVLYtiwYZqB8Vs3C5w/f77R7w4O\nDhb/+te/Gt1254B+S0RERIjIyEhx/fp1cfToUWFnZyeysrIaLZuUlCR69uwpsrKyRElJiRg1apT4\n85//rFdbt+4W+/bbb0VNTY1YsGCBGD58uKZuc34f1HEwuVCbcWdyEUKIOXPmiMmTJze7rbfeeks4\nOjoKR0dHsXDhwgbbBgwYIL766ishhBDbtm0Tffv2FVZWVsLFxUVERUWJ4uLiRts8ePDgXbciJyYm\nioCAAGFrayt69uwpXnzxRVFTUyOEEOLIkSOib9++Wu9Ea+puMblcfs/JpaSkREyaNElYWVmJPn36\niG3btmm2nT9/XlhbW4v8/HzNutWrVwtnZ2dha2srZs6cKZRKpV5tCSHEjz/+KLy9vYWFhYUYPXr0\nXQm1qd8HdUwyITifC1FrWLFiBZycnDBr1ixjh0JkcK0y5pKUlARvb294eXlh1apVjZaZP38+vLy8\n4Ofnh/T0dJ11FyxYAB8fH/j5+WHKlCmaAUcAeP/99+Hl5QVvb2/s27evNXaJqNkWL17MxEKdl9Sn\nQiqVSnh6eorc3FyhVCqFn5/fXdd59+zZIyZMmCCEECI5OVkEBQXprLtv3z6hVquFEEIsXLhQc2qd\nmZkp/Pz8hFKpFLm5ucLT01NTjoiIjEPyM5fU1FQoFAp4eHjAzMwMERERiI+Pb1AmISEBUVFRAICg\noCCUlZWhqKioybpjx46FXC7X1CkoKAAAxMfHIzIyEmZmZvDw8IBCoUBqaqrUu0VERM0g+XMuhYWF\nDV7u5+bmpnkZX1NlCgsLcfHiRZ11AWDjxo2IjIwEAFy8eBHDhg27q63byWSye9spIqJOSrRwWF7y\nMxd9D+QtDXjFihXo0qULpk+f3qwYxM074zr9Z/bs2UaPoa182Beduy/iP6xAoH1Fg3Xygwc7ZV9o\n+9wLyc9cXF1dkZ+fr1nOz8+Hm5tbk2UKCgrg5uaGurq6Jutu3rwZiYmJ2L9/f5Ntubq6SrpPRETU\nPJKfuQQGBiI7Oxt5eXlQKpWIi4vTvEzvlvDwcGzZsgUAkJycDHt7ezg7OzdZNykpCR988AHi4+Mb\nvEAwPDwcX3/9NZRKJXJzc5Gdnd3s14UQEZG0JD9zMTU1xdq1axESEgK1Wo3o6Gj4+PggNjYWADB7\n9myEhoYiMTERCoUCVlZW2LRpU5N1gZtvplUqlRg7diwAYPjw4Vi3bh18fX0xbdo0+Pr6wtTUFOvW\nreMYSxPufDliZ8a++AP74g/6vniUmtYpHqKUyWT3fP2wo8jIyIC/v7+xw2gT2Bd/6Ix9kbC6Eu+9\nB/yn9I+XeZocOoQ0OzvN9Aad3b0cOzv1W5EdHR1RWlpq7DDIABwcHBq8lp+IWlenTi6lpaU8o+kk\neKmUyLD4yn0iIpIckwsREUmOyYWIiCTH5EJERJJjcukEQkND8cUXX0helohIGyaXNsra2lozL7pc\nLoelpaVmedu2bc1qKzExEc8884zkZZvj0KFDkMvlmn1wd3fHU089hbS0NL3bWLp0aavERkTSY3Jp\no6qqqlBZWYnKykr06dMHu3fv1izfeiM0AKhUKiNG2Tyurq6afUhOToa3tzcefvhhHDhwwNihEZHE\nmFzamUOHDsHNzQ1/+9vf4OLigujoaJSVlWHixIlwcnKCo6MjHnvssQbTDgQHB2PDhg0Abr78c8SI\nEViwYAEcHR3Rr18/JCUltahsbm4uRo4cCVtbW4wdOxZz587V+8zC1dUV7777Ll544QUsXLhQs/7V\nV19F7969YWdnh8DAQBw7dgzAzXfLvf/++4iLi4ONjY3mCepNmzbB19cXtra28PT0xKefftrCniUi\nKTG5NEUmk+YjseLiYpSWluLChQuIjY1FfX09oqOjceHCBVy4cAEWFhaYN2/ebbsha/AQYWpqKry9\nvXHt2jW89dZbiI6OblHZ6dOnY9iwYSgpKcHSpUvx5ZdfNvthxcmTJ+PkyZOoqakBAAwdOhSnTp1C\naWkppk+fjqlTp0KpVGL8+PF4++23ERERgcrKSs3U2M7OztizZw8qKiqwadMmvP766w2mzSYi42By\naYoQ0nwkJpfL8e6778LMzAzm5uZwdHTE5MmTYW5uDmtra7z99ts4fPiw1vp9+vRBdHQ0ZDIZnn32\nWVy6dAmXL19uVtkLFy4gLS0Ny5Ytg6mpKR566CGEh4c3+40HvXr1ghACZWVlAIAZM2bAwcEBcrkc\nb7zxBm7cuIHffvsNABqdYyI0NBR9+/YFAIwcORLjxo3D0aNHmxUDEUmPyaUd6tGjB7p06aJZrq6u\nxuzZs+Hh4QE7OzuMGjUK5eXlWg/0PXv21PxsaWkJ4OYYT3PKXrx4EY6Ojg2mP7h9FlF9FRYWQiaT\nwd7eHgDw97//Hb6+vrC3t4eDgwPKy8tx9epVrfX37t2LYcOGoVu3bnBwcEBiYiKuXbvW7DiISFpM\nLu3QnZeePvzwQ5w9exapqakoLy/H4cOHJZlJrikuLi4oKSnRXM4CgAsXLjS7nZ07d+L++++HhYUF\njh49ig8++AA7duxAWVkZSktLYWdnp9mPO/f7xo0beOKJJ/DWW2/h8uXLKC0tRWhoKN8XR9QGMLl0\nAFVVVbCwsICdnR1KSkrw7rvvtvp39unTB4GBgVi6dCnq6upw4sQJ7N69W68xFyEECgsL8e6772LD\nhg1YuXIlAKCyshKmpqbo3r07lEolli1bhoqKCk29nj17Ii8vT5M8lEollEolunfvDrlcjr1792Lf\nvn2ts8NE1CxMLu3QnQfw1157DTU1NejevTsefPBBTJgwQetB/s4B+8ba07fs1q1bceLECXTr1g1/\n+ctf8NRTTzW4XHdnvYsXL2qecxk6dCgyMzNx+PBhPProowCA8ePHY/z48ejfvz88PDxgYWGB3r17\na9qYOnUqAKBbt24IDAyEjY0N1qxZg2nTpsHR0RHbtm3D448/3uj3E5FhderJwjiJmLSeeuop+Pr6\n4p133jF2KHdp7HfdGSfI0qYz9UV6Yg0+XHQDF6/IUVkrw9FrVlj8+++4IQT+UViIk5wsTONejpE8\nc6EWS0tLQ05ODurr67F3714kJCRg0qRJxg6LqEkHt9/AL7mmGDOiHu8uA4qVSmwsKoK3pSW+9PHh\n3D8S6dSThdG9KSoqwpQpU3Dt2jW4u7tj/fr18PPzM3ZYRDr1d1FhyY6bdyier62FrYkJ5rm6AgAy\nLl0yZmgdBpMLtdjEiRMxceJEY4dBRG0QL4sREZHkmFyIiEhyTC5ERCQ5JhciIpIckwsREUmOyYWI\niCTH5NLGffXVV5pXnfTq1QuhoaE4fvw4vv76a82r5m+nUqng5OSExMREre316dMH1tbWmDx5MkpL\nS1t7F4ioE2JyacNWr16N119/HUuWLMHly5eRn5+PuXPn4rvvvsPkyZNRVlZ217wtSUlJMDExwfjx\n4+9qLzMzEy+99BK2bt2K4uJiWFpa4uWXXzbU7hBRJ8KHKNuo8vJyvPPOO9i8eXODV6qEhYUhLCwM\nADBt2jRs2bIFo0aN0mzfsmULpk+fDrn87v83bN26FeHh4RgxYgQA4L333oOPjw+uX78OKyurVt4j\nIupMmFyaIDt0SJJ2RHBws+ucOHECtbW1mDx5stYyUVFRmDBhAv7xj3/A3Nwc5eXl2L17N5KTkxst\nn5WVhYceekiz3K9fP3Tt2hVnz57li/qISFJMLk1oSVKQyrVr1zTzlGjz4IMPwtnZGTt37kRkZCS2\nb9+O++67D4MHD260fFVVFezs7Bqss7W1RWVlpaSxExFxzKWN6tatG65evYr6+vomyz377LPYsmUL\nAOCLL77As88+q7WstbU1ysvLG6wrLy+HjY3NvQdMRHQbJpc2avjw4ejatSt27tzZZLmnn34a+/fv\nx4kTJ5CSkoIZM2ZoLTtgwACcOnVKs5yTkwOlUon+/ftLFjcREcDk0mbZ2dlh2bJlmDt3LuLj41Fd\nXY26ujrs3bsXCxcu1JTz8PDAiBEjEBkZiXHjxsHJyUlrmzNmzMB3332HY8eO4fr16/jLX/6CJ554\ngoP5RCQ5Jpc27I033sDq1auxfPlyODk5oXfv3li3bt1dg/xRUVHIz89v8pIYAPj6+mL9+vWYMWMG\nnJ2dUVNTg3Xr1rXmLhBRJ8UB/TZu+vTpmD59epNloqKiEBUVpVd7kZGRiIyMlCI0IiKteOZCRESS\nY3IhIiLJMbkQEZHkmFyIiEhyTC5ERCQ5JhciIpIckwsREUmOyYWIiCTH5EJERJJjcmnjpJzmuKio\nCOHh4XB1dYVcLseFCxcMsQtE1AkxubRhUk9zLJfLERoaim+//dZQu0BEnVSrJJekpCR4e3vDy8sL\nq1atarTM/Pnz4eXlBT8/P6Snp+usu2PHDgwYMAAmJiY4efKkZn1eXh4sLCwQEBCAgICADjMn/K1p\njtetW4dJkybBwsICJiYmCAsLQ0xMDLp27aqZ5vh2TU1z7OTkhJdeegmBgYGG2g0i6qQkf3GlWq3G\nvHnz8OOPP8LV1RUPPPAAwsPD4ePjoymTmJiIc+fOITs7GykpKZgzZw6Sk5ObrDto0CDs3LkTs2fP\nvus7FQpFgwQllUOyQ5K0EyyCm12nNaY5JiIyFMmTS2pqKhQKBTw8PAAAERERiI+Pb5BcEhISNG/x\nDQoKQllZGYqKipCbm6u1rre39z3F9dJLL2l+DgwM1Ot/7y1JClJpjWmOO7uMjIwGy1VVVXet66w6\nU194jVfBeTiQkXHz8FdbX49FSqVm/ztTX9wpLS0NaWlpkrQleXIpLCyEu7u7ZtnNzQ0pKSk6yxQW\nFuLixYs66zYmNzcXAQEBsLOzw/LlyzFixIi7yqxfv74lu2M0t09z3FSCuTXNcWRkpM5pjjs7f3//\nBssZGRl3reusOlNfHPi/Mvz0EzDjrD0A4HxtLZ5KT8f5/+1/Z+qLO/n7++OFF17QLMfGxra4LcnH\nXGQymV7lhBCSfF+vXr2Qn5+P9PR0rF69GtOnT0dlZaUkbRtTa0xzTERkKJInF1dXV+Tn52uW8/Pz\n4ebm1mSZgoICuLm56VX3Tl26dIGDgwMAYMiQIfD09ER2drYUu2JUrTHNMQDU1taitrb2rp+JiKQk\neXIJDAxEdnY28vLyoFQqERcXh/Dw8AZlwsPDNXc5JScnw97eHs7OznrVBRqe9Vy9ehVqtRoA8Pvv\nvyM7Oxv9+vWTereMQuppjgHA0tIStra2kMlk8Pb2hpWVVWuFT0SdmORjLqampli7di1CQkKgVqsR\nHR0NHx8fzbW72bNnIzQ0FImJiVAoFLCyssKmTZuarAsAO3fuxPz583H16lWEhYUhICAAe/fuxeHD\nh/HOO+/AzMwMcrkcsbGxsLe3l3q3jEbqaY7r6+ulCIuIqEkyIdXgRxsmk8kaHePRtp46nsZ+1515\n4PZOnakvVj93c0D/m9sG9Eemp+P88OEAOldf6HIvx0g+oU9ERJJjciEiIskxuRARkeSYXIiISHJM\nLkREJDkmFyIikhyTCxERSY7JhYiIJMfk0sZJOc3xnj17MGLECDg4OMDFxQWzZs1CVVWVIXaDiDoZ\nJpc2TOppjisqKvDXv/4Vly5dwunTp1FYWIgFCxYYaneIqBOR/N1iJI1b0xxv3rwZkyZN0qwPCwtD\nWFgYAGimOR41apRme1PTHEdGRmp+Njc3x6xZs/DOO++04l4QUWfFM5cmyGTSfFpC32mOv/nmG81r\n829Nc6zvSywPHz6MgQMHtixAIqImMLk0QQhpPi3R3GmOATRrmuMffvgBW7ZswbJly1oWIBFRE5hc\n2qjbpzluyq1pjgHoPc1xcnIyZsyYgW+//RYKhUKSeImIbsfk0ka11jTH6enpePzxx7F582aMHj1a\nypCJiDSYXNqo1pjm+Ndff8X48eOxdu1ahIaGGmI3iKiTYnJpw6Se5nj16tW4du0aZs6cCRsbG9jY\n2GDQoEGtuQtE1EnpdStyXl4ezp07h0cffRTV1dVQqVSwtbVt7dgI0k5zvHHjRmzcuFGq0IiItNJ5\n5vLpp59i6tSpmD17NgCgoKCgydtjiYiIdCaXf/zjHzh27JjmTKV///64fPlyqwdGRETtl87LYl27\ndkXXrl01yyqVCrKWPhnYxjg4OHSYfaGmOTg4GDsEok5FZ3IZNWoUVqxYgerqavzwww9Yt24dHnvs\nMUPE1upKSkqMHYLBZWRkwN/f39hhEFEHp/Oy2KpVq9CjRw8MGjQIsbGxCA0NxfLlyw0RGxERtVM6\nz1x27dqFqKgovPjii4aIh4iIOgCdZy4JCQnw8vLCM888g927d0OlUhkiLiIiasd0JpfNmzfj3Llz\nePLJJ7Ft2zb069cP0dHRhoiNiIjaKb0eouzSpQsmTJgAuVyO6upq7Nq1Cxs2bGjt2IiIqJ3SmVwS\nExOxfft2HDx4EMHBwZg1axZ27NhhiNiIiCRXqlKhqK4ez585AwCoUquNHFHHpDO5bNmyBREREVi/\nfj3Mzc0NERMRUau5UqdEXb0JRtrZadbN7tXLiBF1TDqTy9dff22IOIiIDMZMLsPzLi7GDqND0zqg\n/9BDDwEArK2tNW/QvfXhSyuJiKgpWs9cjh8/DgCoqqoyWDBERNQx6LwV+ZlnntFrHRER0S06k8uv\nv/7aYFmlUuHnn39utYCIiKj905pcVq5cCRsbG/z3v/9tMN7i5OSE8PBwQ8ZIRETtjNbk8vbbb6Oy\nshJ/+tOfUFlZqfmUlJQgJibGkDESEVE7o3VA/8yZM/D29sbUqVNx8uTJu7YPGTKkVQMjIqL2S2ty\n+fDDD/HZZ5/hzTffbHRCrYMHD7ZqYERE1H5pTS6fffYZAODQoUOGioWISHLf/7MKh7+rQ6Vajct1\ndTiX2QUWlvXGDqvD03m32I4dO1BRUQEAeO+99zBlypRGL5MREbVFa2LUOP6zCXKKVCi8XI8ePVWY\nPFMYO6wOT2dyWbZsGWxtbXHs2DHs378fM2fOxEsvvWSI2IiIJBE5rR4BO6sxYY8KSad64s2/OBs7\npA5PZ3IxMTEBAOzevRuzZs3CxIkTUVdX1+qBERFR+6Uzubi6uuLFF19EXFwcwsLCUFtbi/p6Xq8k\nIiLtdCaX7du3IyQkBPv27YO9vT1KS0vxwQcfGCI2IiJqp3QmFysrK3h6eiIpKQlr167F5cuXMW7c\nOEPERkRE7ZTO5PLxxx/j6aefxpUrV1BcXIynn34aa9asMURsRETUTulMLv/617+QkpKCZcuW4b33\n3kNycrLmGRhtkpKS4O3tDS8vL6xatarRMvPnz4eXlxf8/PyQnp6us+6OHTswYMAAmJiY3HUr9Pvv\nvw8vLy94e3tj3759unaJiIhamc7kAgByubzRnxujVqsxb948JCUlISsrC9u2bcPp06cblElMTMS5\nc+eQnZ2NTz/9FHPmzNFZd9CgQdi5cydGjhzZoK2srCzExcUhKysLSUlJePnll3nDARGRkemc5vj5\n559HUFAQpkyZAiEEdu3ahZkzZ2otn5qaCoVCAQ8PDwBAREQE4uPj4ePjoymTkJCAqKgoAEBQUBDK\nyspQVFSE3NxcrXW9vb0b/b74+HhERkbCzMwMHh4eUCgUSE1NxbBhw/TtAyIikpjO5PLGG28gODgY\nR48ehUwmw+bNmxEQEKC1fGFhIdzd3TXLbm5uSElJ0VmmsLAQFy9e1Fn3ThcvXmyQSG61dafbH/wM\nDAxEYGBgk+12VFVVVcjIyDB2GG0C++IPHbEvKq6oUVMlMPlPMvTqKeBUXAcTmQwZpaVN1uuIfaGv\ntLQ0pKWlSdKW1uSSnJyM2bNn49y5cxg8eDA2bNgAX19fnQ029pLLxgjReq9faCyG9evXt9r3tScZ\nGRnw9/c3dhhtAvviDx2xLwbbVMFEBthZ1mPVOlMkOBfB0sQEj/fp02S9jtgX+vL398cLL7ygWY6N\njW1xW1oHUObOnYu///3vuHbtGt544w28/vrrejXo6uqK/Px8zXJ+fj7c3NyaLFNQUAA3Nze96ur6\nvoKCAri6uuoVKxF1XPVChg//T4ZDRbYImmJp7HA6Ha3Jpb6+HmPHjoW5uTmmTp2Ky5cv69VgYGAg\nsrOzkZeXB6VSibi4uLtmrgwPD8eWLVsA3DxDsre3h7Ozs151gYZnPeHh4fj666+hVCqRm5uL7Oxs\nDB06VK9YiYiodWi9LFZeXo5///vfmgP57csymQxTpkxpvEFTU6xduxYhISFQq9WIjo6Gj4+P5vRq\n9uzZCA0NRWJiIhQKBaysrLBp06Ym6wLAzp07MX/+fFy9ehVhYWEICAjA3r174evri2nTpsHX1xem\npqZYt26d3pfmiIiodciElsGP5557rsFB+lZSueVWQmgPZDJZq47xtCed+XryndgXf+iIfTHQ+jrW\nrAHGzLQCACz+/XdYmphgMcdc9HYvx06tZy6bN29uaTxERNTJ6fUQJRERUXMwuRARkeSYXIiISHI6\nk8v169fx3nvvYdasWQCA7Oxs7N69u9UDIyKi9ktncnn++efRpUsX/PTTTwCAXr16YfHixa0eGBGR\nFKrUapyrqUGJSmXsUDoVncklJycHCxcuRJcuXQDcnDyMiKi9mHv2LB46eRI/lJbC08LC2OF0Gjpf\nXNm1a1fU1NRolnNyctC1a9dWDYqISCpKIfB/CgUinZ2NHUqnojO5LF26FOPHj0dBQQGmT5+O48eP\n8xkYIiJqks7kMm7cOAwZMgTJyckAgDVr1qB79+6tHhgREbVfOsdcjh07BnNzc0ycOBGlpaVYuXIl\nzp8/b4jYiIiondKZXObMmQNLS0ucOnUKq1evhqenJ5599llDxEZERO2UzuRiamoKuVyOXbt2Ye7c\nuZg7dy6frXmJAAATT0lEQVQqKysNERsREbVTOsdcbGxssHLlSnz55Zc4evQo1Go16urqDBEbERG1\nUzrPXOLi4tC1a1ds3LgRPXv2RGFhIf70pz8ZIjYiImqndJ65uLi44M0339Qs9+7dG1FRUa0aFBER\ntW9ak4u1tbXWGR1lMhkqKipaLSgiImrftCaXqqoqQ8ZBRNRitbVAr15AZSUgVPUAAAFLRBX9B4WX\nq/EMn843OJ2XxW65fPkyamtrNcu9e/dulYCIiJqrrg5QKoHqaiChx1H4pw6Bi5slzMwDAQBmcs4u\nYmg6ezwhIQFeXl7o27cvRo0aBQ8PD0yYMMEQsRER6U0mA8zMAFOZgKWVHJaWJjCTy5lYjERnry9Z\nsgQnTpxA//79kZubi/379yMoKMgQsRERUTulM7mYmZmhe/fuqK+vh1qtxujRo5GWlmaI2IiIqJ3S\nOebi4OCAyspKPPzww5gxYwacnJxgbW1tiNiIiKid0nrmcuHCBQBAfHw8LC0t8dFHH2H8+PFQKBT4\n7rvvDBYgERG1P1rPXB5//HGkp6fDysoKTzzxBL799ls899xzBgyNiIjaK71uo/j9999bOw4iIupA\neI8eERFJTutlsV9++QU2NjYAgJqaGs3PAF//QkRETdOaXNRqtSHjICLSW3o68NxzQH1dPWpza6EW\nMshVZvjP4HRYVwIy08bfi0iGo/frX4iI2orz5wFHR+D9l2uR86dz8PxQAVvrG+jdywfDszJwwIGH\nNmPjb4CI2iU7O2Bg/3qY2N/AA09aatYXVfKspS3ggD4REUmOyYWIiCTH5EJERJJjciEiIskxuRAR\nkeSYXIiISHJMLkREJDkmFyIikhwfoiSiNi8+HvjqK6DumhI1Z2tQVG2GbuYq5LyVC5mMD022RUwu\nRNTm/fADYGkJPNC1AlXXrsN+sj283VVwcXeBeV9zY4dHjWByIaJ24f77gceUNbjRvQ6K1XbGDod0\n4JgLERFJjsmFiIgkx+RCRESSY3IhIiLJMbkQEZHkWiW5JCUlwdvbG15eXli1alWjZebPnw8vLy/4\n+fkhPT1dZ92SkhKMHTsW/fv3x7hx41BWVgYAyMvLg4WFBQICAhAQEICXX365NXaJiIiaQfLkolar\nMW/ePCQlJSErKwvbtm3D6dOnG5RJTEzEuXPnkJ2djU8//RRz5szRWTcmJgZjx47F2bNn8cgjjyAm\nJkbTnkKhQHp6OtLT07Fu3Tqpd4mIiJpJ8uSSmpoKhUIBDw8PmJmZISIiAvHx8Q3KJCQkICoqCgAQ\nFBSEsrIyFBUVNVn39jpRUVHYtWuX1KETEZFEJH+IsrCwEO7u7pplNzc3pKSk6CxTWFiIixcvaq1b\nXFwMZ2dnAICzszOKi4s15XJzcxEQEAA7OzssX74cI0aMuCuul156SfNzYGAgAgMD73FP26eqqipk\nZGQYO4w2gX3xh7baF9euAZWVgP9gge7mKpwXN2DS2wRVGVVa6yxVqZD7668olLfs/85ttS8MIS0t\nDWlpaZK0JXly0fc9P0IIvco01p5MJtOs79WrF/Lz8+Hg4ICTJ09i0qRJyMzMhI2NTYM669ev1yuu\nji4jIwP+/v7GDqNNYF/8oa32RWgo4OAAdMu+hnEWV+A2ugccwxxh62+rtc7Y48eRNXAgenTp0qLv\nbKt9YQj+/v544YUXNMuxsbEtbkvy5OLq6or8/HzNcn5+Ptzc3JosU1BQADc3N9TV1d213tXVFcDN\ns5WioiL07NkTly5dgpOTEwCgS5cu6PK/P6IhQ4bA09MT2dnZGDJkiNS7RkRGMGMG4JVYAktvG7jO\nczV2OKQnycdcAgMDkZ2djby8PCiVSsTFxSE8PLxBmfDwcGzZsgUAkJycDHt7ezg7OzdZNzw8HJ9/\n/jkA4PPPP8ekSZMAAFevXoVarQYA/P7778jOzka/fv2k3i0iImoGyc9cTE1NsXbtWoSEhECtViM6\nOho+Pj6a06vZs2cjNDQUiYmJUCgUsLKywqZNm5qsCwCLFi3CtGnTsGHDBnh4eGD79u0AgCNHjuCv\nf/0rzMzMIJfLERsbC3t7e6l3i4iImkEm9Bn8aOdkMpleYzydQWe+nnwn9sUf2mpfhIYC8+YBXonZ\nsPS21OuyWI/jx5H1wAMcc5HAvRw7+YQ+ERFJjsmFiIgkx+RCRESSY3IhIiLJMbkQEZHkJL8VmYjo\ndr9VVyOjSvvrWppySWmHI2U1QE01VJVqHLt8WWed2vr6Fn0XSYvJhYha1fLz53G2uhoe5ubNrluk\ntMBP5eVwralBWVUdfr2i1lnnyR49YGfKQ5ux8TdARK1KAHjFzQ1P/+/Fs80Rag3M62MDr0wlLF0s\n4TqAr39pLzjmQkREkmNyISIiyTG5EBGR5JhciIhIckwuREQkOSYXIiKSHJMLERFJjsmFiIgkx4co\niUhS+bW1KFWpNMuldXVGjIaMhcmFiCT1UHo6rExMYCaTAQBkAPq14NUv1L4xuRCRpOqEwAE/P7h0\n7WrsUMiIOOZCRESSY3IhIiLJMbkQEZHkmFyIiEhyTC5ERCQ53i1GRM2258ABrNm1CzfkcnStr8f8\nSZMQNmaMscOiNoTJhYiaZc+BA3h12zbkzJihWZezdSsAMMGQBi+LEVGzrNm1q0FiAYCcGTPwSUKC\nkSKitohnLkSdhBDAbW9lAQCohIAQAup6geob9X+UrRcQatFoO7UqM8hrZXetr1GaoKpKBVktoKyu\nxw11fSO1m6deLUO9Snss1HYxuRB1Ep98Arz2GmBicmuNgErcPGh/9CEQOPSPsnK1DDLcnUBu+gCy\nf9699jgehv0/AWAYPCWKWQ6Bx/dnwEZWCZ8vfCRqlQyBl8WIOomyMmDJEqCu7ubn0NUKPJiaAaGS\nI3iUDEIl13xO+Kah/NcaqIT8rk/8/kPweGE21AePaD4e0S8ifv+hRsvfy0cp5JijGoJRdaPgFOFk\n7C6kZuCZCxE1y61B+0927UItAHMAr0yfzsF8aoDJhYiaLWzMGCYTahIvixERkeSYXIiISHJMLkRE\nJDkmFyIikhyTCxERSY7JhYiIJMfkQkREkuNzLkQdTPh//4us6mpc+cAD1Sl2mvX1Faawj7yEr1Iu\nAgBq1Gr0t7SEukqNiv9UIOXJFE3Z2gu1kHXR9voXIt2YXIg6mJNVVdjm44M3i2wQ+Z4agwNvvUBS\nBRe37ujStbumbHczM6iuqCCUAoOTBmvWy8xkMO9jbuDIqSNhciHqgPpaWMBcboL7+5lg5MCmy96A\nGpABFgoLwwRHnQLHXIiISHJMLkREJDkmFyIikhyTCxERSY7JpZNJS0szdghtBvviDxlnM4wdQpvB\nvwtptEpySUpKgre3N7y8vLBq1apGy8yfPx9eXl7w8/NDenq6zrolJSUYO3Ys+vfvj3HjxqGsrEyz\n7f3334eXlxe8vb2xb9++1tilDoP/cP7Qnvtiz54jCAlZguDgpQgJWYI9e45oLXtgzwHMD5mP14Jf\nw/yQ+Tiw58BdZTKymVxuac9/F/o4smcPloSEYGlwMJaEhODInj2t8j2S34qsVqsxb948/Pjjj3B1\ndcUDDzyA8PBw+Pj8Mf91YmIizp07h+zsbKSkpGDOnDlITk5usm5MTAzGjh2Lt956C6tWrUJMTAxi\nYmKQlZWFuLg4ZGVlobCwEI8++ijOnj0LuZwnZdQx7dlzBK+++j1yclZo1uXkLAYAhIWNbFD25Imf\nkfnZNszImaFZtzVnKwBgTBgn++psjuzZg+9ffRUrcnI06xb/7+eRYWGSfpfkR+DU1FQoFAp4eHjA\nzMwMERERiI+Pb1AmISEBUVFRAICgoCCUlZWhqKioybq314mKisKuXbsAAPHx8YiMjISZmRk8PDyg\nUCiQmpoq9W4RtRlr1uxrkFgAICdnBT755Ie7yh799liDxAIAM3JmIOGThFaNkdqmfWvWNEgsALAi\nJwc/fPKJ5N8l+ZlLYWEh3N3dNctubm5ISUnRWaawsBAXL17UWre4uBjOzs4AAGdnZxQXFwMALl68\niGHDht3V1p1kMr7K4pbY2Fhjh9BmtN++WHHXmu+/B2Sy5QAA99vW/7ux6t8DH8s+brBqs2yzZNG1\nd+3370K3u/9yAHz/PZZLfIyUPLnoexAXQuhVprH2ZDJZk99z5zZ9vouIiKQj+WUxV1dX5Ofna5bz\n8/Ph5ubWZJmCggK4ubk1ut7V1RXAzbOVoqIiAMClS5fg5OSkta1bdYiIyDgkTy6BgYHIzs5GXl4e\nlEol4uLiEB4e3qBMeHg4tmzZAgBITk6Gvb09nJ2dm6wbHh6Ozz//HADw+eefY9KkSZr1X3/9NZRK\nJXJzc5GdnY2hQ4dKvVtERNQMkl8WMzU1xdq1axESEgK1Wo3o6Gj4+PhormHOnj0boaGhSExMhEKh\ngJWVFTZt2tRkXQBYtGgRpk2bhg0bNsDDwwPbt28HAPj6+mLatGnw9fWFqakp1q1bx/EVIiJjEx3I\n3r17xX333ScUCoWIiYlptMwrr7wiFAqFGDx4sDh58qSBIzQcXX3x5ZdfisGDB4tBgwaJBx98UJw6\ndcoIURqGPn8XQgiRmpoqTExMxLfffmvA6AxLn744ePCg8Pf3FwMGDBCjRo0ybIAGpKsvrly5IkJC\nQoSfn58YMGCA2LRpk+GDNIDnn39eODk5iYEDB2ot05LjZodJLiqVSnh6eorc3FyhVCqFn5+fyMrK\nalBmz549YsKECUIIIZKTk0VQUJAxQm11+vTFTz/9JMrKyoQQN/+Rdea+uFVu9OjRIiwsTHzzzTdG\niLT16dMXpaWlwtfXV+Tn5wshbh5gOyJ9+uKdd94RixYtEkLc7AdHR0dRV1dnjHBb1ZEjR8TJkye1\nJpeWHjc7zJOGLX2+5tYtzR2JPn0xfPhw2NndnKUwKCgIBQUFxgi11enTFwDwySef4Mknn0SPHj2M\nEKVh6NMXX331FZ544gnNTTjdu3dvrKl2T5++cHFxQUVFBQCgoqIC3bp1g6lpx5sC6+GHH4aDg4PW\n7S09bnaY5KLt2RldZTriQVWfvrjdhg0bEBoaaojQDE7fv4v4+HjMmTMHQMd9JkqfvsjOzkZJSQlG\njx6NwMBAfPHFF4YO0yD06YtZs2YhMzMTvXr1gp+fHz7++OM7m+kUWnrc7DBpuKXP13TEA0lz9ung\nwYPYuHEjjh8/3ooRGY8+ffHaa68hJiYGMpkM4ualYgNEZnj69EVdXR1OnjyJ/fv3o7q6GsOHD8ew\nYcPg5eVlgAgNR5++WLlyJfz9/XHo0CHk5ORg7NixOHXqFGxsbAwQYdvSkuNmh0kuLX2+piM+E6NP\nXwDAL7/8glmzZiEpKanJ0+L2TJ+++PnnnxEREQEAuHr1Kvbu3QszM7O7bqFv7/TpC3d3d3Tv3h0W\nFhawsLDAyJEjcerUqQ6XXPTpi59++gmLF998Z5unpyf69u2L3377DYGBgQaN1dhafNyUZESoDair\nqxP9+vUTubm54saNGzoH9E+cONFhB7H16Yvz588LT09PceLECSNFaRj69MXtnnvuuQ57t5g+fXH6\n9GnxyCOPCJVKJa5fvy4GDhwoMjMzjRRx69GnL15//XWxdOlSIYQQRUVFwtXVVVy7ds0Y4ba63Nxc\nvQb0m3Pc7DBnLvfyfE1Ho09fLFu2DKWlpZpxBjMzsw75wk99+qKz0KcvvL29MX78eAwePBhyuRyz\nZs2Cr6+vkSOXnj598fbbb+P555+Hn58f6uvr8be//Q2Ojo5Gjlx6kZGROHz4MK5evQp3d3e8++67\nqKurA3Bvx02ZEB30AjMRERlNh7lbjIiI2g4mFyIikhyTCxERSY7JhYiIJMfkQiQRExMTBAQEYPDg\nwZgyZQqqqqokbd/DwwMlJSUAAGtra0nbJpIakwuRRCwtLZGeno5ffvkFtra2kk+Ve/tT0R3xzRLU\nsTC5ELWC4cOHIycnBwCQk5ODCRMmIDAwECNHjsRvv/0GACguLsbkyZPh7+8Pf39/JCcnAwAmT56M\nwMBADBw4EJ999pnR9oHoXnSYhyiJ2gq1Wo19+/bhkUceAQC8+OKLiI2NhUKhQEpKCl5++WXs378f\n8+fPx+jRo7Fz507U19drLqNt3LgRDg4OqKmpwdChQ/Hkk0922NfzUMfFhyiJJGJqaopBgwahsLAQ\nHh4eSE5ORnV1NZycnHDfffdpyimVSmRmZsLJyQmFhYUwMzNr0M7SpUuxa9cuAEBeXh727duHoUOH\nom/fvvj555/h6OgIGxsbVFZWGnT/iJqDZy5EErGwsEB6ejpqamoQEhKC+Ph4PProo7C3t0d6enqj\nde78v92hQ4ewf/9+JCcnw9zcHKNHj0Ztba0hwieSFMdciCRmYWGBNWvWYPHixbC2tkbfvn3xzTff\nALiZTH755RcAwCOPPIJ//vOfAG5eSquoqEBFRQUcHBxgbm6OM2fOaMZhiNobJhciidx+B5e/vz8U\nCgW2b9+OrVu3YsOGDfD398fAgQORkJAAAPj4449x8OBBDB48GIGBgTh9+jTGjx8PlUoFX19f/PnP\nf8bw4cN1fhdRW8QxFyIikhzPXIiISHJMLkREJDkmFyIikhyTCxERSY7JhYiIJMfkQkREkvt/JimT\nueELnd8AAAAASUVORK5CYII=\n",
       "text": [
        "<matplotlib.figure.Figure at 0x11b32eb50>"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "data = {'pos': on3, 'neg': off_all, 'cv_pos': on2, 'cv_neg': off2}\n",
      "with open('../models/benchmark_training_data.json', 'w') as outfile:\n",
      "    json.dump(data, outfile)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 38
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Tasks for this afternoon\n",
      "\n",
      " * Try a hyperparameter optimizing on benchmark_training_data, in a new notebook\n",
      " * Visualize the false positives / negatives. Any trends?"
     ]
    }
   ],
   "metadata": {}
  }
 ]
}