Linear Classifiers in Python - solutions.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Linear Classifiers in Python - solutions.ipynb","provenance":[],"collapsed_sections":["A1khgQHlhJkl","CvY3lSv2hLLF"],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"Q8ghyhgkg4AQ","colab_type":"code","colab":{}},"source":["# https://www.datacamp.com/courses/linear-classifiers-in-python"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"tMi3ERR_FCYD","colab_type":"code","colab":{}},"source":["import numpy as np\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","\n","from sklearn import datasets\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.svm import SVC, LinearSVC\n","from sklearn.neighbors import KNeighborsClassifier\n","\n","from scipy.optimize import minimize"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"z7ZcLEN1ycTg","colab_type":"code","colab":{}},"source":["# print array in full\n","import sys\n","import numpy\n","numpy.set_printoptions(threshold=sys.maxsize)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kYSkcxO6hGcn","colab_type":"text"},"source":["## 1. Applying logistic regression and SVM"]},{"cell_type":"markdown","metadata":{"id":"6qQyX73nhTsQ","colab_type":"text"},"source":["#### Applying logistic regression and SVM"]},{"cell_type":"markdown","metadata":{"id":"3QT7xfXNHqkL","colab_type":"text"},"source":["In this exercise, you'll apply logistic regression and a support vector machine to classify images of handwritten digits.\n","\n","\n","\n","*   Apply logistic regression and SVM (using SVC()) to the handwritten digits data set using the provided train/validation split.\n","*   For each classifier, print out the training and validation accuracy.\n","\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"kBA6wHRohJeF","colab_type":"code","colab":{}},"source":["from sklearn import datasets\n","from sklearn.model_selection import train_test_split"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"3j3D2cw1HhkZ","colab_type":"code","colab":{}},"source":["digits = datasets.load_digits()\n","X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"DxvDbjKDHi_u","colab_type":"code","outputId":"321623ba-7b88-4be6-c8e5-2f9262b76809","executionInfo":{"status":"ok","timestamp":1557748770343,"user_tz":-480,"elapsed":752,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":144}},"source":["# Apply logistic regression and print scores\n","lr = LogisticRegression()\n","lr.fit(X_train, y_train)\n","print(lr.score(X_train, y_train))\n","print(lr.score(X_test, y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n"],"name":"stderr"},{"output_type":"stream","text":["0.9970304380103935\n","0.9622222222222222\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"t1RVcvM9hWr5","colab_type":"code","outputId":"897d6a3d-531f-4065-f77f-41982b0e6f64","executionInfo":{"status":"ok","timestamp":1557748812244,"user_tz":-480,"elapsed":1517,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":110}},"source":["# Apply SVM and print scores\n","svm = SVC()\n","svm.fit(X_train, y_train)\n","print(svm.score(X_train, y_train))\n","print(svm.score(X_test, y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n","  \"avoid this warning.\", FutureWarning)\n"],"name":"stderr"},{"output_type":"stream","text":["1.0\n","0.32666666666666666\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"487fgZ2qESLz","colab_type":"text"},"source":["**Sentiment analysis for movie reviews**\n","\n","In this exercise you'll explore the probabilities outputted by logistic regression on a subset of the [Large Movie Review Dataset](https://http://ai.stanford.edu/~amaas/data/sentiment/).\n","\n","The variables X and y are already loaded into the environment. X contains features based on the number of times words appear in the movie reviews, and y contains labels for whether the review sentiment is positive (+1) or negative (-1)."]},{"cell_type":"code","metadata":{"id":"jG-nd0iHEQ79","colab_type":"code","colab":{}},"source":["# Instantiate logistic regression and train\n","lr = LogisticRegression()\n","lr.fit(X, y)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Hbbg40LBG0Id","colab_type":"code","colab":{}},"source":["# Predict sentiment for a glowing review\n","review1 = \"LOVED IT! This movie was amazing. Top 10 this year.\"\n","review1_features = get_features(review1)\n","print(\"Review:\", review1)\n","print(\"Probability of positive review:\", lr.predict_proba(review1_features)[0,1])\n","# Probability of positive review: 0.8079007873616059"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"HppA-l5UG1gl","colab_type":"code","colab":{}},"source":["# Predict sentiment for a poor review\n","review2 = \"Total junk! I'll never watch a film by that director again, no matter how good the reviews.\"\n","review2_features = get_features(review2)\n","print(\"Review:\", review2)\n","print(\"Probability of positive review:\", lr.predict_proba(review2_features)[0,1])\n","# Probability of positive review: 0.5855117402793947"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"epu_SwKHhWyJ","colab_type":"text"},"source":["#### Linear classifiers\n","\n","In this exercise, you'll visualize the decision boundaries of various classifier types.\n","\n","A subset of scikit-learn's built-in wine dataset is already loaded into X, along with binary labels in y."]},{"cell_type":"code","metadata":{"id":"x5c3XJNGHToq","colab_type":"code","colab":{}},"source":["from sklearn.linear_model import LogisticRegression\n","from sklearn.svm import SVC, LinearSVC\n","from sklearn.neighbors import KNeighborsClassifier"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"LjnMcrOwhYlp","colab_type":"code","colab":{}},"source":["wine = datasets.load_wine()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"s1Xn8OfUIKDr","colab_type":"code","outputId":"e709d77b-b7e6-474a-e53c-90051234b620","executionInfo":{"status":"ok","timestamp":1557752573261,"user_tz":-480,"elapsed":1169,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["X = wine.data[:, :2]\n","print(X.shape)\n","y = wine.target\n","print(y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(178, 2)\n","(178,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"6fWJdPztRLsg","colab_type":"code","colab":{}},"source":["# print(pd.DataFrame(X).to_string())"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ma9m4BS1IftD","colab_type":"code","colab":{}},"source":["# Define the classifiers\n","classifiers = [LogisticRegression(), \n","               LinearSVC(), \n","               SVC(), \n","               KNeighborsClassifier()]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"qXj3FRjoIj5C","colab_type":"code","outputId":"9d634f09-0ae8-4e3b-fe80-10bf39a60678","executionInfo":{"status":"ok","timestamp":1557752582177,"user_tz":-480,"elapsed":1014,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":178}},"source":["# Fit the classifiers\n","for c in classifiers:\n","    c.fit(X,y)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n","  \"the number of iterations.\", ConvergenceWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n","  \"avoid this warning.\", FutureWarning)\n"],"name":"stderr"}]},{"cell_type":"markdown","metadata":{"id":"iigsPwWSJ4di","colab_type":"text"},"source":["**Plot Classifier**"]},{"cell_type":"code","metadata":{"id":"FxTxGm0nQEvS","colab_type":"code","colab":{}},"source":["def plot_contours(ax, clf, xx, yy, proba=False, **params):\n","    \"\"\"Plot the decision boundaries for a classifier.\n","\n","    Parameters\n","    ----------\n","    ax: matplotlib axes object\n","    clf: a classifier\n","    xx: meshgrid ndarray\n","    yy: meshgrid ndarray\n","    params: dictionary of params to pass to contourf, optional\n","    \"\"\"\n","    if proba:\n","        Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:,-1]\n","        Z = Z.reshape(xx.shape)\n","        out = ax.imshow(Z,extent=(np.min(xx), np.max(xx), np.min(yy), np.max(yy)), origin='lower', vmin=0, vmax=1, **params)\n","        ax.contour(xx, yy, Z, levels=[0.5])\n","    else:\n","        Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])\n","        Z = Z.reshape(xx.shape)\n","        out = ax.contourf(xx, yy, Z, **params)\n","    return out"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"fEAXZa3_PxBB","colab_type":"code","colab":{}},"source":["def make_meshgrid(x, y, h=.02, lims=None):\n","    \"\"\"Create a mesh of points to plot in\n","\n","    Parameters\n","    ----------\n","    x: data to base x-axis meshgrid on\n","    y: data to base y-axis meshgrid on\n","    h: stepsize for meshgrid, optional\n","\n","    Returns\n","    -------\n","    xx, yy : ndarray\n","    \"\"\"\n","\n","    if lims is None:\n","        x_min, x_max = x.min() - 1, x.max() + 1\n","        y_min, y_max = y.min() - 1, y.max() + 1\n","    else:\n","        x_min, x_max, y_min, y_max = lims\n","    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n","                         np.arange(y_min, y_max, h))\n","    return xx, yy\n"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ThT0tZWzOTJU","colab_type":"code","colab":{}},"source":["def plot_classifier(X, y, clf, ax=None, ticks=False, proba=False, lims=None): \n","  X0, X1 = X[:, 0], X[:, 1]\n","  xx, yy = make_meshgrid(X0, X1, lims=lims)\n","  \n","  if ax is None:\n","    plt.figure()\n","    ax = plt.gca()\n","    show = True\n","  else:\n","    show = False\n","\n","    # can abstract some of this into a higher-level function for learners to call\n","    cs = plot_contours(ax, clf, xx, yy, cmap=plt.cm.coolwarm, alpha=0.8, proba=proba)\n","    if proba:\n","        cbar = plt.colorbar(cs)\n","        cbar.ax.set_ylabel('probability of red $\\Delta$ class', fontsize=20, rotation=270, labelpad=30)\n","        cbar.ax.tick_params(labelsize=14)\n","    #ax.scatter(X0, X1, c=y, cmap=plt.cm.coolwarm, s=30, edgecolors='k', linewidth=1)\n","    labels = np.unique(y)\n","    if len(labels) == 2:\n","        ax.scatter(X0[y==labels[0]], X1[y==labels[0]], cmap=plt.cm.coolwarm, s=60, c='b', marker='o', edgecolors='k')\n","        ax.scatter(X0[y==labels[1]], X1[y==labels[1]], cmap=plt.cm.coolwarm, s=60, c='r', marker='^', edgecolors='k')\n","    else:\n","        ax.scatter(X0, X1, c=y, cmap=plt.cm.coolwarm, s=50, edgecolors='k', linewidth=1)\n","\n","    ax.set_xlim(xx.min(), xx.max())\n","    ax.set_ylim(yy.min(), yy.max())\n","#     ax.set_xlabel(data.feature_names[0])\n","#     ax.set_ylabel(data.feature_names[1])\n","    if ticks:\n","        ax.set_xticks(())\n","        ax.set_yticks(())\n","#     ax.set_title(title)\n","    if show:\n","        plt.show()\n","    else:\n","        return ax    "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"KX_1JVJWIxMb","colab_type":"code","colab":{}},"source":["def plot_4_classifiers(X, y, clfs):\n","  # Set-up 2*2 grid for plotting.\n","  fig, sub = plt.subplots(2, 2)\n","  plt.subplots_adjust(wspace=0.2, hspace=0.2)\n","  \n","  for clf, ax, title in zip(clfs, sub.flatten(), (\"(1)\", \"(2)\", \"(3)\", \"(4)\")):\n","    # clf.fit(X, y)\n","    plot_classifier(X, y, clf, ax, ticks=True)\n","    ax.set_title(title)\n","  plt.show()  "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"sJvMVfqgREgR","colab_type":"code","outputId":"e617cf0b-21e7-4236-b2b5-e8752cf320be","executionInfo":{"status":"ok","timestamp":1557752591961,"user_tz":-480,"elapsed":1116,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["X.shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(178, 2)"]},"metadata":{"tags":[]},"execution_count":55}]},{"cell_type":"code","metadata":{"id":"Ow-LFAUmIoET","colab_type":"code","outputId":"4b248018-193d-4f23-9103-925758afcb23","executionInfo":{"status":"ok","timestamp":1557752597136,"user_tz":-480,"elapsed":4621,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":268}},"source":["# Plot the classifiers\n","plot_4_classifiers(X, y, classifiers)\n","plt.show()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAWQAAAD7CAYAAABdXO4CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnWdgFFXbhq8zs7vZ3Wx6ISGFEHrv\nvYMgYgUVVFRQXxVFsetr+bD33lCx+6pYEEUQBZWO9CK9hHTSe98yc74fCQtLgoCAUub6lczMmZmd\nPbnzzHOeIqSUGBgYGBj8+yj/9g0YGBgYGNRiCLKBgYHBKYIhyAYGBganCIYgGxgYGJwiGIJsYGBg\ncIpgCLKBgYHBKYIhyMeJEOJZIcSdR3nsy0KIW072PRkYnAiMuf3PI4w45L+PECIC2AQ0l1JWCyEs\nwJdAd6AJMERKufig46OBNUAzKaXrX7hlA4OjooG53Rt4EugGaMBiYIqUMrvueGNunwAMC/n4mAjM\nk1JWH7RtOXA1kHPowXWTdydw0T9ydwYGf5+J+M7tEGA6kECtsVEOfLz/YGNunxgMQT4+zgOW7P9F\nSumSUr4mpVxOrRXREIuB8/+BezMwOB4Onds/Sym/lVKWSSmrgLeAfoeMWYwxt48LQ5CPjw7ArmMc\nswPodBLuxcDgRHKkuT0Q2HbINmNuHyemf/sGTnOCqX11OxbK68YZGJzKHHZuCyE6AlOBiw/ZZczt\n48SwkI+PYiDgGMcEACUn4V4MDE4kDc5tIURz4GfgDinlskN2G3P7ODEE+fjYDLQ8xjFtgD9Pwr0Y\nGJxI6s1tIUQT4DfgSSnl/xoYY8zt48QQ5ONjHjDo4A1CCD8hhLXuV4sQwiqEEAcdMohaC8PA4FTG\nZ24LIWKAhcBbUsp3DzPGmNvHiRGHfBwIIcKpjdVssT88SAiRSm1Y0ME0lVKm1sVqrgUSjVhNg1OZ\nQ+e2EOJR4DGg8uDjpJSOuuONuX0CMAT5OBFCPAPkSSlfO4pjXwb2Simnnfw7MzA4Poy5/c9jCLKB\ngYHBKYLhQzYwMDA4RTAE2cDAwOAUwRBkAwMDg1MEQ5ANDAwMThGOKXU6xGqVjR2Ok3UvBmc52wsL\nC6SUEf/0dY15bXAyOZZ5fUyC3Njh4JuLL/h7d2VgcATaf/Rp2r9xXWNeG5xMjmVeGy4LAwMDg1ME\nQ5ANDAwMThEMQTYwMDA4RTAE2cDAwOAUwRBkAwMDg1MEQ5ANDAwMThEMQTYwMDA4RTB66p1ksioq\nWJmVjUkI+sfGEGaz/du3ZGBw3OhSsi4nh9TSMqL8/ekb0xiTYth3x4vxBE8Smq7z9Np1jJ03nw0B\nQazws3HBD3N4b/PWf/vWDAyOi/SyMi796Wee37mbXVGNmb4vi1E/zGFbQeG/fWunPYaFfJL4aNt2\nklQzi5ctIyCgtldkXl4eV19xBXF7kxnVLPGoz6VLyaa8fMpdLtqEhRJpt5+s2zYw+Es0XWfy4qVc\nM+kWrp0wgf3dyX755RcmP/ggP11yIf5m81Gfr9TpZHNePn4mlS6RkZhV9WTd+mmBIcgnAU3X+XLX\nHj776iuvGANERkby4NSpvD516lEL8trsHB5ZvAyLRxImTOzUqhneJJ7/G9AXy1k+eQ3+eZbvyyIw\nIpIJEyf6bB85ciRzvp/F3L3JjGvd6ojn0aXkjbXr+WrHLlqoNqqlTr7w8GDfXoxMbHqS7v7UxxDk\no0TTddbn5lJS46R1WCjxgYGHPbbU6cKt67RsWb8hdc8ePUgqKDiqa6aXlXHXrwu5Q4+gu/BHCEEV\nGq+m5/HU8pU8Maj/3/48Bgb7Ka6pYX1OLiZFoWd0FPa/sHCTiovp0bt3g/t69uvPnu+/O6prfrhp\nMyt2pvAO8YTotTK0W6/hqeUribTb6RrV6Ng/yBmAIchHwbqcHB5cuJQATRAhzGzVKukRHcUzQwY2\nOHkdFjMej4f8/HwiInyLPCWnpBB+kNX8V8zYuoPhMoAeyoFKZHahcqeM5IbUNKb07Ea4sUho8DfR\npeT1Nev4eudu2qp2nOg8pDu5u2d3Lmtd35gACLfZ+XPPngb3Je/eTbjF74jXdWsan23dznOyMSHi\ngAS1FFau0kP4eNMWuo48OwXZWNQ7AvvKK7hzwUImuUJ5RcbyoN6Ij0mAnHIeXrSswTEWVeW8Zom8\n/sorHNyz0OPx8NpLLzG66aFNqRtmS24+XanvL/YXKi1UG7sKi/7WZzIwAPhk81ZW7ErlXeKZqkfx\ntN6Y52UM09asZ1lGZoNjhifEs2HjRtZv2OCzPTk5mTlz5nDRUbjisioqsSGIEZZ6+7oKf7Yc5Rvk\nmYhhIR+Br7fvZAgBdFP8vdssQmGSjOCG7FQyy8uJbcDivatzJ25cuIgrL7uMiy67DJfLxcwZXxKu\naUwceHSuhiCrHwWlHhC+26WUFEg3gX5HtkYMDBrCo+t8tmUbj8kogg+yUuOEhQl6KB9t3MyAuNh6\n4+xmM8/168NN113HqPPPp0uPHuzavp2Z337L/d26EO3wrzfmUAIsFsp0DSc6fsLXJizEQ5ClvlCf\nLRiCfAS25+UzUrfVe5ewCoW2ip1dRcUNCnKgn4XPR5zDb2nprPjyS1QBtzVNoH9MY9SjjNe8qHUL\n3ilYQz/dgeWgibtSVqBaTLQPDzuej2ZwFlNYXY2u6SSI+v/Uuwh/3i0+fAnf/rExfHfBeXyflMSS\nzX8SZbXy5cjhf7mucjChNiudwsP5Kb+UMSLEu12XkpmihItbtzj2D3SGYAjyEQix2cijut52KSW5\n0k3IX1ipZlXlvMSmnJfYlDKniy+3beeNVevw6Dp942K4tmM7ov+iU8XwhCb8npzKfVn7uFALJAwT\n60Q1S5Ry3h4y3BtyZGBwrDgsFmqkTiUa/sI3WicPN8FH8AU38vdnUqcOAGwrKOTttRvYll9IsJ8f\nF7VpwZiWLf4yUeThAX2YOOdnMjQ3/XR/qtD5RSnDEmJnfNu2x/8BT1MMH/IRuKR1C+YoZVRJzWf7\nalmJyyTo3CjyiOcoqalh/Oy5bNuayvVVgdxeE0Llnjyu+GEue0tKDjtOEYLnhg7i9gG92BxlYnaI\nk/DW0Xw75iI6Rf7jnY4MziD8zWYGxMQwi2Kf7bqUfC2KueQordSFaelMmjefqIwq7nWGcXGpH9+v\n2cKdCxai6fphx8UHBvLdpRfTpkMC80Jd/BEhGdenM++POhc/09kbzmlYyEegb0xj+iXGc1dyOhfo\nAURiZpNSzTKlgreGnYNyFFbqh39uoVW1ym0i0usPbo6VCI/KCytW89755x52rCIE5yQ04ZyEAwuB\n2RUVvLp6HVty8wm2+nFxm5YMjI0xLGaDY+KBfr2Y+OM89rlz6K/540KyQCnHGmxnQod2Rxzv1nWe\nWPYHD8ko2ii10T4Jwo8u0p8H8vfxe1o6I5omHHZ8iNXKTV06cVOXTkDtW+fKrGxm7dhFUVU1rSPC\nuaJd66N2hZwJGIJ8BIQQPNyvN6uaNmH2zj1sra6hfVQs37VpRSP/Iy9gAPyclMxU6ofxjBBBfJaf\nQpnTRaBf/YWM7QWFrMrOxqwonNMknmiHgw25eUyZ/xuDZADnSxsFuHgx5w9+b9KYxwf2M0TZ4KiJ\ntNv5ZsxFzN6zlxVpGZgUlWtadGVEQpOjyphbn5NLOCbaCN/QS5MQjNICmbsrqUFBdmkaC9MyyCgv\nJybAwbD4eCyqwvMr17A4KZWL9ED6YmFzcS5X7UnixWGD6RPT+AR96lMbQ5CPAiEEfWIa/+1JUaVp\nBFB/glsQ+AmFGs1DIAcE2enRuO/3xWzLzaO39KdGSN5Zt5Er2rRm3t5kbtcj6KU4vNb2IBnAvWn7\nWJq5j0ENrIwbHAG3Cy0/CzXi7PijPxiHxcL4dm0Y367NMY+tcrsJbGBeAwSiUOV219u+vaCQyfN/\nI1Y30UyzsEx188Ifq7m1exd+T0rhVRmLQ6k9Z1f86arbeWjRUhZcNRbzWVC8yBDkf4CukRGsyqng\nPBHss30nNTgs5nrJHa+uWYczt4z3ZBNMdRbvtYRyz47dmBH0FL6WuVUoXKgH8sOOXYYg/w3MATak\npuHJyUAAalTcv31LpwUdIyLYrlVRgYbjkIXBVUoVPWITfLbVeDxM/uVX/uMJpZ8SULuCJWG1VsEr\nK9dyhQj1irH3GoqdCEyszsqmf2zMSf5E/z5n/r+cY8ClaWzKy2NLfj6ev1iQOFZu7NqZL5ViNuiV\n3kSRFOnkNZHHpG5dfPzQNR4Ps/ckcaMM84oxQJAwMUh3EKQrDbolIjBRWuM8Yfd8NuEJbYxl2iws\n02YhAU9OBp6cjH/7tk4oeVVVrMvJIaOs/ISdM9xu4/zEprwgcimUHgA8UjJPL2GDUs3lh9S0+DU1\njSbSUivGB9FLceBAIeIw9mEEJkqcZ8fcNizkOr7evpO31m8kXJjwSEmlIrmvT0/OOwGFTjpFRvDs\n0EE8u3wVLlcBFqFQqejc2q0zl7Rs7nNsQXU1dqESTv2U7D7CwQ96MdVSx3ZIQP1GpZp2jaKO+17P\ndizTZgHgunWMV5RNp7HFXOZ08djS5azMyiZetZKlOWkZGsJTQwYeVRLHkXiwX29eM6/jtl17aKRY\nKNTcJAQH8cHgkYTarD7HppaW0cJjpiEvRzRm1lJFP3zF2i0lm7Uq7gk7O2LuDUEGfkzay0frNvG0\njCa+LlB+l6eaZ5evItjP76h8x1JKlmfuY+7uJCqcLrrERHFpq5aEWGsnZf/YGOaOG8PekhLcuk7z\nkBCvT2xnYRHzk1NwahpdoxpRoXsoFR6ChO/XU4GGw2TmLfK4XUZiFQpSStbJShYp5XzddsgJfjJn\nL/uFOW7HLPa++TnAaefOkFIyZf5vRJa4+YgEbLqCG8msomJumPszP1w++qgqBpY5XXy/ew9rMvZh\nM5s5v1UzBsXFoQiBSVG4t3dPbunWhZTSUoL9/LyJUm5N47e0dDbn5hFoteKnqqSbPCDrX8OmqKym\nkuV6Of2EAyEETqnzniigU2QEicFBJ/rxnJKc9YIspWT6+k3cJiO8YgzQSti4Tg9l+vpNRxRkXUoe\nWrSUrZm5jNIDCEZlbUEKX2zZzgfnj6RZSK3vWAhB85ADmUlSSp5avpLfU9IYojuwofD2njT8TSY+\n8hRyJ5Fe90SN1PlSKeGmbp3YnJPH9ZmptFHs5Es3TpPgjSHDiAk4fJKJwd8jo80YLNPGAKef1fxn\nfj7ZJWVMlfFet5hZCMYRylZ3FgtS07jgCLUn0svKuH7uL7T2WOir2ymnhtdzVjG70R5eOmeIN/nD\n32ymfXi4d1x2RQU3zP2FELegq2YlU2gsphy3lGwVVbQXB2q07JDVbBU1vHrOUJ5c9gczXCVECTM7\ntSp6REXx3OABJ+HpnJqc9YJc5nKRX11NexFdb19P4eD1wmT2FBfT2OE4bOHtX5JT2JWZx8syBr+6\nCdqXAH7RSnh40VK+GnNRg+NmJ+1lQ8o+psl47HXjLtNDmCby2WCq5m65j36aHSeShUoFfeJjuLJt\nG8a3a0t2RQU7CosI9vOjc6PIo4qHNjg+9lvNkV9NJXNpbeeXU9lq3pSbT3dpb3Bu9NBsLEvNoH14\nGPGBgYedP48vWcEol4MxSoh3xWmYHsTUnCxm7d7D2MPUPr7/9yUMddq4TIR6x42WwdwrMnmSbHqL\nABI1MymKmzWikueHDKR342jmjh3Dn/n5FFXX0DI0pMGyBGcyZ70g+6kqGpIqdPwPcW6V4AEpuXfd\nRvILC7mgWSL3dOlcL5No1vZdjNGDvGK8n+EE8VVFGimlpTQNqv/K9fWW7VylB3vFGGoTQa6RYSzX\n0rh/YG82ZudiVRVeaNIDh8VMdkUlMQEOoh2Ov0y7Njh55F3xBJYran8+la1mh8VMiaI36CIolB6W\n5uawaekKVF3nrk4dGJ7gW4Uwp7KSnUVF/Fck+Gw3C8FlejDfbd/doCDvLSkho7SMx/A9X7gwc6ke\nTHJjK10aNyK9pIzOQYFMjmmMJiXlLhcBFgudI4+c/XqmclYI8u6iYpZn7gMBg+PiaBoUyMa8fHIq\nK2kaFMjAmBh+zCrhSnwXDmZSwuVjx/LEM09TUFDAo488woMrV/HKgH4+xxXX1NDokJA2AFUIIhUz\nxdU19QRZSsm+ykriqC/UQUIlUKi0CQ1lREITPti0mTsW/E6QMFGme4gLDOD/BvalzVmy0HEq828u\nArp1ncXpGSSXlBBptzOiaQJSSlZn56BLSbdGjXhJX0s2LqIPKnVZJjUWmqv5auZMWrVqxdq1a7nj\nttuwqKpP2GRJjZNQxYxZ1g/GaiTMFDmL620HyCgrp7GwIBr4R9BU+LGqspJr27cjr6qKx5es4L0N\nm4hQLeRpLkYlNuX+Pj2xmk5vafq7UTqn96c+Ah5dZ+qqNazKy2fU+eejaRrXzvkRk6ISFBhIy5Yt\neX3lGoIVwSZTJUWaxiDpwCPhJ0rJiw7m6fvuBSA8PJxX33iDQf37s6e4mBYH+YLbRISzOa2Mlviu\nKpdJjXTNSdODFiSklMzctZsPN26mwu1mMmkMkAFMUMK9xbqLpYdyqRFht/Hehk38ti2JF2QMjbGg\nIVlcWsZN8xbw9egLaWxYyacEDS4CqupJSzbZW1LC5EVLiY6Pp1vv3izduZPnvp2FEILO7dtjtlhY\n9/N8ujaO4sHsLC7Vg2gtbKRJJ1+byhl79Xhat24NQM+ePXnq2Wd58/HHfQQ5PjCAAt1NIR7CDllg\n/lNW1TMISmpqeGX1On5OSUXTdf5DCqOVEC4Qwd61kN3SSUJwEE6PxnVzfqZvjR+3k4BVVyjFw3sp\nuTxQvYTXRww7Kc/tZHKoCH/a/8PaH+YdfaTWGS3IH27dRkFAIItmfofVaqWqqopff/+dKVOmMHbs\nWIQQaJrGtLffZu6MGSQ0juZ/qekU1jjpMKAf77z4ok9PPIvFwrChQ1mbme4jyFd3aMeN6b/QQdpo\nVZdGWiN13hb5jExI8EZaAHywaTM/btnFHTKc1qqVMnRm6oU8qGXwihqPHwofiAJGNa39Ev+3bQev\nyVgiRa3/WhWCYSKIdN3N51u3c3/vnv/EozQ4SvYvAu4X5pORbKLpOlOWLGfyffcxduxYAObPn8+O\nvXv532efER8fD0BBQQH/mTiRka0CSSuvYHFxKenl5bzy+tuMGD7c55yDBw/mtttuo9rjwVZnndrN\nZsa0bMGbezK5X0Zir0v+SJY1fKOU8HrnA6JZ7fFw3ZyfaVWlMl00IcRkYresYZqWS4nQuEYNJ0u6\nmK2U8lb7nvySkkq4SzBeHBD1IGHiLtmI/2SnklRc7LMAfipzsBB7RfhvcsYKspSSr3cn8cmMGVjr\nBHHOnDm0b9+ecePGeY9TVZXbbr+dn+fOoUtUIyZ368Kza9cT1bGjjxjvp6y0tN7rVOuwUB4f1J9H\nl64gRlgIRmWzVsnA2Fge7Hug/1iFy8VHm7fyBnFE1AlsECo3qJHs0/bxtJ5FnqrTLDyUe/v0ZEdB\nIY0VC5F6/cXE3vjz7K4k2kaEMyqxqXdRJruigp/2plDqdNIhIpwhTeLPipTTU42TGZ2xMiubgPAw\nrxgDvP/BBzzy8MNeMYbat7onn3uOWyZOZP7oixBAry+/pmuXLvXOWVVVhQCfZCSAO3t15ymXmxtS\nU+mo+FOBRpru5MF+vX18vfOSUwiqkdxMuNcabimsPKrGcKOWQhYeNlLFvT170CEinK+3bqe3Vr/O\nuFkIOmhWpi5dwWMD+tEytFaUpZSsyc5h5b4szIrCiMQEH6Pon+ZgEY4d2J6n9btOyHnPWEGu9ngo\nra6mVasDiw7bt2+nb79+9Y4VQtBvwEB2bt9K35jGnBsfxyMzZjBx4kTs9gPhOVlZWSxbsYL/XnJh\nvXMMbRJP/6tiWJ2VTYXbzf9FRNQLQ1ufm0dz1UZEAwI7XATxla2cl4cMoENE7aS2mU1USA0pZb3s\nvAqpEagLPlqxnuVpGTw7ZCAztu/krXUbGCAchOkm3iWZR8UfjGzRnBs7tjXcG/8SDfmZj8edkVZW\nRqfOvqK6Y8cO+vbtW+/YDu3bU+l2UeZ0Emy1MrJZIp98/DH33nefz3FffP45Q5om1CsqZFYUHh/U\nj1u6d2Zjbh42k4nejaPrGSVLk9MYrPkjFN95GiJMtFXtRDSN5KfuXQmrKxNgNZupoOFs2BqpE1Dk\n5Pq5P/PskIF0i2rErb/8SkFJBX01O9XoTNi8jSC7Pzd3ac95iYn/SMlOLT8LqR0ow/vF+T/gKS3k\nMB/jb3HGCrLVZMLfz4+UlBSa1r3+h4aGkpnRsLM9Iy2NxDpLuktkBN2Cgxg/diy33303zZs3Z/36\n9bz20ovc0rG9jwviYCyq2mDbm/2oQuCRDX97GpJohz8dD6pz3Co0FMWssr6miu4H1a/QpWSOLGGU\nEswwGcg9mfv4fNsOPtzwJ68RRyRmUGAsoXyjFzNrbzazk1J5ZXAfBsefWpEAZxOH+pn/rtXc2OHP\ngu3bfbaFhISQmZnpY4AAFBYW4vFo3ma8t3ZoxzVff01lRQWXjxuHqqp8/913/DBzJp/+hd82yt//\nL7NWVUVBw9PwPlVhQFysV4wBRjVP5IHkNC6UwVgPyjrNkS62UM0HSlNSpZNHl65gQFwsQcUupspY\nlDrBv1KGcm91Dk9tSGXa5p18d9G5BJyk1k8HW8Orp8xn54a6foOlhSf8WuLgJpxHol14uPzm4gtO\n+E2cLF7fuInU4BDenPYOZrOZtLQ0xlx6KT/Onk1MzIFCJdu2beOaK69k+rAhbMzLRxWCgXExrM3J\nZVZqGrmVVSQGBzG+eTOfAie5lZVsLywkwGKhS2Rkg62ZssoreH/TZpIKirBaTKzPzeMNtQmxB616\nSyl5UM9kn0ljaJN4ruvUnoS6qIzVWdnc89siLtGD6CX8KZYa38tiqqTOk2oMFqHwk17Cz9YqhtZY\nGaOE+lxfk5Jr2UdE+9tJ3fo6f1w15i/bvP+btP/o0/VSyu7/+HU7dpHfzln8T18WqLWa93O0wuzW\ndc77/kemPvMM555bW0v71ddeIykpiTffeAPloHn41BNPULRmNcNjY0gtLSPK358OEWF8tXsPi/Zl\noyMZGB3NhDatvanUUkq2FhSQX1VN85DgBusRSylZnJ7BzO27qHC6KPK4UMtcvKLG+cQ050k3k7RU\nmgQEMLp1S8a1aY2fSUVKyaNLV7A1LZuxejBxwsI2Wc0MvZDLlVBGKbVRS3eSQZbu4j3RxKdDNcB6\nvZLXbSaUoOYksouPRp64hcCDRbj0ue/4Ye7hG0kciVcfanrU8/qMFmSnR+PuZStId7u5aMwYdF3n\ny88/x6Pr3HD99bRs1YqN69fz9VczaO3vYFdBEb2Fgyo01mgVRPhZSQgO4ryWzRiZ2NTri3VpGk+t\nXc/vael07tCBvPx8SosKebJXT3o1jia3spIt+QUsTMtgfnIKQ0UAvXGQj5v/6UUoQmGyCKersJOP\nhy/1QlKkk3uURqyikp9EGe+fP5J2dT3zdhcV8+zylewuKCICE4OUQEaJIG+DyOV6OR8rRUzRI+ik\n1O9S/aAowtP5bjL2fMbYOJW7e/zjmndUnI2CvJ9jTTbZVlDI5EVL6NK1K9379GHntm38unAhLRIT\nueraazGbzcyeOZOkbVuRNS4CPIKWuh/bqSZHdxEXEECbiDCuaN/GJ1piV1ER//1jNR6zmYQmTdi8\nbRsdw8N4qndPAi0WNucXkFFezgd/bqGopIKxSgjRWFinV/AbFXRWArhGBBGFmT9lFe/q+QwQDroK\nf76hCELsfHzBSMyqii4lP+1N5uUVaxCaTqLw4yIR4jOHHxJZZOlOPhH1rfNi6eFGcmg36EM2Lr6W\ntVePxe84w+VO5ALdfo5FkE9bl0VBVTULUlOpcLvpHBlJj6hG9fysfiaVtwYPYH1uLkuXLEIAr/fv\ng0VRmPXbr2yYO5cm/jbOj41jW1Im79OEfOnmYS2TgcJBX1cAFXkanxduZM6uJN4eeQ5mVeXZdRso\ni4hkyRdfehf+li9fzp233UbnsEhWZuVgtUfhLEvjHtGIvgdVtxosArlRZvCWn05Z9V6sCM4TQUxW\na1exm2GjkW7m/t8W89MVlwJ4i8GMnTWb54jzecUDWKNWERXgYE9JDZ3wFWSPlKTJKprZY7D5x7Mq\ne9NJ+DYMjpf9ySZSStZedQ7rklKxqSrnduxEVAONENqFhzHvkguZl5xC8ry5JFqtfD9qJBvy8vjt\n/ffRkPQPC2NPjYvRrgDOFUHMkkWU6G7GiRASKqzsrizl5rT53NO7Jxe3bE6p08mkhUu4/5FHGD16\ndG09CaeTZ556ituXLKWgwkmZx4xbVuNXU8U0tYm37GY3xZ/uuj/PyAL+VDRqPOXEYmGCEs7Auvnf\nTtq4ryiD/23bwfUd26MIwYXNm5FeWkb6tgxuwbctWYXU2KtXI4SgSHoIPcRCTpI1+Nsb4WeLQtc9\nbMjN+1s1y7WcDJ/cmRMlxH+H01KQZ+zcxZubNnPOsGGEN2rEMwsW4Ni8lbcHDyDokKajQgi6R0XR\nPcq3Elrburx7TdcZ8sXXPC2jsQqFtz15XKGEeV+ZAPrrATxWmM23u3ZzbtMEfklJYcln//OJwujf\nvz9jr7qKr77+iY5DvmHn6psJ97PRx+27kGYVCtfIYL5xNEHXPTzpsdNM+PqkB4tA3qnK8wn9iQlw\nMCAullcy85gsIwgSKpqU/CJL2WJ28lzPXtz/22L6SYdPEsBXlGBxNMHmiKW0cCMJob61lw1OHSor\nyply89VkZxcxfPg55BUUcOmceUxoEsNNfeov2NnNZi5r1dJn2/kBDs6vq0+xOD2DQI/gXBFElnQx\nUy/mDTWe8LoIn87Y6SMd3LdyNYPiY/lxbzJ9+vVjzJgDbhQ/Pz+mPvYYvXr3JiDyYuIjepOy7m4u\nIaheDeRuioMYWYHW/Co8STOYdkiXHFUILhUhfLR5O9d3bO/dPq5Nay7dsYvmnlKGikBUISiWHl4X\n+ZyfmIgiBB8l53KXjEStM7oqpMaHopzQ5hMpLdyI2RxwTOUDDl2g+zdF+GBOO0Fel5PDh7v2MGfe\nPGJjaxfQ7n/gAZ587DEeXb1iLdWZAAAgAElEQVSK1wb2P6bzlbtcuDSdOMWPPOkmAxcjhG/2nCoE\nl+pBzNixh8TgYNq2bEVQA6nQQ4cO5fsflwM61dWFtLIGIDz1J0kkZnRnCZruwp/6oXUmIfBD4YWV\na5g+6kC/vccH9uf5lau5OTmFWMWPPM1FfHAgHw0ZSdOgIG7r2ZU7V62lm24nGjOrVZ1iP3+ad3uU\n1B3TkbqTpgEhLEhJpX9szCnrSz5beebxB4iJjuDTj6aj1kU73HXffVw5bizNN29kYGT4MUVnJJeW\n0kq3gIDFspwhIsArxvuJExa6C38WpKaxrbSMIZeNrXceVVXp168/O9LCKSvaRERICJF5urdjzcFE\nSkG6u7RWrBvwhvqjUuFysrOwiNZhtesd4XYb00eNYOri5XxRkUa4YiZTc3JJ82bc07snHl3njtLf\nuSEvlUE4qFYUlshKQuNHERjaiW2r70NqVaSWlWFWVbpERhy2ldnBLglvlMQpxGknyF8npXDzbbd5\nxRhqreB7H3iAAX37klNZ2eAr3uHwt1gQAvKlmwp0glHrxWIChAkzpS4nQX4WsnNz2LNnDx+//wEb\n16wlMDCQMeOvxGKxoJqDURQTSMnOmjKqZVC92sVrZAU1Tif+QS1ZXpTGZcI3nnK3rMEPwbqcHDbk\n5NI1qtbS8DOpTB3Qlzt6diOltJRQq9VnweXy1q0YEh/PLb8uZFlRKRZzGI6gFmxeMRmpe5DSzR8V\n0SzbVsYjK9byYM+ujG7Z7KiflcHJo6SkmN9+mcviRQu9YgwQGRnJlDvv4tsf53Hd7eOPKdkkyt+f\nJUptuctyqdFINPwPOExXKHW6CDKb2JeZybx58/jqk0/Jz8ujVZu2XDfpJjL35WC29EBKN35BQWwq\nzaf/IR2aPFKyRS/HUZ1PmlZBsQiutxD3hyynDVbumv87c6+41LsQ3io0lK/HXERKaSmlTieJQcHe\nPpNmReG9USNYkZnFlIVLcOuSgOC2VFRlsW/JBAQCf/9oPk63U7FlE6EmjWnn9PcWJjrsAt0pJsZw\nGgpyWkUFN3XqVG+73W4nsUkTMsvLjyjIFS4XxTVOIu12/EwqFzZvxmd7c7hVD6cID3nS7c2M288G\nWUmHiAhah4aiVddw+cWXcJEM5FbdRmFWEV8++QIZ0kVkm7tQVCsBwc2xqYU8VpjL/2mR3tY06/VK\nFstyznMFMqtgPTOAaEz0EQ4UIdgra3hJy+ZKJYwtsoopvy5k0fhxPskdQX5+hy3AEm638c1Fo/hs\n2w7e27ydkpxlCMVMYGh7WnZ/ClWtdY9Ulafx/Jo7iA900C2qfgNWg3+WnKxMoqOjCQ6uXxOlY6dO\nvPr6G0dMNpFSkldVhQQa2e0MjY/juRWr2KRX0lJYWSjLuATff/5SSjYqNZwXEY7dZGLSu+8Rrli4\nzOlPjDCzNXsj1y8ZT6Wu0334U1RX57BtxbukCWisSUYrIQghcEmd6Xo+zfCD7D8olW6mso+H1Wii\nhAW3lCyQpaySFbygxDGpJpU3123kzp7dfO6noSJcUGt09Y+LYf7ll/DMqo0sSt+OJnUUxUJi+zuJ\niB2BEAIpJTkp33DD/K/5sV8nr4vD65I4jmiJf4LTTpAb+9vZuXMnHTt29NnudDpJzcggum3rw44t\nqq7hhQ2bWJyeTpDDQWVVFZe1bMHk7l24veA3HirLpoVu5UUtm0fUxt4C8dtlNTOVEt7p3BMJaNVO\n7vGE013xBwHNBbRxWblbzyB95zvk7vwIf3cFrXSFNOFmgp5MU92CC6hE5yE1mvbCjkvCPK2Uj/R8\n3iUPK7UFxK9QQhmhBLFXq8Gm1foCD63E9VcIIZjQvi3XtmtDQXU1o76bS7NOD3rFGMAe0IRGza/j\n/a2zDUE+BYhsFE1OTg4VFRU4Dkng2bVzJzGx8T7bDk02+SO/kDfTssiprkEIQYTdxl0dO/DK8CHc\nuWARnbGR6nEySy/iYhGCKgRuKZlBEVZ/P3pFR7E+Nxc/t8bzIhJbnQHQDCtRHhPP6FlsX/Nfaop3\n0E74EaTDN7KIb7UiWmFlD07aChsPq40xI7heTyFTurhbSycQE2V4SBB+PK3GEoEZFcHMnbuY3K3z\nUXW43k+E3c6rQ/vh0fvw7c7dfJQqiYw74NYTQhCdOI5dWfPZGelgXas3/u5X8q9w2gny5U0TePrN\nNxk6bBjhB4XrvDNtGm3CQg9bpN2lady4cBH9Rp7HshkzCAwMJDMzk/978EFe3LCJjy88jxX7sliU\nms72/AJuLEujlWqnUmoUKzpPDOxP+/BwtuQXYNEl3eoKbEsp+UoWMVsvJg4LppoKSijlahHJILXW\nnbCaCl7Vc7hbiaKb8CcfD6v1ChKwEGAxYXFJnlJicQtJDJa6PxadlbKCntKfXUVFxyTI+xFCUFhd\ng90WhsVavzJcUFhXdqd8esznNTjxhIaF03fAEF597XUeefghrw+0tLSUN958i5tuf6DBcZZps1i3\negVTb72WZ555hiFDarvGLFm6lP/ecw8v9O3F3LGjmb1nL7a8fObn5jHHXUaCYmWvVk3LsFCmDRuG\nEIKfdu/lAg642Iqlh5f1HNKkk1bCRnrhnzTBwh1qGBGqGbeUvKxnUyl1XlLjaISZndRQLjW6CQe7\nrR5GVtvopPrjQPH6r1frFcRjoRRJfnX138ogNSkKSaUV+IUObnC/NbwX07Jj6dlwueZTltNOkPvF\nxnBhURGjRozgktGjCW/UiN9//pmS7GymDxt82HELUtMIjY3joUce8U722NhYpk2fzoC+fdnXrg0D\n42IZWJdpV1xTw5b8AmwmE10aRXo7I1S53QQJk/ccP8oSVukVvKU28U64JFnDE9o+QnQTHRU7vRQH\nXaWddOnkd1nGFllFK2EjU7qodHnwmEx8rRVxnQhHFYI86Wa6nkdbYaNEkWzOzWfi7J8Islq5uHUL\nBsfHHfWKcqjNSlVNMbrmRFF9I1CqKzN9sqcM/l3+78mXuWnCGC697HLOHTGcktJSfvjhB0ZddDkj\nz7/ksOOmvf4cDz30EMOGHUiMGDJ4MI88/jjvvPIyH58zlAkd2nn37SoqIruikiZBgT4ugkqXm9i6\nmuC6lDyq7aOH8OcxNQaTEGhSMksW839aJm+pCZiFYLLSiP9oKWRIF1P1ffghCMfEVqoJ0C38j0JC\npZm+woGUkg2yirf1XP4jInhdy+OFFasprq6hdWQYV7Zr402IOhrCtRo8zrQG9zmrs7A7Oh/1uU4V\nTjtBBrilYwdGNoln3p+byNY8XBUWxpCunf6yiM6qvALOHz++3uqrzWZj6ODBrM7KIT4wkL3FJSzJ\nrE2NHBQb622/tJ/WYaGkaDWU4CEAlVl6MY+qMT6r182FlWuVcL6TRXSsiwtuL+z8oBfTTtj4WE3E\nr64f3jJZwZueHFZQzjKtnABUqtBphh+J+PGLVsp5+Sa66XYKcfN63ip+jU3l6cEDjkqUI+122keE\nk5PyDY2bX+PdrmtO8vd+wpS2x255nyjOtM7Ox0tIaBhf/bCQJQvns3bVCmy2QKZ/9j0tW7c77BhN\n01i7+g8+ev/devtGnnsu9959N9VZaSjAcrcgqbiECLuNEQlNcBySatw1JopF2TsYJoPYIKtQgKuV\nMO/fjCoEl4tQ1uuVrJIV9BcBBAiVKMy8pOdwrxJFD+GPEIJqqfOKK5tdeHhTz+VNcjEBfih0w85M\nWUyQUGmV7SIaK1tKCxif9BPPDx3kkw17KAfPmckP38L0m54gsuk4bI4DC5zlJTspLdhIi/ZvHt2D\nP4U4LQUZap3/k7vUX9w7HGZFUFlZ2eC+yspKTIpg6qo1LMvK5rxRo5DADfN+YmDjaB7t2cO7Ghzk\n58eYli14dnca/5Hh6EgShV+9c3YT/nyk53t/T5FOivFwq9IIc90EF0IwUASwSVYShokVspxs3ASh\nUoSbJGq4V4mmDw5vVawBegD3Ze5jYVo65xylG+Ppft25et53JJdux9FoCJq7nOKM7+kWZuXi5ic3\nykKr+wM6XD6oT/znMdSNPVNRVZWhw0cxdPioozpeCIGqmqipqcHvkBj8mpoaFJOZ6sm3cfU1NxAe\nE0OPvv1YtmcPL8+azbP9+njfCAEubN6MDzduZpariCqp01XYGwwf66b4s1vW0J8A3FInCzf9cdBT\nOeB6sAmFu5VortOSuVtE8qzMAQRWJH9SjQvJu0qCN5a5M3a6SjsPLVrKb1eNrdd8tcEMun0weJSH\npT9PITLufGwBiVSW7iB/36+MvPxF/KynX/un01aQj5URsTE8/8UXXHP11T4TNysrixUrV6I1imRb\ndQ3/ueUWLh0zhpCQEO7/738ZP24ct/++iDahoXSNiqRP48bc06s7z2ka9+3ejaDWGjg0tK0Qj7cl\n1D7pYikVtBI2rxgfTFfhzzy9hGI0LhDB9FIcbNWrWSBL6S18I0b8hMLFWiCzd+45akFu7HAwZ/Qo\nftyzlxXZ32A3q1zcpzV9Gzc+bLzmsXJottPBhD33Im8sa3HKxXyeCSiKwvCRF/DFl19y6y23+Oz7\ncsYM+g0YzOjrbqFdx05cc/XVDBo0CFVVWb9+PddNmMDoxAQi7f6MaNqE2IAAPrrgPG75eQFZVVX0\non4aPkCBdBNW90Y4mxL8FIWBDcTT24RCa6zMl+VEYmacEkpjYeEjLZ8+wlEvsaSdsBEjLPyxL4vB\n8XFHVeKyfY9xNG7Snc1rZlBavIbomKacO2YOQSGHL/J1KnPWCHLvxtEk7k3humuuYcrdd9OsWTPW\nrFnDc08/jcmjkZaWyxDhz+rX3uftV1/j6ReeJy05mZSkZHp7/CjLrOKF7XuwOWy8e95wHu7fh+AA\nf/63aQs/yGKuPKjQtpSSr/RCEoSF6VoeS5RKxrRuwYa9GQ2W6suXbtJwcacSRa86K6Na6MRIS4OC\nGYaZ0prqY/r8/mYzV7ZtzZVtj+25HcpfuRkOm+00F8AQ45PFbXc/zLVjz6OqqopLL70UAcz6/ns+\n++RTNKeLPrqNyOXbeGn9A7wYHsQb773LM48+RohH4N6Rxy5F8uHGP7mmQzsmde3MN2MuYuxPv7C6\nqJhs6dv+KV+6WSTLuZxQHpNZZFt02gWGk59fv9KblJJcPFSg8Z6a4C1wHyZM3nrghxKuKxTnZeOp\nu6R3Tv1FicvQyGYMvuCRv/XsTjXOeEFOKS3l5+RUqjSNUfEx5FRW8cS991JQWUHT4BBkWTnj9WBG\nqnW+YjckS38euvc+bELlbT2GULX2MV2lSz4oK+CxpSt4bcQwJnfqiE0I3l63kRzpYqAIxInOPL2U\nXNxIJKk2nRnnX0C0vz/D96awRVbR4aAW6JVSY44swYKg50HWcKKwsocaqqTmncj7WS8q6XBIKviJ\n5nDCm3jb1WyMHX1c1a8Mjh9N01iycD7r1/yB3d/B869OZ8HPs7n6mmuRUtKuQxcsmuQFGUMjpVb8\nrqqRfJFVysSxV9DDaeJJGYtQa//hXyFD+O+WHbSNCGdgXCxfjTqX8XPmcXdpOpcpobTASrKsYZYs\npglmvpGFjG7Zimndu7ImO4eXFq9koAzwqbOyRlZSisZoEewzh1sIKxtkZT2r2i0lm6ki4aJP+dTu\nW9fibOGMFuS3/tzMt0nJXHzJJYSHh/PJzz/jLCrk/SGDCbVZWZOdw1O/LeXcQxqNJgo/hnjsZOtO\nQg+qHiWE4GoZxvXZqeRXVRFht3NR8+a8s24jW2U1WdKNTSj0Ef6USY2ZFGOp1vHoErOq8syQgdz/\n+2KG6g46YydbupgtS2iOH+VoPtZwmDDRWzh4RcvhbjUae90i4DpZyQLK+K7dkBPyjA4nvIftgrAJ\n2GSI8b9JUWEBN147GotJZfg5wygqzuO+KTcw9qqJLFxZWyf5/kkTGOP298nOE0IwTg/ix7K9nKPE\n+hSTDxEmxuohfLl+I33NEkdUHBc0S+TdDZv4Qy9nNRVECQs3K5HM0otxoLI1Ow+72cyguFh+i4/m\n3vR9XKgHEo6JDbI2Aaoplnrp2sNFEJP1VBZppQxWAhFC1LY803MJatKN4LNUjOEMFuTF6RnMz83n\nl19/JTS0Nmd+0qRJPP/sszyxdAmvDexPRnk5zfBr0C3QCj/2CWe97Tah0FjxI6uikgi7ndl7kugr\nHERiZq4swSIVtlFNGCaa40cSTt5cv4FXhw2hb0xjvrzkAr7Ysp1pyak4PR66CTuh0sRsSiiXGgEH\nWRK3KpHcq2VwjbaXRFMIWbIKj+4myt/O76npjGnV4ohFubX8LACfQioHU/rcd+zMchwour2fE9gF\nweDE8uiDU+jftzcP3H+/d+7eMmkSY6+4kg6duzN42EgykpPoT/25YRaCOGGhhPrzoZmw8oOlVjw9\nORl8v30HdytR/CrL2C6ryZFuVshymmBBBfaUFLM1P5/2ERE8Oag/SzIy+WLzNrbn5xCIifNFEJtF\nNctlOUM4kOIfJFSmKjE8rGfwiSwlTNhI08owAzFmG6m7l9CkxcATtr5xOnFGCrKUkunbdhDfogWv\nvfYa/fr3Z9jQoZhMJqbceScDvvmGvKoq4gIcJONssEXSLmpQGlilqpE6WZrTW8w7o7SMGCzYhEK4\nNBGAyj1KHGF1r4mZ0sUDGZlsysujc2Qk8YGBPNivN//t24t1ObksTE1DSuhXbueF7BzuI4pAoXqt\n4X24ie/8CFm7PyGyxsUlSjB+1QrLNu7hy63b+fSiUd5U8b9yMwA8uqkBq3puCWBYvKcLqSlJ/LFs\nEaGB5/PSSy9xySWX0KJFC0JDQ5l8yyS++vwDBg8bSVzTZuzdu4r2hyzMuaUkQ7oIFvWz4/ZKJzHx\nzbxZgPuahmAXCi3wYyuVjCGC89Qgb8jm93oxDyxcypyxY1CEYHB8HIPj46hyu5mTlMzuwkJ6+/nx\n/c7dzNNKOFcEoQpBldT5XhZjsoQS1Goi6dveYqgSQC/8yd+7ne/TppDS/lwGX/r8WSfKZ5wge3Sd\nB/9YRZ6UDO/RA5vNxvvTp/Pee+/xyccfExgYSGx0NDkVlXSPikKzqPxaU8aIg9wWqdLJb1Rgt5go\n1jzeAilSSr4QRfRo1IjIul572ZWVrJdF9MNBK2FjrazkXZnPfTIKi1CIFRbGyhA+37yNloNCWJ2d\nTUlNreUdbPXj9u5dsZvNuDWN51eu4cakvcRhIV9zU46Gf0gb8jPm0by6mEeVaG/scW8cfOEs4LmF\ni3mpx4E08gYrWBklkM8IkpN2c91VF9KtWzc6dOhAVlYWV19zDddNnMikSZNo27Yt707/AIArbryV\nu5YupI/L36cuyzcUY/HzY6GsIlE78HZYIj18a63ioZtvB2rdInbVzLuePLoKfzoJf2bIQgKkyjBR\n62YYrYSwxLWP1VnZxAQ42F5QRJnLid1kpnVYKJcG20AIRoX488j6bcxwlhOBiVStAiFMBEcPJWf7\nO9xHmHcxG2CQpnPn1vmkdhhF01aD/7kHfApwxgny1zt3UejvYOH3P3jD2yZOmMD/TZ3Ks88+y8MP\nP0xGVhaNu3dBEYI3zj2Hm+bNZ35NKb0Uf7L8BGtkJc8++yKLFy7i5p/mMVD3J0yYWKVU4eew8t6g\n2hKfv6amkZ5fzPtqU69ou6XkRT2bT/UCblRrCwC1EzbmZOVwzpdfEystKBJ2SydWvyCcziXYVAVV\nCPpGhDKtZ0fmZuayIDuPaCyUlezGLXX6KuH1EkFGixCuLcpgevfXDsRclhYidZ2C3J3oukZ4VGtU\n1SizebojpeS+O27gjttv48orr/RunzBhApdedhk9evYkee9e4pvUxnJ37d6bG+76L7e/8AT9pJ1I\nqbLBpuMKDeD9l6dx1623sb4onUHSQYkiWW6qZvx1kxk8bCQA/711IsMI4AY11CvaGdLFg1oGCcJC\nM2FFCEFL3cIrq9aSVVFBc92PXNwUCpAmC36aE49qwW4LolXfSbSO7cT6BS9jytlBqGolP30uVgTt\nD6kHbhcKYzxWfl/9RT1Brqkuo6QgGat/KMGhvvU9zgSOTZDdrlM+u2pm0l6eeONNn1hjIQR33nln\nbWqpptEnPJTg0gI8pRAP/DioFxeuWEvxoN707tKFx0eO5NnnnmPTpk3cetcdZO7bR3JyMrlbtnBN\neBDfrl+LU9NZkpXPNXoIIcqBx2gWgpuUCG7T0rhGhmMVCkv0cmqE5AURR6xS69crlR4edu7DX1i5\nnUgsUrA4v4JbC7YRoVh4jljihAVdkaySFbyl59Ja2GhyUBKKXahYhRlXTblXkPfu+JXl3z+C2VWD\nSSiUCUmvc++jQ6/x/8wXYHBS2LZlE1UV5YwbN85ne2RkJNdNnMinn3zCn1u28OjTr3v3XXvTbcQ2\nbcbD997CuMtHM6VbN2JjY7l50iTadu5I+/bt2bN7N3uTk/ErLiE6Pp43Xn4KTdPYumk9H8sYH5dB\nnLBwsRLCT3oJU9TaKJ91egUdyu08LRKwqLURFhv0Sp52ZXG5CGMQgeRVuPli8YcskDUMw8HVxGHT\nFSqUcKbr+Twts3laxvpcKwIzzvIC7++ax8XyOY+zfeMsokx2irQagsITGTruNUIjz5wSssckyKLJ\nAf/Sv4mzpob5835g6aIFKIrCsHMvZNiI8zGZTOR1SqB58+b1xoSHhaGqKjv2pvD+j8uwBB1IibYA\nU779go+nv8bQc87hggsvBODXBQsICQlh/YYNvPDCC3jcHj5JSmewEogDhVS9mqZKBL/rZWRIJ2HC\nxCARSLgw449KMR4CpcpCUc7NRPg0Ng0SJu5Vo5iqZRKNGVUIRstgZnkKuV+NJq7uWEUI+ooAMqSL\n7/Vi7lQPhLulSie6yYx/QK0lnpmymsVf3cl/tVA6iCCEECRLJ0/Nex6T2U6brqNPxtdhcALZl5nO\nd199RnLSLqIax3DpuGtp0aotOdn7aNasmU8D0/20aNGCt6dN44ZJd9Fv4FCffUPOGUmLVm3w6Dor\nV61i5syZXHjBBTz99NPous77H3zAqjVrqCwq5r0H7qW/cLBb9RClSQoVjaV6CdVSp62w0lX40wor\n62VtxuvvWikVUmeyEonloHC3roo/58kgStAIESZCMDFS81CkV/EfNdgrvA6hcqfSiFu0VLZTQzsO\n1FVZr7gISzjQhm7xd/dj3rGMD2Rjgj0mPFLyS14+X753OVfd8zs2u29Z0dOVwxd/OEUpLS1h/GUj\n+HHm/xg8oDd9e3bhs/df5z9XX0x1dRVNm7Vgw4YN9calp6cjJXz29c8EBtWvOXvJZVcxaNgoHnrw\nIfr26cPV48cTEhLC5s2bmXTzzbRt04Zos40P1aZMViKZoIQTicq9egbLZTl2obBb1nCTlsJCvYxS\nPPxKOVMsebiEoJOon/WUIPyQQHFd+/QsXAShEt9AKnZfJYDNssr7e7H08KpaSudBN6PUxUlvWPAK\nN2gBdFQOpLwmCj/u0oNY/+tLHEtDW4N/nqWLfmXsRYOpqSjggvPOIdBu4obxF/PV/z4gsVlLtm7d\ngtvtrjdu3bp1DDlnFDdMuqPePiEEr7/7OYsWL2Xt2rWoqsqtt94KwHPPPcdvv/1GfGQjLlZDeUdt\nwtVqONfLEPZJF3dpaf/P3nmHR1F1cfidbclueu+FJPTQe+/S0dAEpCkdC0hTQUBQqdIVkKIgAkHp\nvUUg9A7SQ03vyaZvts33RyAQNp+iiATZ93l8fJi5M3N35uQ3d8499xzSRD0KQWCdMZVRhihuiHkY\nEFkgT+d7eQYVVQ6FxXafpJpgxV0eRyldFzU0ldiYTNJJBIHagjWXjI/TGpw1ZhMmyaVy/X4AZKnj\nuHNtH58YHLF/6BqUCQIdBDuqG6RcOxP6N+52yeSV8yHPm/kFVSsHM3XKlMKH261bNz78aATLv5tL\n7/5DmT1nBlWrVsX5Yd28/Px8pkydytu938PC0rLY8wqCgAD07NWT7KwsXN0KcgR/++23fDxqFNvW\nb6BnvhW2DxPNG0SRTIwMFlwK02xCQaa3zwzR2Ns7YtEhhAVde/Jhny4k5RbkqHiSbNGABhHVw+2W\nSMjBiEEUCxNrPyILAzmIjJGlYoGECH0WlWv3oXrjIYVtomMuUUcwrSIRjJLs7Cg0uekorRz/yu02\n8y+Rm5vDp6OGsPz7pVSvXh2A9kDXLl14q3NnQre2oHTZisydN49xY8cW2v6NGzdYtz6UVRt2/d9z\n29jakZ2dxbeLfqBjx464urqSmJjIrxs38tPq1bzX8x0mGh+7J9JFPXIEFkj9CudGegiOrDKmsFlM\np3KtulRp8QadAkrz9dD3EAVHE6FNRIftE/ZuiUCmWHzopVowcpxcImQykkUdWZYy2vX8AduHy5/j\noy9SUWaDSm8q/PX1cn69HQ5Nh5nsexV5pQRZp9Oxe/smDuzfV8QAJBIJH330Ie8NGEjYiWs8uH+H\nN1q3oXnz5iiVlhw4cJA69RvzwcfjC48xGAxcPH+anOwsKgRXxcXVjVs3r9K319skJycTFhZGr549\nOXr0KHPnzmXR7DmUfqLU0jkxBw/kRcQYCjK9NRRsOJqdyXsfjMLRyYVsnY6fyWCyaFFkYm6jMY1a\nghWqhyMMN0GOLVLCxSyaCY/PK4oiWyS5VG7yAe6+1TAYtNTzq2nymaaQWZChN2D9lPBrEDGIRmTy\n4l9GZl4+Yft3UbVq1UIxfoSPjw8hb4WwbdN6Zs5fzgeDevJGm7Y0atiA2Ng4zp49yxfT5hNU+nFh\nhvS0VH6/fB5LC0uq16qHOj0Ng15PubJlqVKlCocOHyYvN5cG9eujVqvxV1ghf6L2424xg94S5yLl\nlwRBoKfEid0GNVKplPeGjuTQwT1kYOSUmEM94XGURK5oYJsxnaGSx1VtGktsGGeIppvoWCTWPlXU\nc1qST8jADeRmJRGgssfTryYSyeM2MrmKDKPp0myATAzIX8EkQv+PV0qQ83JzEBFxLaZ8kb+fH6kp\nyQiCwPARn1C7XiOmf5JoFGgAACAASURBVPEJsTGRSCRSoiLvc/X3C1SrUYcTxw4z6ZMPsbW1xsnR\nid+v/E7bDp1xcHTiQWQkPd5+m8VLlvDjqlXI5HLy8/Px9vTkboS6cA1+IjqTatGPKCcouW3UsnHt\nKlq07YhC5USk0ZqPs2LoZFRhgcABcrgmZvOu4IRBFJEAl8U80qQSlghqYkQDDVGRg4GtklyiHJwI\naTQQhYVpMu+83HT2/jKWfK2GjaQxQlp0WfUuMvEvVRe5ovhkMWZePqkpSfj7FR814O/ny7WIBzg6\nOfPzxn0snPM1O7aEkpWViZXKivBDB2jQuDlKlRVzpk9i04Y1BFeqRGZmJsnJyXwycRr52nzS0tIY\nNnQoX0yZQt8+fQrs2seHB9ocdKJ9YeKrBFFHoMTUbWYpSPDFgovnTxMbE8WxI4ex82nP3Jj9NDJq\nqIMliejYJGYA4PRQXnJFI0eFXES5JaOFJN7RWxEgWBAhavhZlkuNJh/g6Vut2N8eeecYBzaPR69V\nEyG1ocwTf3M60cg2WT7VanZ7rntfknilBNnaxhY7W3uuXL1KpeCCMuIpKSmsW7eOg2FhWFtbU79q\nKeQKBTnZWQwcOJCOHTqQnZPD6tWrGdy3M4OGj2L1iu9YMH8eDRo0ACAzM5MPPvwQS5Udq1f/RMhb\nb/HT6tWMGTsWhULB+vXr6TNkEIsmTCY4X4n1wxyw4WJWsf28I2rwQcaNC2fJzsslLycTrJ3J8mjI\nurwEdJoU8gUnJIIra3KjWCPNQiZIUCgdadZuHk7uZbl4eAmH7hxHJrcksOY7hNTtg8LCtFag0aDn\n1+XvkKlOwCXgbU7G7EajS6K9YI0cgTBjFscsoctbU1/cgzHz3JQpF8z2jWuLLFI6d+4c60NDOX36\nNPlaHfv3bMfCwgKFXMbn4z8jODiYq1evMmvWLNo3r0mHt7pz5eJpwg4ewOlhNZ0LFy4waMgQataq\nx3eLl/D5hPGkq9XMnDmT3NxcFHI5ZcuVY+O1GHqKBXMrHoKcO2K+yYBDIxqJR0uAhQOHw/Zy5tQJ\nMlLSUbrU4LLUkovZ0Wh12ejlAehyExltTMBSIifPoMOvVD16hHxNYszvbDq2gsz0WOydS9GwyVD8\nyzQp9p7ER11kx5phIJHjUeY9Jt75mbdFW6oJKhJFHeuFTKwD6hFQtkWxx7+KCH9loie4cjXx1x2H\nX1xvnoE1Pyxh17ZQVi5fTnp6Or379KFFixa0ad2azMxMvl+2jKioKCZOnEjnkKJRBeM++YTdu3cz\n4L33+Pjjonka0tLSaNSkCd179ufg3u28804vvL29+fXXjZw7d5ZBAweSHBfPvl27aaRXYmkQ2UEG\nwwQXmksfLyq5JeYxxRBLc5k9YXIdNm71sXdtjtGoIzFqJ9r8NILrziMvO5J7V2bj4WFLszZv4lmx\nK+pk67+8Mun21b38tn0aEmnB6FenzUAmtYScWBAkGGSWNO74ORVrdCUx9irpyXextnPHy68Wwh8k\n9H8ZzBtf6rwoijX/vOU/S0mwa6PRSNf2jWjfrg1Dhwzhhx9+YNXq1QwcMIBy5cpx8dIllixZgiAI\nHDl8uEgx1PT0dJq3aIFWq2Xf3r1FKrID/LhqFdt27EajycPd1YVOnTqSk5PDd4sXI5VIGDFiBCu+\nXYw8LZOa+TJuiBruiBoWSv1wemJR1EpjCgmiljsKIzlSBW7+3bG2q0BO1n3i7m3A3a8THqW6khi1\ni+iIHwko14zA8i3wK9MEperZK4E8YvOP75GSGIOVXRBpCcexVHmAJhVjfjoyuQ15gp4hE84iIBB1\n9xi6/Fw8fKsV+p5LCn/Frl85QRZFkXkzv+CXdauwslIxZPBg+vR5XAkjPj6e1m3acOH8eWSyoh8A\n169fp2evXqxetYqqVU3Lu7Tv0IG3uvWlSrVabN20DnVaKuUqVqZug6b8su4HfjuwB71Oh5ePL1Wq\n1KB8pWrMmjiGMqIFlSQqokQtl8QcBkhcWEka1j6tCag0Bk1uPMmRO9BnPSAzJxYDAnp9Jh5+b6Kw\ndCFbfZqs9Ai6DVqLg3PxSdpzs1O4fmEz8dFXEEUjuTlJ5OWko9dpMBit0GuT8Q58B49SnREECaJo\nJO7eL8TcXY9SZUdOViKCIEUqswaMWFha8VbfpTi7//+isP82r7MgAyTExzJyaB9SU5LIyFCzb+9e\n3J/I6vfV11+j0+mY8sUXJsd+MWUKW7du5dLFiyb77t+/T0jnzhw6fZN9u7Zy7MhBZDIZrdp0QqfX\n8/OPi7l54xpWKiv8/YOo36QFt29c5dDubTQTbLFHyhkxBykCjSS2rBNTKdt4JSprX9Qp58mI3odB\nl0VS2mWkMmuUVh64eLXGoM8mOXYP3v7VaNt9TrEDAFEUSYi5zM1L28nKSAQMZKRGYTQaSE+9i4AF\nMoWKinW+QWXjD4BOq+bG2c/JzbyHhdKGvNx0ZDIrpDJL9NoMgoLb0KrzNGQyU7fLy+Cv2PUr5bKA\ngsmFUZ9OoXX7zgzs3ckkUF6n02FtbW0ixgC2trYYjUbi4+NNBNloNJKens79e3foO2A4VarXKrK/\nSjXT+5mv0ZCensbyBTPJFHMJEGW0lDmwWpaFRgcVyg0hNe4IkZen0wJryokyItCyz6jGr/Io3Hw7\nAODm2474B5vZHTqKMpXbEHP/NA6OpajeaABXzoTy+9lQtJosbBwq4urVCr02g9TEc8gV9iity5MS\nfwhruzJ4BnR94j5J8ArsQVLMfqztq1K+Vl/yNclER6wmI/UiOVk5rFvchSHjT72SlRX+i7h7eLF+\naxhfjP8YpcxYRIyhoNyYlar4eQA7W1s0Gg1arRbFUwmn4hMSkEgkqNPTCOn2DiHdii4SatvBND49\nJjqStOQkjlw4Q6BRQSVBSYpSxiZysbGvhcrKm3tnP0dIvUIHUYkKgTBRwj1DHmWrf1FYVNfdP4Qb\nZ0Zz+vB3ZKnjyMlOpnRwG1w8gjm+bybR984iigbcfTtiZVeNzLTLpKU8wNGtASobAU1ODH7lBheK\nMYBcYU9QlbFcPjqUgEoTUNn4kZF6iXtXF2Iw6rlz7QBabS5v9l7ydx7DS6VkfbP+BeRyGc4uLibG\n5+XlhUQi4fLlyybH7NmzB0EQWLxkCfn5RTO57dixA1EUMRqNnD970iTeU61O5+TxI1y7cglRFDny\n2z6a1i3Htl/X4OXnS6JM5JajEmX3N1m4ZhNShZL8vGTuXfySGbgyVHCkqcSWwRJn5kh9iL66CG1+\neuH53X07kZZ8lwvHfyVTrefymfX88E0zzhxegoAl3oG9CK47B1efNngFvk31Zj+BIGBp5Y29Sy3s\nnIrOzj/Cyb0RUrk1cgt7rO1KU67ml9g4VMDVpw2iKLI79COTYzLSY4i8c4z0lPvP/DzM/DMIgoBK\npcTD08NkX926ddm5axdGY9FUfEajkd179iCTyVgfWjQmV6/Xs3TpUvLy8rhy+QL37942Oe+d2zc5\ncewwCfGx6PV6Jn82gpA29UlTJ2Pv5sptiY57ZX1oPG4svQd/iIXKl/vXl2CTfJHFuPGmxJ5WEjtm\nSL1pZ1QSfXlW4bmlUgu8Avtw5vAy4qJjSElKZv+mz1i7qBOxD64BUKXRMkpVfB9X71YEVR5DcN05\npCedwrdMf0TRiJ2j6desytoPmdwGC6ULcoUdzh5NqNzgWyQSBQ5uDXhw6wgpiXeeuk8G4qMvEnX3\nBPma4ud/Xjav3Aj5Eb7+AaSmphIdHY2Pz+PYW6lUSq2aNRk2fDhLly6lcqVKGAwG9u7dy8JFi3B0\nciEyMpJu3bvTv39/XJydCfvtN7Zt3YomP59b1y4y88p5EpOSGPPZVNp27MI30z5n68b1lCtXjuSU\nFPR6AynJScjlMpo0boSHhwdHwsO5ffs2kTH3qVajDkqlistHB1MbC5PJEX/BgrpYExV7EM+Aghli\nQSLD0soHuYUD6uSzCBIpSisv8rKj0ean4uTemISonSAasXephaXKA98y/Yi4OA1LlQfxqZfQ6zIJ\nqPghEqmCzLQrxN3fSGbq70jlNuh1WchEEYncBgfXeqQnncS7dB+i76zDaDQgkUjJy0ljzy9jiY++\ngLVdIDmZkTi5BtLu7bnY2Hv+q8/3dSa4cg02h/7A4EGDimyvV7cuarWasePG8fmECTg4OJCens6X\nX31FcnIyzq5uzJs3j5s3b9KmdWsyMjNZs2YNUZGRSKVSNvy0lHv37+HrF8D0OUvR6fV8NmoIifGx\n+Pv7c/PmDVzdPUhLScbLy4t27dqhyctj85YtxCTF4unlQ5myFVi+dAlSTRbvS9yRP7UopLtgz47U\ni+i0GcgVBX5jpbUvgkRBfl4SOVl3sVC6ISCg1aRgbV8RXb6a+ORzyC0ccHCrj41DBaztyxJxcRoK\nC0cuHx9KQMWPcPZs+nAuZhdJMfsw6HO4f+07LFVuSIx6LGwDsXGoiINLLTJTL3F0z3RC+hdUHHkQ\ncYQDWz5HECyQyW3IybxHlTp9aPDGqBI1l/LK+ZCfZPGCmRw7tIfvv/8e54ezyufOn2f48OE4OzsT\nGxuLnZ0dGo0GDw8PAgICOHnyFI6ODmRkZODp6YkAaHU6kpKSWLt2LaX8/cnIyGDW7Nns3r0bQRCQ\nSCR07dKF999/H2tra1asXMnixYvZuWMHXl6PK+QuW7aMxYuX0PbNrmzftB47WzuaJ+t4R+ps0vdf\njGkc8m+Bb4WCgPbo22uJv7WCCoISZ4mcs4ZsRIUVpWrPJvr2ajJTL+PoVg9BkJGWeAJnz2a4+4dw\n7eTH2LvUJKDiR0RcnoFcYYedU1UibyzDu3RvVNb+PLgwFVddHk0EK9IEOChmIiocKFtnFr8fH867\no/ZjbefBusUhyC3K4lt2ABKpBUajntg7a4m9F0rlOr1p0GrkCw2de919yI/Q5ufzZut6dO/WhYED\nBiCTydBqtSxcuJDfDh0iLi4Og8GAm5sbKSkp1K9Xj/uRkaSlpaOQy3BwcEAul6O0tOT2nTu0aNGC\nSRMnYmlpyfnz55n65Zc8iIxEIgg4OTkx/rPPaNq0KRqNhs5dulCuXDnmzplTuEw7OzubXr16kZSS\nSsVKVTl9Ihyp3sBCo1eRTHKP6E88Pg2/Q2Xti8GQz/VD/TBokqglsSYbI5eNOdi51cEzqD/Xz4xD\nIrXA0a0+mtw4sjMiKFN1POqUiyRG7aB8za+QSC24cW4iZaqOJ/ZeKKJoxCuwB7kZd4m/tYJ6qAgU\nLLgsMXLFkIlr6d4Y9Dlkphxn0KfHSIq7xsYVfQiqOhF754IvyXxNCtdPj0NAQ/NOkwiq2PqFPc9X\ndlLPYDDw4P4dpFIZfv4BfxpxYDQamTvzC9au+p4KFSoU+HTVat7t35958+cTfuQIarUaCwuLQuF8\nb8AAjEYjjRo2JCwsjNi4OLKzsvj++++pWbMm6enpvN2jB5WCg+nTpw9KpZJt27ezZs0aZDIZP/7w\nA+tDQwkKDGTQUyMYg8FAvfr1ycjIYPWqVaSlpbFy/BS+0jiZ9P0TMQltpeG4er9BRspl7p36mDlS\nH3weLps2iCILhWROSwxUbL6B82E9qNLoeyxVHuj1uVw7ORqJTIVEIicz7RK1WvwKEilnD3RBEAQq\nN1iMysaf2ydG0kQdSz/J40UkGaKej8Q4rMv0JebOWgZ/eoyk+Ovs2ziRyg1Xmtz3KydGAHpUVkq6\nD173wrLH/ZcFOTNDTUJ8LC6u7jg4mtrD08TGRPHRkHeIj42mfPny3Llzh3LlymFvZ4dOp2P69Okk\nJibi5uaGra0tycnJNGnalG9mz+bsuXOcOnWKjIwMZDIZhw8dQiKRsP/AASZOnMiQIUNo2qQJcXFx\nLFy0iDt37hBcsSLz58+nRcuWHNi/v3CV6yOOHj3KBx9+iLe3N6Hr1/PBwMHUuRhNM0nRhVHJoo5h\nxFP1ja1IpZbcOvExpdIjGC/xLBLnPNoYjcqvHTYOVUiM2klwvXkAqFMucvPcROSWTtg7VSEvO5rg\nevNIjN5DVMQqLCydqVR/IVpNClcP9WO24EapJ1INnDBmMV+ag9KlJujj6DtyD7vWj0Cr98IrsOh8\nkyYnjktHByO3sKZBq4+oXLsnL4JXclJvx5YNLJzzFYIgoNNpsbV1YNznX5skS3kSiUTCmM+mcuPa\nZZITYgtHxPPmz6d27drIZDKOHjvGjevXcXJyonPnzrz11ltMnjyZhPh4EpOSaNeuHTt37qRGjRoA\nLFu+nJo1azLt668LrzNu7Fi8vLyYPXs2AwYMoEzZsnTq1MmkP1KplICAALRaHXXr1kWr1TJj6lfs\nzcukNQXr+EVR5KAxk9tiJqrYg2SkXiIz5gDvSJwKxRhAKggMEZ05prtHWsJxbB2rkJF6CZ02g/g7\n3yIYE7CUysnO0SKTKVCn/Y6zewMslW7IFLaobPzR5CaQrb5JT6FoGJCdIKO3aMfKu79gbeeGpcqe\nhOhL2DnXLvYl6OhWn/y8RPKyb3P3+n7KVGr/l5/v60pubg4zvxzP3p1bcHN3IykxifqNmvH51G9w\ndDL9cnqEl7cvP6zbQZNaZYiNjUWj0XDt2jWkUinTvv6a+Ph4tmzdSoZaTXBwMG+++Sbly5Vj4qRJ\nWFlZIZfLqV+/PvZ2dkgkEnQ6HZMnT2bpkiVUq1awCCMgIIC6devyVkgIV65cYdr06VhbW5uIMUD5\nChXQ6XTMmDEDGxsbBrw/jM/e/4hyWsvCIqga0cgcYwJGAe5dXYA2X01u2iVGSEsVqbbuLsjpLzix\nInovPqUHcvvyLPS6XFJid5HwYD3OTrZkZ6eizb1OTmYSAI6u9bh/7Ts8yw9DEKQkR+2imWBdRIyh\nIOfLBjGXewnhNO9UEHsfH32JoKpdTH6TpZUnCktn/MoO4Ni+2VSo1hmZ/OVGZpQI58m+XVtZMHsq\nC+bN5cih3zh+9CifjBvNJyMHcfH86T89Pi0lmblz5nDq5ElC169nxvTpZGZm8kbr1ly4cKEwoqJH\nz54cPHiQhg0asGXLFj4eOZLTp0+Tl5dHcnIyALt27eLd/v1NrtG1SxeMRiNanQ69Xs/vxUwa6vV6\nIiIiaP1GKwAUCgWr1v3MbjcZIy2T+YYkhoqxLBWTKddwMa7erbGxL4dcakFFQWlyPpUgwU9hRezd\nUNQp57h7ZR4R50YybnQ/zp09yckTJ9iyKRQPdwdSY3c97EMOCsuCPyitJgk3qbJIJq5HlBIsMOqy\n6NBjIQAWlrbotWnF3l9tfioyuQ1Onq2JuLr/zx6HmScY9X5/8nPU/BZ2kH179hB+5DAero4M6huC\nXl/8cuBHxEQ9IDAwiEO//ca+vXvZvm0bwcHBbNu+nX79+yOXy6lcpQrHT5ygdZs25Gk0LJg/nzU/\n/USD+vXZtWsXN27eRBRFzp47h6eHR6EYP0ImkzFo4EDcPTw4ePAg2dnZJCQkmPTlypUryGTywgVZ\njRs3ZtjY0YySJfC1ZTpfGWLpY3zAHUGkQsPF2NhXwNahAnbIChMCPUkFQYloNHDz/Ocg6jkf9haG\n7N1s3riOE8fDuXD+HJ+NG4JMqiUn8z7a/BQEiQyZomBErs+OIUg0rXoCEGAApZUTwTW7AwW2rdWY\n2rbRqEevVWNlVxqllRdxUef/8Hn8G7x0QRZFkcULZjJj+rRCYxEEgWZNmzJm9CiWfzfnT8/h4OjE\n/AULaNO2LSGdO7N9xw5u3brFhPHjWTB/Pj169GDs2LFs37aNY8eO4eTkRNNmzQg/epR2bdtSrWpV\nWrdpw+nTp8nNzcXOzjSIXaFQoFQq6dC+PRqNhu+XLeP+gweF++/evUtI+45oMrJYsmAREz75lLi4\nOAICAth3+De+WLqIG85WqK08yBdkSKUWuHi1wN2vEzKFHTGi1uSaelEkXq/BoM+hSsOleAd0oG/f\n/nQOCSkM6wsKCmLlypXkqH8nLfEkolGPOuUCBkM+FipP4g25aETTAnk3RQ2epWrh6lURgNLBbUhL\nPEleTmyRdlpNKskx+3HxaomABMwZ456ZK5cvcO/2TWbPmllY19HGxoYJ4z/DUiHn0MHdf3i8i6sb\n0dFRDBgwgPYdOtCzVy8yMjI4deoUO3fsYPSoUfTs0YPF333HBx98QGxsLOFHj9K1a0H4Y5/evblz\n5w59+vQhNiYGG1vbYq9ja2eHy8NRccWKFZk0aVLhy0IURX755RdGDXsfbU4OrRo1ZvXq1RgMBnr3\n7UP4yRM0ef89LiggXyLHILPCyjYAd7+OuPm2JxNDsfb3yN5tnapRs+UmFAoVK1YsL0ydK5PJ6BwS\nQp8+fUiJ2Ub07Z9RWvuSmnAUALmNP9eF4pMV3ZLoaRUyrTAfRoXqb5FwfwPiU/1Iit6L0toHS5U7\nCCXDtl+6IGeo00mIj6V+/fom+9q0acOZ08f/8Hi1Op3IB/dwcnJi1Y8/snvXLry9vXF2dqZDhw5F\n2np4eNC/Xz+2bdvGzJkzWbF8OaNGjWLDhg3MnjWL9wYMQKfTsXfvXpPrbN22DU1ODru3bufe9Zto\nsnPo2LoNQ4cO5bPx4+nUth3V76exVOrHPIM7hm2H6dLxTWJiYpBIJORkZ5OZmYFen4urd0vSEk8W\nntsxsCdrjClkP5UNa7uQgcFoxNmjKSlxB8hOO0v7dm1M+lbK3x8XFxduXfiCMtUmoLTy4ua5SQiC\nFHunaqwkvUjqzSRRxy/SXKq1eJyuUWXtTOO2n3Lt1Ahi724gM+0q8ZHb+P3EB3iU6oKllTepCQcI\nqtjyD5+HmcecPX2cli1bmMTEC4JAm9ZvcPbUsT88/vdL55FIJLRu3Zq9e/awdMkS1Go1Q4cMMXEr\nvN29OzY2NoSFhbF//36mTp3K+PHjOX7sGCorK6ZMncqZM2dIT08vcpxer+f7pUt5cCsCTXY2F06e\n4tSRcGrVqsX06dPp8lYI8yZMZoTBiVXSAAYlSdjyzSJGfzgCURRRqVScOn0OkODu2xGjMQ9dfkGN\nRoWFI0qVBxvEotfUiEbWSjMwKm0QMBB9exXOLi6U8vc3uQcd2rcnMyWc3Mx7BFT8kKTovSRE7cLZ\nuzXHyeG6mFek/T4xkxwr2yLLsavU6Y2Fpci1UyNJTTj6MGZ5EVERKwkI/pi87Bhys6Lw8C0+dPTf\n5KX7kOUKBQaDHo1Gg1JZ9LM9MzMTpaXpp/yTrPlhCQ3q12fG9OmF26pUrkxiYmLx1aTLlsXewYGm\nTYqun2/ZsiV169alcqVKzJs/Hy8vL5o1a4ZGo2FQ3/5cvnQJP0HBCIk7PjIL0kU9a6UZnD15Gp1O\ny7s40V7yeDlrPxwRslIZM2IkTh4eHD58GAf39viVe48H15eSnRGB0aBFREQUDWQIMNj4gI4SBxxE\nCSdl+VzTZWGUy2lUW4q7mxM/rcklM8s0flIURbKyMrB1qk707dXka1KRSDI4/1sPLFVeHDJmcFbM\npJHEBrVMyiljFnVbjcE7oG6R81Su0wsXj/Ic2DyB2HuhqGwCCao0BqW1N3evzEYmM1C6oukLwUzx\nWFpakplZfLxrZlYWlsr/H7FiMBiYMeVTli9fTs2H8xvOzs64ublRunRpk/YSiQRfX19q1qiBg8Pj\nCVyZTMbUKVNo264d7du1Y8DAgSxauBAvLy/OnD3LkPcGoMvLo4vEkU6SAJRIuIGGBdo0fgndgC4v\nj+USv0K3QyVUfKG1ZPjhI3z51VccORJOaloOgZXH4+BSi4TIbeRk3cXeogaanDhQebEj73xBPmTR\niizByG6yyDYYcHJ1JqSNI/fuZ3LiRGqxxYYzs7IQMWLnVJXrZz7Fyq4MD24sJVKQIrWwZ0JuDBVQ\nUlqi5IpcJEkhp9N7Pz+VLc6CLgNWc+Xsek7sn4tEqsTBtR6VG36PJieGB9fmU6/FCOSKP9aaf4OX\nLshWVtbUrtuQ0A0bTHy3q1avpnX7P65ycXDvNmbNmF5kW2BgIFfnzcNgMCCVFvUznT5zBi/P4mNq\nq1SujN5g4Ltvv2X4++9jYWGBITePwNyCGzVV4l2YKtNBkPG+wZFPZcnc0ObTUmr6OdhesGPg71fw\n1tWncuOhyBV2GPS5pCUdxd7Jj9P7CibHvAPq03X4FpLirnLg2DJ0uWp0EjkKSxv279uLi4sLAEqV\nipUrV1Kvbt0ihhsWFkZ+voZc9VkUFg7o8tUorbwJDB5NviaJ5NgwBGsr4iu2RKmyp09wO6xsXIq9\nBx6+1egzYheXT/3E2fAV3Dg3HqlUQflqITRsPR2pzLS0vJniadm6Awu/+aowGuIRWVlZbNq0mW9X\nrP+/x167cglra6tCMX5EUGAg586fp2HDhkW2a7Varl+/zpjRo03O5e7ujrW1NQMHDmTxkiW0euMN\nPD09ibsfSR1UqARbekgeR35UQMlEvQujDLHUQGXiA7YQJLTUq9i6JRzPskOoVLEWgiAlJe4wVrZu\nPLj2DXm5acjkllSs3pUajRZw4uAcNtwOR280ojEoaN2qGXPmzCmc6G7dpg1hYWG0bPn4C0wURZYt\nW052dha5uWFIJBbkZETg6NYAB9faZKVfJzkuDH3lNsQ5+lLGtTStyzUvNgpIJrOgWr3+lAluz9G9\ns7h9ZReJ0TuwdfClYZuRVKhWMqrpvHRBBhg9/kve69mRlJQUOnXqhFarZcOGDRw7foKfN/7xJJJW\nq8PyqaTzwcHBuLu7M2/ePEaPHl0oXhcuXOCXX36hVq1axZ2KGzdv0qxpU+rXr8+G0FDeCglBojNQ\nXXDEDmmhGD9CEAQaaxTcRUCG6WjcEgGjaMTOuQYgJTnuMIkPNhJU4Q1ad52BXp+PgFAocm5ewVSq\n1QOAPaFD6NejaaEYA/Tq2ZOdO3cycNAghg0bhqODA/v272fJkhXoDBZYKq3xr/AhSutSpMb9RkLU\nDjS5MZSv0pamHb945nA1QRCoWq8fVer2Ra/TIJNZlKjg+VcFVzcPBg37mB49ezFyxEdUrVqVWxER\nLFy4iOZvdKBCQotBvAAAIABJREFUsOkKtEfodFoslaYjtnfeeYcePXvSqmVLKlYs8P8bjUamT5+O\nVColJyfH5JiU1FSysrJwdXVl1syZyGUyNm3aRAOZDfH6fN6VmFbQ8RYUuAhy8v9PUnklEqSCBbaO\nFdHkxJGSEE7C/V95s88SvErVMbGb1l1mAwU5WX5e0IIpTxSYEASBL6dOZeiwYQwaNIg2rVuTlp7O\n0u9XcPV6LFKZNfbO1fEM6IHRqCUxaiexd0PR5acQ0m8ZPk996f0RVjYutOk2mze6zMCg1yKTW/7l\nhF4vkhIhyIFBZVm3+QCrVnzHsOEfIJPJaNm6I+u3hP1haBBA/YbN2LlrF6PKlCmyfdiwYXz88cfs\nO3CAenXqciviFrdu3aJs2bKcOXOGs2fPFhHmEydPcu7sWb6ZXWA4ZcqUQaVS4ZZnwFInQfd/HP46\nRAQEzoo51BWK5io+ImbjZO/D7fOj0OnysLd3ID8vC5m0HPmaTCwsbcnXZJKech8rW9ciCecz06MI\nfjij/QiVSsXan3+meYuWDBw0FIlUhbV9JXzKjeLBjQXUbTack2FzUVp7IwgS8vOiqd/yI6o3ePfP\nH0IxCIJQIj7jXmUGDBtJUNnyrF31PXPmzcfTy4d3h35M+05d//C4CsFViIqKJDIyEj8/v8LtpUuX\nplKlyvTo2Ys6derg7uZG2G9huDg7o9VqmTZ9Or9Wq4a1dYEtGo1GZs+eTbt27VA9zIPRrl07wnbv\noWqOJUnkF9h2MZqkE41cFbVoRWORSB1RFAm30GHrLOXqsR6IooilpQqVSkFGeiTeAXWRK5Rkpseg\n0+Zh7+xfOBjIyojHw9MbG5ui+VPq1KnD7FmzGDV6DMuXr0FuYYeNc1MsbW3wCbQiOyuZm2fHYeNQ\njrzsGCxVNnR59xec3cv+recikUiRlEDbLhGCDODl48eEKbP+vOFT9B/8Ib1CWuLu7k6Xzp1RKBSc\nv3CBL6ZMYdzn07C1tWfRvK/R5Gbz7aJF/PDjj0ilUgYNHkyzpk2pULEiFy5c4NzZsyxatAgrq4Kc\nw+Hh4djZ2pKWn0p1VKwUk0kX9UWqKOhFkV1kYGFvy7yMRMYYoYZghQgcF7NZLclEhh0e7q60a9eG\nPn36IAgC02fMZsead3FyLU3Eld04u7iSkpJEqTJNaNR+KkqVA3YOvly9epUKFSoU+b0SiYTc3FzK\n1lqC0tqHbPUN7v4+g7otPqJavb5Uqv02sZHnQBTx9KtpFtQSQJPmrWnS/K+tBFMqVQwePpohQ4cx\ne/YsKgUHk5eXx89r13IrIoKNu8PZsXkDq1Z8S/NmzRg4YAC9+/QhOTmZN1q3JiQkBKVSya5du7Cz\ns2P5smVAgZiGhoYSWLYscZcfUBdr9ooZVBVVRUaKt8Q80oxaJFIJX0mSeN/ohJsgRy3qWSvLRGOr\nIjUxkmrVqvFOr160aNGCa9ev88GHI8lIiybuwXEy06OxtrYhOzuX6o2GUaVuP2zsPImPiyErK8tE\nlHNycrBUeVC29rKHI+FdpCeG077bNmzsPclIjyEt6TYqaxdcPSuWqJHtP0WJWqn3d4m4eY3ZX3/O\n5YvnsLCwwNrGluEjPkGdnsZ382fQqFFDXF1c2LdvHzk5ObRp25aTJ0+i1WqxsbEhKjKSTp06MWzY\nMOzs7Ni3bx9z5s5l9qxZLJg1m/p308k06jlqzOJdiTMVBRUxaPnBmIzG24VdB/bz3oAB3Pr9Gtr8\nfPRGEWsrR7K0GXTpHEL58uW4cPEihw4dYvF331G9enXq1K1H/foN+GLyRBwdHcnKymLO3AX8duQc\nXQdtIfreSU7sncCWTRsKK6SIosiXX01j+859ZGVmIJVZIlcoqdf8A4Jrvf0nd6nk819eqfd3EEWR\nDWt/YMWS+ei0+eTm5VKrdn0+GDWBb6Z9TnxsNE2aNCYzI4O9+/YhkUgYNHAgK1auxN3dnby8PNRq\nNV9MnkyrVq1ISUlh+YoVXLp0ienTpjGwd1++0row2xhPGcGSEIkjDg9TbS4Tkxk88iM6d+5M21Zv\nAAKWgoQ8ox57eweklgp69eqFVCpl+/btWFpYsGLFCsLDwxk7bhzTvv6a9u3bI5VKuXXrFu9/MBL/\n4J5Urfcu+zd+RJVydkydMrlQVFNSUwnp3JW09FxEEQx6DR6+NWjWcSJOrqZV5F8lXtml089KUmI8\nZ04dQy5X0KBRM6xtCibU1Op08jV5uLi6c/rkUSaNe5/Q9evwfDiJZzQaGT9hAuHh4RwND+fUqVNE\nRUdjb2/PmdOn+XXjRvR6PbVq1WLkyJHUqF6de/fu0bt7D8rmC1hr9FwQc0lBj0qh4N3Bgxg6fDgK\nhYLIyEg6d+1Gz7GnUMfcZOeafmzetIGAgIDCfh87dowxY8eyaOFCRn78MUcOHy4SEiWKIp3e6kpg\nteEEVXiDc0eXcPHYMtq3b4erixN794WRly+jQ+8fESRSdNpcrG3c/jP+3dddkEVR5PKFs0Q+uIuX\nty81atdHEAQMBgPJSQmoVFbY2tkzbsRArCylfPXll4X5JuLi4ujcpQtt27Rh5MiRHD58GK1Wi4WF\nBT+tWcPVq1exsLCgR48eDBs6FHt7e35YuZIl8xbQRGtJrFHDNTTkIxLg48vEr78sDEX9Zs4cNHoJ\nvd8dxq/rVhFx/SJLFn9XaLtGo5ExY8bg5OREbl4eTk5OjHqqAERERARv9+pH/1HH0Ony2PXze8gk\nubRr25K0NDU7du6iUu2+1G76EdlZicgVKiyVfz2pfUnklVw6/SwYDAZmfTWe7Zs3UK9ePfI0GiZ/\n+hEfjZ5Ar36Dsbd3AAr8sOtWf8/7w4cVijEUfO5PGD+enTt3MnHSJMZ/9hkNGjQgKyuL8CNHEASB\nDh064OfnR42HxSYDAgLYdyiMGdNnsHPTZvSiyJqf11KrVq0in0yenp5kZqixEGU8uHWQkJA3i4gx\nQMOGDSldujTrQ0Np3bp1sfGpHTu04eDxUwRVeIOajYZROrgjEVd28iAxh+D64/ALalQowP8VgzUD\nMdEPGDG0D5q8XCoFB3Pz5k2MCMxf/BMBQWVw9yjIxZKakkz4oQOEHzlcKMZQYH8jR45k1qxZNG7S\nhE6dOiEIArciIkhISMDf35+kpCT69O5dWG3kvQEDaNCwIUMGDiQjJYfS5YJZvHRJkYiQR+e+fDUC\nD09vtm1az4rl3xexXYlEwujRo+nYqRMODg4sXrzY5PeVKVMGeztb0pLv4uJRns4DfiXq7nHOXjuD\nXOFF18FbsXcsqCloY2eaevR14ZUS5OWL53Lj6gUOH/oN24erjqKioujTrx+e3r40bfE4Rjby/l2q\njnjf5Bw2NjYEBgYSHRVFo8aN8fH25v6DBwQElsHH15+goCB+/PFHrK2s6N69OxYWFhw9epRdu3eh\nk0lw9yjIt/y0/+ro0aN4+1VGEATyshMoU6Zysb8hKCiImJgYjAbT1UsAGRmZyOSP/b52Dt7Uajz0\nL98rM68Oer2eIf260qvn27zbv39hKNgvv/7K4H5d2BV2FouHkUQx0ZH4+vmZ+F8BqlWtip2tLTOm\nT2fy5MkolUpiYmLo8FZ3du/YxJDBgxkwYABTp06ldu3apKamsj40lGyNhoBKlclITy0S1fOI8KNH\nqd+k4G8rMTGewMBAkzZeXl7odDoMBgNZmZkm+41GI9nZ2YVzGoJEgl/pRviVbvRc9+6/xivzravT\n6Vi3ehnTvvqqUIwBfH19GTdmDKuWLyrS3sPLh1u3bpmcJy8vj9jYWGbPns2B/fuZNGkSFhaWzFqw\ngs+/nMP3y5YxYsQIzpw5Q81atQiuVInPJ06kVu3aVK9Zl4HDPubziZOIjX28xPj+/ftM+uJLKtcb\nAoCdUxBnzlwwubYoipw+fZpr125yMCyMpKSkIvuzsrL4deMWgiqak/e8ThwO24uDvR3vvftukVCw\nt7t3JzCgFPt2by1s6+7hSUxMNBqNxuQ8t27dwr9UKfbu3csPK1fSOSSE8hUqMXXGQtp17MLRo0fp\n1r07EydNomJwME2aNmXnzp14eHjSs+8gHJxcmDFzJlptwbJmo9HI+tBQrly5Soc3C/J2BwaV4cIF\nU9u+fv06FhYWJCWnsvKHVTztCt23bx9KKxfsHP1MjjXzmFdGkFOSE5FIJcW+nevVq8etG9eKbOvW\nsz/fLl6MWq0u3CaKIosWLaJKlSq4u7uj1WpZ+v0yGjZpgX9AEDVq1eOLr+cze/Zszp0/j5OTEy4u\nLtStW5fLl39n7ISv6Ny9N50696Jjpzfp3acvPXq9w5sh3ahQaxClgwtGEeWrd+XQ4cOEh4cXufaq\nVauIi0ugWpOx1Gg8nK7derJ9+3YiIyPZt38/Xbv3wr9sa1w9i0ZWmPlvc+vGVerVKz6Wtn79ety8\nfrXw327unlSrXodvv/uuiOip1WoWLlpEzx4Fcezx8fH8uGo174+aAMDEL+fg6OLB3LnzCss8lS9f\nnsDAIByd3WjTPoRFy9YRcecBjZs0ZeCgwbRs9QZr14WyfM0WrKwLRuS9+w/l62nTiyzBzs7O5rPx\n41Ha+ND53XVcvRHNBx+O5MKFC9y9e5fvFi/hswmTadB28n8yMuKf5JWZ1MvJzqJp3fIcPXLEJPnP\npUuXGPvJZ+wMO1u47VEx1O2b1xMSEoKjgwP7Dx7k/v0H5Gs0ODs7o1ar6dy9NyPGTipSCio9LZXv\n5k3nWHgYiCL1Gzen/6AP8fUrVaQ/vx05wdlrMtxdy5uEl8U+OMveDe9TunQQZcsEcfbceTKzdLTr\ntRx7p4JRwt0bB7l+7ifSU+5ja+9FuWo9KVul02trtK/rpF7ozz9w8fQh5s+bZ7Jvwuef4+VfngFD\nH+cdSU5KZFCfEGxtrGjVsgVp6els3LgJiUSCKIpIJAK2dg58/MkXNGvZtsj5Tp8IZ8WSedy7G4Gj\nozOdOvfk7XfeRWHxOO3k/bu3uXc3Ajd3TypWqlrEHkVRZO6MyWwMXU2zZi0QBAkHw8IIqtiOJu2n\nIkgk5GuyuHxqNfeu70SnzcPTvzZV6g3C2a3oWoHXhf9slMXYjwbg5e7EJ+PGFW7T6/UMGTqUGnWa\nMmDYSJNjbt+6zq5tG8nNzaZG7QY0b9WOnJxs1GmpuHl4ovyDfAJ/RHKmyM04a25eiPm/bfT6fO7f\n/I3szAQcXQLxDWz4n4mIeBG8roKsVqfTtkk11v68hvLlyxduv3//Pl26dmPb/pO4uhWd6NLpdITt\n38WFsydQqaxp/2ZXgsqUJy4mChERL2+/F/pivxYRzcnw/Vz6PQe/Mk0KJ+TMmPKfFeSU5CT692iP\nr68PnTq0Jy8vjw2//IqtvRPfLl9f5C3/InkWMTbz13ldBRlg784tfDlpNN26dqVypUpcv3GD0NAN\njP5sqkmV6JJAcqbI1p3qP29o5r8b9ubs4sqvO4+wa9tG9oeFIZPLGTh8DE1btjUJIXtRJGcWvMDM\nYmzmn6RNhxDKB1fh13U/snXnXry8fVn9y24Cg/7e0mAzryavlCBDwZLSrj360rVH33/92hm5Bf83\njwzMvAj8/AMYM/7Ll90NMy8Rs0PzGcnIBa1eZOfR4svGmDFjxszzYhbkZ+SRGOszUl92V8yYMfMf\nxSzIz8Ajv7FZjM2YMfMiMQvyn/BIjM1+YzNmzLxozIL8B5jF2IwZM/8mZkH+P5jF2IwZM/82ZkH+\nA8xibMaMKY8GK2b+ecyCXAxmgzNj5o8xD1ZeDGZBfgqzq8KMGTMvi7+Uy0IQhGQg8sV1x8xrjp8o\niqYZ0l8wZrs284J5Zrv+S4JsxowZM2ZeHGaXhRkzZsyUEMyCbMaMGTMlBLMgPyeCIEwXBME0M37x\nbecIgjDsRffJjJl/gr9o25sEQWj75y3N/BFmH/JzIAiCC3AJCBJFMU8QhArAT8Cjwn/ngY9EUbz+\nsL0HcAYIFEVR+zL6bMbMs/C0bT+1bxIwBWgliuLBh9tqA0tEUazxr3f2P4R5hPx89Ad2P2GwcUBX\nwBFwBrYDoY8ai6IYD9wEOv273TRj5i/Tn6K2DYAgCIFANyD+ye2iKJ4BbAVB+NcrvvyXMAvy89EW\nOPLoH6IoqkVRfCAWfHYIgAEIeuqYw0D7f62HZsz8PYrY9hN8B3wCFPeFdxizbT8Xr1zFkBJGJeDW\n0xsFQVAD1hS88CY9tfsG0OXFd82MmefCxLYFQegG5IuiuPv/FFC9ATT8F/r2n8UsyM+HPZD19EZR\nFO0FQbAC+mG64CDr4XFmzJRkiti2IAg2wDSg1R8cY7bt58QsyM9HOmBT3A5RFHMEQVgKJAuCUF4U\nxaSHu2wA87psMyWdp237C2CNKIoP/uAYs20/J2Yf8vPxO1DmD/ZLABXg9cS28sDlF9kpM2b+AZ62\n7RbAR4IgJAiCkAD4AL8IgvDJE23Mtv2cmAX5+dgNNHn0D0EQWgmCUE0QBKkgCLbAXApGGjeeOKYJ\nsOff7aYZM3+ZIrZNgSAHA1Uf/hcHDKFgku8RZtt+Tswui+fjJ+CSIAjKh+FB9sAiwBvIoyDmuI0o\nihoojEOuAGx9Sf01Y+ZZKWLboigWKSgpCIIBSBdFMfvhv2sB2Q/D38z8TcwLQ54TQRCmAUmiKM5/\nhrZzgLuiKC5+8T0zY+b5+Iu2vQlYKYri7hffs/8uZkE2Y8aMmRKC2YdsxowZMyUEsyCbMWPGTAnB\nLMhmzJgxU0L4S1EWDo5Oope374vqS7EoNOnkJ6aBXPGvXtfMP4SuIOWB4Bf4Jw3h2pVLKS+jhNO/\nYdeytDh0WXlmO34NuZ6a+sx2/ZcE2cvbl193HP5bnfo7aId3BhyRNWr6r13TzD+PPiEaAMXizX/Y\nroK//Uupa/dv2XWBPYMglSJ18Xzh1zNTMgj+YfUz23WJdVk8Ml6Zu89L7omZ50Xm7oNAwTN1yLn7\nsrvz0lAs3ozb7NmIBkPhS8qMmScpkYKsHd4ZAbMY/5eQPhTlxLFjcQ19OgHe60O6VWDhl4I+IRpD\nctxL7pGZkkSJE+RHI2OpWYz/c0jdfZC5+xATfrXwOb+uKBZvJvDD3ubRspkilDhBBvPI+L/Oo+f7\nuotydPnO5tGymSKUKEF+nf2LrxuPRPl1dl884knfssE8Wn6tKVGCDAV1j8y8HjxyX5hF+bFvWQSz\nC+M1psQJsjmzxuuFIJUSE371ZXejxPCkC8MszK8fJUqQ060KFg+YP9teH6QunoUhcWYKUCzeXESY\nzbw+lLh8yG6zZ5M4diyG5LhXPng+NS+P9bciOJ6YjEwi0NzDnW5lSmOtMK/WehKpu4/5JVwMisWb\n8bmxmbuLfkagZEUeXUhMIvTOXR5kZeOuUtKtlD+NfLxfdrdeeUrUCBkKRsmPwoFeZWKysnh79z4y\nAkszfvZsPv7qa27YOdBn/0Ey8vNfdvdKHCVJbEoSjyIxSpJv+ZdbEYw9eZqa3brz9bff0nrgIGZe\nv8GiS7+/7K698pS4ETIUGKF340vEhF8tcSODZ2Xupd/p9e67vP/BB4XbGjRowPhPPmHZ1WuMrVH9\nmc4jiiKn4uLZfus2mZp8qni40aVcGZyUyhfVdTMlEMXizWiHdy4U5ZcVGpqWp2H+xcts27kTX9+C\n/B+VK1emefPmtHvjDdr6+RDk4PBM58rWatl2+y6nomJQyGS0KR1Ac18fpJISN0781yixvzypx9QS\nNzJ4VvL0eo5FRdO3Xz+TfYOGDmX3/Wdb2i6KIlOOnmBq2FE8o3JokAQ3rzyg86Zt3ExN+6e7baaE\n87Rv+WW4eQ5GRtKkceNCMX6Eo6Mjnbt1e2bbTsjJoeumbRy9cJPaiSJlY/NZevQMH+w7iO4V/zp+\nHkrkCPlJnhwZvKxRgSiKHI+NY8uNCNR5GoLdXehRoRwe1tbFts/X65HJpFgXs9/Z2ZmcZ3RZHIqK\n5tyDWOaK3igfjhrqY8MhfSaf/XaEzV3fQhDMgYKvG49E+Z/4u8jSatl86zbHI6ORSSS0Lh1A24BS\nKKTSYttn63Q4u7kVu8/F1ZV7Bv0zXferoydoolXSU3AqHBY2N9oyNTme0Bu36BNc4W/9nledEjtC\nfhLF4s14Nw5+KaFAoijy5bGTTPvtGKVjNbRLk5N6I55um7dzISGx2GPsLCxwVKo4f/68yb6wsDCq\neLg/07W33ojgLaMtSqHoY2oq2JCr0XIzzTxKfp153hC5hJwcum3axpmLt2meIqVOksgvpy4zYOde\n8vTFC2tVFxcOHzyI0Wg02Re2Zw9VHR3/9LpqjYZzCYmEUNS1IRMEuov2bLlx6y//lv8Kr4Qgw2MX\nBvy7MZrHYmI5dT+ab0Rv2kjsqSGxYqDgzEdGFz47FI6xmJqEgiAwqEI5Ph0zmrt3H68+vHjxIjO+\n/poB5co+07VT8/JwE+TFnt9NkJOWp/n7P8zMf4LnCZGbefw0jfKVjMWNuhJrmkhs+dLogSpDy6rf\ni48Nr+bmipNEYMrkyeTm5gKg1Wr5dtEiEqIiaeXv96fXVefnYyuVYSmYyo8bctJe40nvEu+yeJqn\nP9fgxU5wbLt1m45GW1RPTTTUEqxYq1dzMTGJGu6mn3BvBgWSq9fTs+v/2DvP8Ciqt43/zsy27KaT\nXiAJvRfpHQHp0pvSVJoNUUH9o4KAXRQVFRXp2MFGBxUElCa9l0AgCem9bZ3zfkhYCAmI9RXkvq58\nyOzMmd2zZ+95zlPupz+hIaHY7XbycnJ4qmF9moeFXte9awQGcDA3g7qYSx0vlBqnXEVU8fP94x/s\nFm4q/F7XXoHDwbYLF1gsokodV4RgoPRl1olT3N+oQZnrhBC81bY103ftpk3LllSOiiIuPp7qfr7M\n69gB/VVcHZcj1OJJodRIlnZCROkU0AOykBrXYWXfrLjhCPkiriTmvysbI7vISiDlW6mBQkf2NZ7m\nQ2tUp3/VKhxJz0CnCKr5+XEqO5v9qanU9K+AUXftxXt3nVqMOLOa+tKDOqKYlO1S4z2RRrvwCIIt\nlj/34W7hpsLlpPxbIvj5djseQsEsyq7BIPTk2O1XvdbHaOSNNq1IKSjgfG4eITWq4W00EJuVTYHD\nQWXfaxsKRp3KkFo1ePvoGabIEDxL3sN5aeNjJYuXG7S/vg98E+KGJeSL+LuJuXZwIPsyk2hMafKz\nSo1jrt9+mhtUlYbBQfx47jyPbdyEwSUxCoVU6WR8w3oMq1P7qtfG+Prw8u3teGbzVoLQUQEdh1yF\nNA0NYWrbln/J57uFmwuG977CryCWlMmTr2ktV/DwQFVVzrpsRAtjqdf2ygJqVfhtKzXYYsHfZOLV\n7btYGXuGiqqRdM1BgMXMzA5tqH6N38b9jRqQY7UxJvYM9RQLhWjEalYmN2tCs+vcQd6MuOEJ+SLK\nEPNf1CZnSK0aDDx+kvqaiSbCghACq9R4V6TRIjSMcK/yMy0ux97kFJ77aRtPyOBiS1dCgrTz4t7D\nGHU6Bl7Dp9w6IpyNdw1i54Ukcm02/hcYQEVvbxyaRkJeHl4GAz5G41Wvv4X/Hi4KFdkf6IcrOb5c\nA0WnKIyqV5s5+4/xrAzBTxRTwTlpY4mSycxGba/rXs9v2078uVQ+oBI+mopLSjbl5TJ2zQZW9OtN\ngLn8fHlVUXi6dQvGNqrPr8kpmHQqzcPC8NDpyLXZybXbCDabr8sFcjPhpiHki7heC+F6Eerpydtd\nOvG/H39imTOLIKHnmKuQFmFhzGjX+rrG+HDvAYZp/tRRLvmCI4SBR2Qgs/YeoH/1aijXSF/TKwqt\nI8IB0KTkowOHWHrwCKqUFGguGgcHM6V1i+t6ONzCfwcXZQiuhhF1a5Nrs3P/0WNUVzywI4nXbDze\nrAmtwsN/c/zUwkI2xJ1jPpXcrg9VCDoJH05odr44foIHyvFDX45As5luMdHu8f73409sv5CEp6LD\nLiRDa9VgXMP6/5likZuOkOGShXBRBwD+XOCvUXAQa4cMYF9KKtk2GzX8/X8X+R1MT2e8KHv/apiw\nOpykFxURZC4duDufm8vC/YfYnngBg6LSuUo0I+rUYtGBQ2w5fpaZMoSKwohVaHyXms2olWtY3r/3\nLWv5Fty4XKyrPCtZCMHDTRoxol5t9qWkolMUGocEY9JdHy0cTkunlmrGrJW1YptqHqxPTIJyCPn7\nuHN8eugo8Xl5hHt6MrRuLVqFhzHquzW0sBlZKKIwo5Ko2Xn3SCzZRVamtG7x+z78/zNcaRf+kPzD\nTUnIF1GsA9CPoM+muiUe/ygxK0KUm01xPTCrOrKdLnyvmG4bEqvU8LjiB3A8I5PRq9fTTXrxPwKx\norH2yDmGnIoly2ZjLhXdW0yTUBiEP4lOBytOnOTeenX/0Hu8hZsTv2UlQ3GQrn3F3/+7sOj1ZMvy\nSScHV7kiWnN+3cvao6cYovlRVQQTm2nj7a07+TLAlwiHynAR4D43XBiYIkMYE3uGMY3qE3iF0fJv\nwdXSDWMeGsa0/R1gTfR1j3VTE/JFpA6ZgWHIP5cqdyV6VI3hm+MJPCKDSlXWrZHZNA4KwuuKhfvK\nzzsYrvnRRfFxH6uOB1OtifgIg5uML0crzcL6uIRbhHwLpfBbVvKfwW0hwWQqLg47C91ZQAAOqbFa\nzePB6k1LnR+fm8dnR47xHhXxUYrXcKgwUF+aGZscx3glsExlhKdQqadY2J2cQveY6ye2vwuu5Phy\nNds/7vENzpyM0gf3//7x/xOEfBHXymF2aBo/xJ1n67nzKIpCx+hKtIkI/0t8V/fVr8fI8wm8XJhC\nN80Lg1DYKvLZoStkUavupc7NLLJyLDOTKSKmzDgtsbDSlV3ut2ZFYviNNLpb+G8ie9q7fDGsO+cO\nxxLi7Um/6tWo5OP9p8fVKQoz2rbif5u20Ev60AgzqdLB12oO0SEVuL1Sab2LjXHnaI0XPlcYFF5C\npYJQsV6lPYUNDcP/ow/5SvdDhZdf441VV2SQXEnGfxD/KUK+iCuJucg/mNGr1iEK7LR1WXAhefP8\nDj7x9+aUlDGzAAAgAElEQVSdrp2uWtdfHgodDoqcTvxMJnegzttoYFnvHnx5/CRfnD6Dw6XRqlIE\nX9SuWWYbZnM5MSoqunLWZn3MzCONM9JKjDC5j2tSsk7NY3DVawdQbuG/h1+2buLxscNoaddTQxo5\nJ/K469hJJjVvQt/qVa97HCklmVYrRlUt5YpoExnBwl7dWHrwCO+npuNrNDKiViN6xESXCVRbnU4s\nUpTbpy0SA6vIoZP0Rr3sukRp56RmpWX4P6+NfrkrYueE9Rzfm1D8z6q/757/SUK+CMN7XxH02VTG\nv7OciDyNhwhDKMWLobvmy4sZySw8eJhxDev/5ljnc3N57Zdd7EhORi8E3gYDYxs1oF/Jorfo9Yyq\nW5tRdUvnHacWFvLBnv1siDuHXdNoEhyMogpOOK3UoHTK0G4KqObvx4zcZO7W/LhNWEjDwXKRjc7b\nRLeYqL9mYm7hpkBRUSGTxo/gf3a/YpdCCc/dIb15YvtOmoWHEnYVgazLsTr2DHN/3UeG1YpTSm4L\nCmJyy6buApCqfn7lZhz9nJjIR3sOcDgzE2+9nsZhIRxUChkmZSmyllKSqLqweJmZmZfEIOlHCHoO\nyEKWKplMbHIbZn3Z4qy/CxeJOOflFXyzKrv44EUy/pvx38gluQYS+j3DutQM7qJCKf+uKgR3Sz9W\nXIfQSXphESNXrqVSso0lIpqPieYRewXm7dzLx4ePXvO6u79Zhf1MBq9p4XxIJWolO7A6XLxEMqdl\nsVaFJiXbtXw+V7KZ2qYlr3Zuz55AwSNKAnOMWTStV5kPe3T5XZb8Ldz8+H79KqoJUyn/LhSnXLYT\nXny9d+9v6sJ8feIUb/68i7FFvnxCNEtFNLVSndy7ch0X8vOvet13p2N55octtM9UWSiimOoMpig+\ni1zp5G2ZSn5JMDBfunhfpOHp5cHiXt1o36A6H3hk84iSwLYKGjNub8ugmten/fJncXEuYh4axuLW\n8y+R8T+I/7SFDFBYkA8IKpQTKAvHQKrVyku79xDj5UmPmOhyI8efHDlKU5eJAcolv1JNPJgiQ5iy\n7wADa1YvlywXHDhEE4eJ+y6LLHcXvvhoKssMObykpaLTin1o/mYPZre+nZoVKgDQtMd/t5rpFq4P\nqSnJhF+lAjpcGvhJ8aAorBYDKnsQvuVXhBClgt1OTeOdX/fylAymqlLsIjMhuFP4kam5WHLwCE+1\nbFZmbIfLxaztu3hOhhBTcp0nKo/KIGaIZFJ8dIzOOUeA0JMuHbQLD+f91i0x6nSMqleHUfXq/PWT\ncQ1c/kBa3Hr+HwrG/VX4z1vIXt4+eHh4ECfLalIclkX4e3oR1etOdpvM9Px2FYfS0suct+1cAm2l\nV5njkcJAADqOZpTv8P/h7DnuoGxwpbnwJNduZ8md3XmvZxfm9+rKvQ3rsv70GWbv+pVTWVl/4JPe\nwn8NVavV5Kih/LS0vRQSFBqIv7eBiQu+YUKRSkCLGqXI6WxODnoNql4Wr7iIdnix9Xz5lvX+1DQC\nhb5UnAOK8567al6YFZX1Qwbweo9OLO7ZjRh/P97c9SufHD1G3jU0NP5qXL47WNx6fjEZ/z/jpreQ\nLyTGs2jeHLZs2oBA0L5TNxo3a8UP61eSdCGB6MrV6NZnIHO/+JKpdn8sJRVHWdLJfEMOTz73HH36\n9OHee+9l48aNPPrUU6zteyf6y6K+OkXBIbVygxV2JLorIsRSSnYlJZPjsLNLyydA0bkFVqD4KakX\nChKwGPTct2od/nZo7PIgR7gYffwk/WpU55Gmt/0dU3YLNwi2bNrIsoVziT11gqCQUHoPGIqCwg8b\nV6O5nLRs0xGrl5nvrDn0kt5ul9wvWh6nzYIf33wTLy8vxo0dy4RHHmF6kpEHuWQx6jy8cUgNKWWZ\nRgh2tDLrGopFi36KTyDH5WCfLKC+MJfyF+sRODQNH6ORn84n8PIvO2grvAjXdPwcl8r7e/bzbtdO\n1A0M/Fvm7Mq0tX8DCV+Om5qQz8WdYcSgbvTt05sP35+LpmlMe2463331KQ/cfz/97uzGvv37Wbx4\nCTXr12HMvl9prHpR5LSz35nPmHvH0bt3b/d4nTt3Zv7777M1PqFUSk/nKtGs33+K+tJcauEelUXY\ndVDzMpGV9KIiHly7kYICK7dLT05IK1+5znK/EkQ7pdhaPiSLsBj1hHt6Mmb1etpYTQwR/u79TB/p\nxxPHT9MoNPhWp9//KJYumMvShe/x2MSJ3HbbVA4dPsyzzz5L9erVGT5sGHq9nm+/W4lDwMZgIxuz\n06mu6Tluy6HA04P5ixbj5VW8q9PpdDwxeTIDBw/mkd2nUBQF+wP9iPL2xmw0sLeokNtEaXGt9SKP\nzpVL5wV/H3eOZ3/aRl3FTBPNzHyZhkAwTQ0joETX+ycln7bRMSTk5fHKLzt4hXAiMYICvSTs1PJ5\nZMOPrB86sJTR82dwpY/cXbDxL8RNTchvvjqdkSOGc//48QCcOXOG2NjTrF61iuCSNjStWrXijs6d\nGTxkKEu+/Z5DB/bwzfJPmNixPaNHjy4zZo1atUg8c7rUsYE1qvHtiVPMKUylr/TDD5WdsoDFIoNn\nW7Qslcv85PebqZ2nMJyI4owOBeKkjaddCVTCQCYu5ihpPNm0OYn5+ZzIyORJokrdz1uoDNB8WHLg\n8C1C/g8iKzODd2a/xJrVqwgLK04H+/TTT2nbti2vz5rlNgo6duzIa7NmkZCUTv/BI4g7e5p1Tz/G\n0e17MV5RYh8VFUVRYSFFhQVYPIuJWgjBU62a89SPPzFM86eF8CQfF9+Rw3Gjg6m1a7qvP5+by7Qt\nPzOTMKpIE6jFO8HPtUxediUxXQ3nW7I5qrczo0Z1lhw6Qge8ibxCaa6Z4skKVw4bz8bRvXLZXPzf\ng8ut4VKW8P+jj/i3cNMSssPhYPMP63nlhefcx7759lv69evnJuOLqFatGi1btuTQgT30HzyCzIx0\nYs8cL3fc/Xv30qpS6YonT4OBxXd2Z96+g0w9fYYCl4MGgYHManQ7TS9r1xSbnU1sVjb/o1IpSzpK\nGOkmfHhCSyDc05Onm7akU1Qlfk1OJkw1otfK+kIqCSMfpSYw8tvVzGjfhko+3hQ6HCw8eJiVJ0+T\n47BT09efQXVr0qlSxXK3l7dwY+LH79fQpk0bNxkDfPPNNyxevLiMa2HM6NG0aduWma+8Q7OWbVn8\n0bscOXKERo1Kdz0/cfIkXt4+qJNHYC8Zw5UcT6uISOZ07cSHew4wPzUOs6qja0w0SxvWw9d0yUf8\nxdETdJJeVLnMbyyEYKDizypXNiO0s7SLCGdJiw74GI0k5eYTJfXluvnCXCrPbPmZI6npPNqsMTpF\n4XhGJvP2HmBHUhJ6RaFdxUhG1Ktdrvby5RZxuRV0/2LctITscjnRpIaHx6Vc3pzsbKJjyn/qhoaG\nkJ1dnObSd+Aw7ryjGf369qVJkybuc7744gty09Np0aJpmet9jEYmNW/CpOZNyrx2EXE5uVRRTOjK\nIdhawoNTfgqLend3/6gqensT77JiRSvT7ua4LKKhMFMrS3DPqrV81rcXj274Ee9cB5NlAAHo2ZtZ\nwHObtvG0quPeerUZX7/2f0Y162aG3WbD6wpxq+ycHEJDy2be+Pj4oNPpKSoqxGgycdeIsbw4fTqL\nPv7Y3YTXarXy/NRnGeTn6S7KkCV/zuR46gBzGtZADel81fd0NjOLVtJYhmBVIain96Rzs9r0rXap\nECXa35cTCdncccU4UkrOShsTlSC+PxXPyy4XPatV5qF13zNQ8+UuEYnVpfFdbBqDTn9HsG8AM1o2\nonFISNlsCfjLKuj+Kdy0hGwyeVCzVl1+3LSJOzoXL6S69eqxdu1aRo0cWepcKSVbt23j6el9AQgI\nDGLW2wu4/8H7aFSrFlVq1mTPzp2kJMTzbvt2v5vUnJpGamEhFr2OM04rmpBlqpjOYcfDoKPQ6cRS\nkgQfZDbTPCyUhUnpjJOB7muSpZ3lWhZPqaHUFB6c1Oy8smMXap6NJ2Som9A7CG8qCSNPksaS2DyO\nZf7MOx3b/P7JvIV/FRo3a8X7b7+KzWZzux7q1q3Lli1b6NatW6lzDxw4gLePD94+xZbkXSPHcDb2\nJLe3bUu3Ht1R9AbWrVtHx/rVGRXg5d7iB7/2mlsHA0rLDVyOPIeTXIcDP7MHZ8mmJaWzjS4SrCZl\nqeBg3+pV6X3wMO2lpVSe9FqZgx1Ja+FFI2lhdOwZDqemMUar4I6xANyvBoPUsUMXxegNW5jTsBo9\n5y8oW9J8g0HIcpp0Xg116jWUX67c/Pe9m78YWzZtZOqTD/LG66/TvHlzrFYrt3fsyLixYxkxYkRx\n8MJu58233uKnLT/TvFkrtm5Yi6KqdOjZh1btO5O47GWSTl2gclgE7SpGugMN6UVFrDh5ioM5uXjp\ndPSsGEmr8LBSW0aXpjHz5x2sO3MWnYRCzYWKYIQSQC/l0lYrXTqY4DqHr6onExcj69RifKMGCCHI\ntdmZsP574rNyaKJ5kIfGr7KAUUoA3UrG2KblsVjNYqTLj9ZK2fS7h0nD2GASJ/fN5JPuHakTGFDm\nnH8D6ixYvEdK2fgfv+8Ntq4BHntwFJqjkOnPPUdQUBCrV6/m+RdeYMnixVStWmyJXrhwgdFjxtKk\nRTsSThzn7JlThISGc+fd9xAYFMzhQ/vR6/W07dCZqtUu+YP379nFis+XkJ6eQtVqteh05z2ERlxq\nXrpqq0rq6V18/8mDpKWexoKgQLrQoTBHrViqMe9GVw7zZRqeig6Th4EXOrSlflBxBsX2xAtM/mEz\nkS49lTFwVFopwMVUNZzwkl57T4tETrusfCJiSpVUA8RLO5PVbEJqjiPpzFLG/u/nv22+/wxmT4m+\n7nV9UxMywPfrVjL71elYiwpxaS48PCyoOhW7zUqVKlU5euQwERWjOXfiBG1setq7PPhZy2OdzEUI\nQBFU8bQwpX1bapS0tTmekcn4Hzdze+fOdOjcmdSUFJYsWEA9TzMT6tXlyxOn+SUpg7NZqfjYJU8o\nIUQJI1apsUTLYL3MobrOlxaaSpJ0sEnmMlDxp5/iT6p08Jx2gTvr12B8iZaslJIFhw7z2b7D9Je+\ntBBe+FyWJrdGy+YbXS5jnP40UcqWwj4u0lEbPklK/Brq6c8yp9NfE2G+3i7H16usd4uQrx82q5U3\nXpnGtys+o0JAABnp6VSMiiEhPo4qlaug1+s5evQoDRs24djOHQy2mamIgaVaOsew4q0zYtMJevUZ\nzKTnXsJkKnbtzX3rVZZ/toiRI0cQHRXFrt27WfHV19zeZzZ6oycHd35K6oVjFKQeo4fwZZDwwyJU\nEqWdaVoS2bjopPgSLgW7tALOY2OGGkFFDGyX+bxNGiv693briRc6HIxcuYaAHBcdhBe3CUsp4n2Y\n82RqTpaJ6DL+8VTp4CGRRv3OK9i+tiv3TvoJH/9/TsXxevF7CPmGdFkUFhbw8aIPWPPdcvLz82jQ\nqCn3jH2YWnXKiut06tqLjl16cv7cWYQQRFaMQgjBsSMHSU5KJCq6Cq8/9xT1Cg0MEL6skdlsl/nM\nVMOpIkw4peTHvFxGr17HJ316EunlxdM7dvHk1Kn069vXfZ9+/foxoE8feqxYiX9oe6zChM2RyEyl\nkrsK0CQUxqqBpLs0TgTU4dOsY1Sz2ZmlVnRbBEFCz5NKCJMOHOa+BvXQKwpCCIbVqsWiA4epoplK\nkbFdaqxR82haMZytcek0oTQhp0sH57RC6vvVJuncdyQV/vkW6+X66q6C6Q02ceadZe7//0nZ0xsR\np04cZeGHb7N758+YzWa69OjH8HvG4+XtU+o8o8nE/6a9woRJz5KclEiFCoH4+vljtRaxd/cOnE4n\nUdGVGdClFW87Q6ggdDzhiidGGHlSCcMblUyHk3lfrmDixpW83aw+x3Ly+PzoKVauXUdAQPEuqlOn\nTnTu1InRY8fhdJkIqtiHvMwz1Dd4c4+rtHbxm0oEI7Tz7Alrzsb4tYwQ/jythLnjHy2FF4dcRbz0\nyw7e6dIJALNez30N6jPv59000kqT8RFZRL4q8TWYOGQtot4VJeCbZT6+QU3QNAdCKGSlx/4rCfn3\n4IaL8FitRYwe1ocj+3cwY/o0lixaSIO6NRg7oj/bfvqh3GuEEFSKiqFipUtP2Zq169GhUzdCwsL5\nefsWuuONU0o+0zJ4Ug1zR4t1QnCH4kNX6c2iA4c5kZlFAdDnsvxkALPZzJj778fsH0VkncfJyThA\ndbNvuSXZdwhP1KI0XC4rD6jBbjK+iErCiEHC6tOx7mNGncozrVswQyTxjZbFWWlju5bPFOUCNcOC\neLxpY47p7HyiZVAoNQDOSBtTySAkZiAuZxE56Xup6ef3h+f+YmVThZdfu+7Kpmn7O5Q693qt6v8i\n9uzezqihPakaE8niRQt55aUXOR97hJGDu5Ofl1vuNRaLJ5WrVMfXr3j3ZjJ50LJNB9p26Mz2n3+i\niepJkNCzRxagIXlACcK75IHuL3Q8TggncvI5rvNgVW4Rdw0f4Sbji2jSpAlRlSoRUWUUJo9g/M0e\ntHWUrd7zFCp1hRnF4EW03pfeql+ZYHQL4cneCylcvjPvHFWRsCA/nhEX2KXlc1baWK5l8pJI5tl6\n1XmwWiVmixT2a4VIKXFIyUYthy9FAcHVRpJ8bhU6vScWr6A/Nf//BtxwhLzisyX4elt47913aXzb\nbURFRXHvPfcw+43XeWHqJDRN+13j2e12VCHwQJCAHTNKmS68AK3xZEfiBbJtNsJDQ1EUhYKCAg4e\nPMjZuDgAIiMjURUdLkcBqiKwi/LdQVY0QKDXmcnEWfY9SQ0bGm/t2lNq4XaJjuK97ndwIdLMbFMm\n6/0cDG/eiFdub4efycSSO7uTFGLiblcsg51neEpkoFQZhMm3Bod/mYBAsj8jm+FrN7Pi5Ckcv6PF\nzOUlpn80cHKLlK8OKSUvTpvMzOnTGT9uHNFRUdSvX583Xn+dqlVi+Hjxh797zMLCfLxKltcBWUgr\nxavMtl8nBC2kmZ0Xksh2OIiILLYwL1y4wIEDB9yZR1FRMUipYStKwdvHGxvl/86KcKIoJrKknfLc\noZk4EVIyb/9B9zFVUZjd+XYGNq/PSks+s/WpZASrfNC8AaNmTCRp0CqaDXqTWcYCBrpiGaydYZmn\nN1G3TSMjeSvxpxajqDrWr3iaNZ8/RlL8vt89V/8W3HAui/VrvuH+caPLLKxWrVqhqArHjx4s13Vx\nNXh7+xAaHMaBC4UECj1FSDRZNguiCA2DolLF15ejP23lxRkz+fKLLwjWm8hy2AkJD6NmwwYYzFXQ\n6T0Rip5kl53T0loqN1OTku+0LDJzUjF7RfOZLYlpSulg4FqZQ008SLTbeWn7LqZcJuBSJyCAVzu2\nL/ezhHp6MqdrJzafj+fRTduQQuH86aXo9F44HXmERHTBEtqBIkce7xz/gq9Pb2b+He0x/oaw/eVk\n/GexuPV8txvjlvviEuLPx5GRkUaXLl1KHRdCMHz4cJ6d+hzjHpr0u8Zs3LQVS9VXuc8l0SMokuWT\naKGQGFSVGt5ebFi7hhUff8qxY8cIMnhwwVZAt27d2L59OxE1u2MrSqGwSGWjh53OVlnKxXBB2jmt\nFaLEfo4iBLtlAU3FJReaQ2p8q2XRS/gyb99B7oiJIsqn2BWjUxT6V6tK/8tS45zJ8XyqDAcSqFq3\nO5Vrd2X1Zw8Td2IrWQUJZO+djqKYUFUT4ZWHY/GOIS/7ON8sGkurLo9Rr+nQ3zVf/wbccIRss1rd\nJZ+XQwiBl5cXVqv1mtdnZWaw+rvlpKYkUaVaTbp0682DT03jlckP86TNDy8UdskCmovSvtg1Ipdu\nVWMIMHsQZvJg28df8hYhBDr1uKRk89k83o1dTsW6T5KXfRyj8EKzp/KsvEB7zZMBwo8M4WK5lgkI\nPlAqMinvLIeRPONKoJfihxmFrTKPnTKfF9VI1mrZrDhxkjEN6v6ufmLtK0aycVAflp84zZHMPA6m\npeNb9SFCKvVyn+Mf0orTuyfzxYmTDL+s4upKuP5CMr6Iafs7MJJlf0lX8JsFNmsRFosnSjkpld7X\nsa4BDh/cx6bv1wLQ7vYu1PjwBaJNOubYU7hdeDNbS2aA9MfjMjdCtnSyg3wmV6qElBrvLv+GIUoF\nniAcvUshV/rx1srvKRJ2PDwrkpt1hMz4eHSKYDznGaH5U0d4cFAWskRLZ7QSiAWFt1wpvEwhvaUv\ntykWMqSTb7QsgoWeAcKP5TKTF7ZuZ17Prtc9R4qi0Ouud7lwfi9H9qwgNyuRpPN7adB2EQZTsQqi\nt389/INbsmXN/VSpdQdmzwrXPf6/ATecy6JJ89asXbuuzPGEhATOnztHjVpX7ym3Yc23dO9wG0f3\n78DbQ2H1Vx/TrUMjKletzoTpL/OydwEFJj2ztGS+0jJJkQ5ipZU3SSHeQ3JX7Zrk2Gwk5ObyDCEE\nlqT3OJDY0QhBT+KhVzm1fSIDCq1M14IZQwA7ZD73a+d4w5VEZWFkhhpOsKJnlBKISVXQgJUym2Va\nOj6ovKVWIkIYSMNJFWFk3dm43z1PAR4ejG9Ql/81rUeR00lwZOn8VCFUAmOGsfz0td0HkuJuCX81\nLhK865b7AoComKrk5+Vx4kRZ/e2169bRtMXV88edTieTJ9zHo/cPR7PlIO25PD5mCE8cPsms7l2w\nVKzAiyIZTYHHXef5VSsgXTr4RctjirjA3bVrEeppYWtCIvUVMwOEH/oS0s7DRYjUobic7F/XE+XY\nAh4ngGdlEC00M29qyYx3xbFB5jJeCaK74ks7xZvq+mIDIgMni1zpbJJ59FH8mKyEUlDiyjuQlkaO\n7dpB5qHa0jLHwio2onPfFwiNrE9QRDc3GV+EhyUC/+DmnDpSlif+7bjhLOS7R41j8J0dqFw5hv79\n+mEwGDh9+jSPPT6JkaMfxGy2lHvdhcR4nnt6IsuWLqFWrVru4199/TUPj72L1T/uoVe/IcSeOk5c\nXCzrv/qCJzdtwENvoHvVGGbUrYOXwcDOC0nEqCb8tOKpy5ROprjiCRMGBin+2JCs07LZKwrogQ9V\nFBNNhYUHXed4Qg3FC5UvtUzOYMMAKKpCrGbjPaWSW4AF4Ky0cUAW0kV6k1lUxPncXLwMBvxMZYMp\n10KuzY7J6I1Qyn7VBlMF0q7xg7joqjj+N3VLyHl5BT5P9f9bxr7RoNfrGffQJB5+5BHemj2bmjVr\n4nQ6WbVqFUuXLWPZl1d/KC744C2yM1LYuGG9u1BkwsMPM370aD47fpIXOrRlstVKYl4+e1JSWH4y\nlqSCdCp5eTGxfjPuiI4CYE9iMi2kxV1tt0XL430tlTuENxOUYE5JGxu0HHIUC02EJ9VVD6poJlZo\nmcwUYRzEyhuuZPJw4UKjRgV/bJk2XlMjS7nkVrgyaSm8OKhaSS4oIMdmI9hsKeM604VEFru2egwo\nt/zZWpSL3li+BawzVMBWlPM7voF/B244Qg4Ni+CjZd/w/NRJzHr9dXy8fSgoKGDUmIe5Z+zDV71u\nxedL6X3nnaXIGKBf374sXbqM7ds207pdR6rXrEP1mnXo0q039gf6ldlSm/U6cqTTXXU0T0ujufBk\nlHpJLrCT8GamdoFvZTYDhD+eQqWT8Ga5lsl+WUR74UVX4cMFaeewwwoCHnadY4DwJ0oYOS6LWCNz\neEAEslhkYjt6gu+OnqJAc9IgMJApbVpQ0fv6mlRG+Xhjs2ZhLUzCZC5dWpudsp0GQdcuEvm4xzd/\nW/npN6uyGQm3XBclGDpiDEII7hszFoNBT0F+ARWjYpi74EuiYqpc9brPls3now8/KCUYZDQaeerZ\nZxk1ZAhj69XB12TC12SidmAAI+rUdj9sL4+UmA16cikAIFe6eFdL4RU1kqiSIHdLvOgkvXncdZ6G\nwkIFoaO18GQBabyhpXAKKz0VXwLRs0PmsyUjAx0KT7ni6aH4oQM2yzzOSxt3iQrsdhUw4rs1eKs6\nCqXG4JrVeeC2hmV0V+5e3adcl1lYpYbEb1pCeOXBpY5LKclJ30Vo+zt/z/T/K3DDETJAtRq1WfLF\nWlKSL1CQn09ExSgM5XTyuByJ8XG0blF+bnbt2rVIOB9HclIiiz96h83fF2912ugl93j5E2y5ZHXX\nDghA0ynssxVSAw9+lQUsVEvLEKpCMFjx521XiruLiJ/QsUbLZrISSiPl0ngdpQ8TXOcIRcfnMgN/\nqSOf4uyHBTIdT1Smy1AihRGb0FidlsOolWtZ3q83/h6/bS2bdDqG1arB8v3PEdPoRff2Ljt9H6ln\nlvFq1/bXvP7vFmbZOWE9zd7u8tsn/gcghGDoiDEMvOse4s+dxeThQWjYtdX8nE4nqSnJ1KhRo8xr\n1atVI6ugALtLY09KCstOneZUVjZBRgO9A/14cGgnkrddajHWo2plnon7ie7Sl60yj9uExU3GFxEu\nDLQSXmyWufQX/ihCYEbhKEW8o1bCXJJS1xxPGmsW5mupJONggZaKHkEOGmZgjkyhl+LHAOGPWSok\nSzvvHIsjq8jKtLat3PfThUTiSo5n5Lb7ynR7rlK7Cz9vmE3C6Y8JixmMoujQXDbOn1yA2eJFREzz\nP/I1/L/ihvMhX47gkDBiqlT7TTIGiKgYzeHDh8t97fDhI5g8PBjatxM67Myd+y5z33sHQ6t23LVu\nAwl5ee5zFSGY1rYVr5HCJ1o6JkQpcfmLCEVPzmUpbVtkHgZEKTKGYinNXoovYcJANWEiDSdmVFxI\n8tCYoUS4JQqNQqGf4kcDl4kvj/92r7+LeLBhHe6MMHH4p2Gc3n4/x7YMJ+XQdGa1beZuCXUlLuYb\n/9246A65lQp3CTqdjujKVX+TjC+eGxwcyrFjx8q8duLECfzNZpbv2c1ze/bR6/4H+PSrr3jkhRdZ\nJVXu/eBblOBL92gcEkzTiuE8JuM5pBUSSvmNRUOFnpySnnip0kEyDvoKPzcZX0Qr4YkJlTEiEBuS\nLMc9uewAACAASURBVFz4oyMXSTU8GKEEYC7xVYcIA1NkMOvPxpGUX1BqHDUkEl1IJBlPTWbktvt4\nWpld8tmNDByzDHvhAfZtHsqx3ZPYs2kIOiWVfvcsKJOJdSPghrSQ/wj6DRrGgB5t6d+vH3XqXOrZ\nteKrr0hJSeXt55+loDCfres3EhoczLDhw5kwcSJHDh9h8FcrURVBw6AgRjeqT4vwMBb26sq9qzdg\nc2rESzuRVxR3HJCFRAkjDin5mizS9BrVXeVnSgSjZ5PMxYjCO2olQoWBXVo+X2qZ5RaWtNIsrDyf\ncF3dsKH4IfJ44waMqVuLoxkZeOh01AmocFWRpIvk+E8Jtdyykv8cho4YzUsvv8K8Dz/AVBJjsNls\nzHz+BSrWrMtbu3di9DCxZsXXREZE0K5dOywWC+NH3kObpZ/iqdfTvUoMo+rVZXq7Vszx3MfHR46R\n6HIwnLIurQNaIbcr3qRLB7NEKoEmExH2skaREIJgdHwg0+gpfOmv+GMSCjNdibQWZTOlzEKlgbDw\na3IyvapULvP6RbdWwpbDjOQ+oDgOMWLYes6fPcWvh9PJy/KkQnSDG0py83L8Zwg5LDySGa/MYcTI\nUbRr15aYmBh27d7N0UNHMNgcDHd4UVeEcyHJzuez57Jt02aSLlwgLDmXGYTio6nsSCrg/rUbeOX2\ndrSKCGdRz64MX72O1x1JPK9GuC3lC9LOPC0NDwTDtFjqBATwYqO2PPX9ZpxSorviyb1fFpCEg3lq\nNP4Xy6xRcFJ+YUkRGsY/0GHa22igedj1NUf9J1vbHN+bQNlWmbdwvRg15mFOnThKp8530LNnD4QQ\nrFy5CmdBEVWKJM8r4fjbVH7ddZoH77mPofeO4uMFixjo8KSF8CTP7uLbY/GMjDvP0t49eKRxIyxG\nI+/v3svXrkx6K34oQiClZK2WwzGKSNMcvK9oDKtVi0KHnQOn0mhI6d2fVWocw0oVjNytXiJ2CypF\nVykssQr5m2v7IjG70i64g8J1S/4AKKfR+7+5S8jluKkJOTsrkw/ffZ21q76isKCARk2a8cqb80hM\nOEdqchJdew7m4K9PMssZQohSvD3zR0d1uweP7NmPp1R4TLskZ9lT+BKm6Xlh23ZWD+5PNX8/Ngzo\nS6dPv+Be11kaCDNWNE5KK72FL2tkDia9jg96dEGnKNQIqMC8jHRGywD0JWPu1PL5WeYThdFNxlCs\nj5yBk5PSSrUrCktWiRwGV7v+4pfrxcUOCzEPDfvHuyosbj0f1kT/9om3ABSXWX809w327t6J2WKh\nW89+vPjGB+zZ9QtSSnr1HcKeRYuYogW4129X4UuUw8izcz9gEsE0U4tz7QPRM1Eaea0ohc+OHWd0\n/XqMrlubfJuVTw8eZa0rh6rCxFlpQwAD8OMLshhVuzYPN2nE+dxc7jq9igaamQZK8S7QLjXe01Lx\nQeV2pXQAupXw5BMtgzuETynjJFHaOeYqpFVE+HXNgRoY9tsnleDMO8sYSbGmypW+6H8Tbmgf8rWQ\nm5PN8IFdseZnsHTxIn784Xu6db6dZyY/QGhYBI9MfhazxUIdvRchorSvTC8E3Z2emB1lmzs2FGac\nDiensopLSmNzczEqKirF0d2awoORogI/yjxsaAiHxg/nzgPwaqf25AaYuEeeZaozgYeccXyopTJC\nVCgpp74EnRCMVYKY7kpknZZNunRwXBbxjHaBLAN0j/lryctZQsaLW8+/ISyJ/zI2/7COieOH07VT\nB3784XuWLl6ENT+D5599nGGjxvHgxKc4+MtWullNZdZvDeGBn1SwXKExIYSgu/Rm/amz7mNpeQWE\nokND4oGgm+JDY8wsJwsPBJ8ePY5D06jo7c3rnTrwtj6dh7RzPOdKYJTrLDY0qmIiX5Yu0W8sLAQI\nHVNc8RyWhWRIJ9+7cnjcdZ5769d164H/ldCV+KEv90WP3HbfX36fP4ublpA/XfoRtWrW4PmZM6lc\nuTL+/v4MHDiQ2bPf4JWZU9A0DZvNiocs3/FvQiDL0aIQQmBCxVaiA7HxzFl64sMstSKqEHynZbFa\n5tBd8eEJEUqI1PPWzl+xu1z4GI3M7X4HS/r0pF6dKAqNClJVWKHkkIqTw7Kw1L1aCk+8UVms2hhH\nMtO0FI7LQvJsdgas+JYNf6BgpDz8laXRt/D3QtM0XpnxP2bPfoNBgwbh7+9P5cqVeeH556ldqyaf\nLv0IKO4qYiqvPxLFqoPlucNMKNgv0zf54Xw8zyrhjFaCOCKtfK5lEoedSUooQ0UFpEtjyaHiQHmz\nsFA2DB3Ikx1bYwr1waXABZ2LXUoB35GD/bKybVUIRooATmDlRdXKOJnAXJmOQDBv/0EmrP+BpPz8\nv3La3MJYF9d6zssr/pXr/aZ0WZw/d5ZPl8wjJyebGjXX0LJlSx5+6CEaNWpEi+bNQWqcOnGUpi3a\n8Kozl0Lp5Y72XsRGmVtujDle2skSTqr7F6um5dvtxMsitml5pOOkBibGqUGElgT5GkkLz9gS+erk\nKYbULE5NivH1YVLTJjzWpDFxOblIKYnPy+OZTVsZoPnQRHiSIZ18qmWQY/QmusnznNo+kfHCnzaK\nFypwsKiIl7ZuB3An9v9eXFyc/+Yt3C2UxqKP3iEnJ4uRI0fi6+tL3z59ePjhh/H29mbokME8/+Ir\njHtoEi07d2Pb2QU0cJb266ZJB/HSXkaFDWAr+bSIvOQusGku5pHGSWmlCI3+wp/eiq+7iq+RtPDY\n/kMMqFEdH6MRnaLQJjKCNpERFDgcJOTl4Wc08ur2XfwvMZGhmj/hQs8hWcRCLZ2QmEHoUBBxq5is\nVqCSMFIoNb5NzmLEd2tZ3v9OfIxlhb6uhsubmpaHnRPWXypyWpV93eP+k7jpCDkhPo7hA7ty911D\nGXb33ZhMJlavXs248eOZ8/bbNG/eHKPRiNPpJDyiIp179OGltet50O5NiDBQIF18qeSQXcGToqIi\nVluzuQNv9ELhtLQyW6QyrmE9DKpKjs3GzsRkYlB4UA3GA4WtWh6TXfHMUMOJESYUIeir+fLt8VOY\ndDrWnDhNakEhVkVHkKcXPaJCCTJbEAJe79yBL48eZ2VyMhoSDYHVnsWpXyYwFG86XOaLqy/MPKIF\n8c6uvXSOqvS7UnzK6Bmv+ku/glv4m/DJ4g/5bOk8Zs2aRZvWrUlKSuK9995j2PDhfP7ZZ5hMJpxO\nBwCDR4ym/5KPWJGXRQ98MAmFOGljtj6TZo2b8vbBozxqF1QVJhxSY6PMZZOaz2f1it1Vm8/HY0BQ\nHRN3qwFkSCcrZCYHtEKmKeHohCBMGGggLHx18hRZhVZ+TUwi1+EAg4na/v7cUSmURJHPkNo1ORsR\nxoqjJ0gqLEQVAuFSSYv7FqnZWKDG4FcSPzELhaFUINGZwtcnTjGqXp2rzkd5qZLXtHr/porTvxI3\nHSHPe282gwYO4OGHHnIfGzBgAGazmVmzZvH888+TlZ1NtRq1AZj66hzmBMzggQ/n4OdhId/loF3r\n1sx7/DFeePFFluzYyXxXGgYEiqrwQKMG3F27uNpv0cHD1HLqmahe6mIdpRoJ1vTM09J4SS2OBhsR\nnMnJ5cvtB+iieWHEi+9FIfsLUjmclECkMOCj03NSK6J/9Wp4G7MxWTWauzwoUDRWu7I5K2ylepIB\n1BceZFpTSCsqIqhEfEhKyc6kZDaeOYvTJWlTKYL2FSMhNbHUPP0bt2u3cHUUFhbwzuyXWLH8S6Ki\nooBiudcXX3yR0WPG8O2333LqdCyt2nYEoEJAIIu/2chj9w7hk7hYfAweaHqVsfc/QHTlyjw95Wmm\nWBNRJTjQiPL2Zn7HroR5euLUNGZs/YVnlTB3v7tIYaCu9GCKK4GfZB4dRbFx4NRcfLBnPx2FN/2l\nmTR0fFmUz095RXwfe5LaqoUsxYVTr9CvZjUW7j9EK+FFlKbnuLCyHTvnpN1NyBfRRrPw/bn4UoSc\nb7fz3elYjqWm4yud9IoMJcazeAdws6znG5qQCwryEUKU0q/Y/P1avvzi8zLndunShSlPP8248ffz\n4CNPoS8JHOh0Oh6dMoOzcbFUiYlk3NixbNm6lbuHDaN/v37Mfe89rFYrO3fuZPYbbzDv4GEWHjiM\nS0ocLhczCSvTabeD8GaBlkaWdOIndHwqM6klTEyRoShK8cnN8GSJlk6sUJiuRoAsFmKZeOwE7YU3\n94pgRMm5fVQ/nnLFs1nm0UFcspJdgFNq7j5/Dk3jsY0/ciY1g44uT/QIPjiXyHyPPcxt3pAVHRb/\nldN/C38TXC4X+fl5WCye6HTFP9Ffd/5C9erV3WR8EUIIBg4YwJx33iEjM4vPv/3R/VqlqBjmfbmG\nru0a8u6C+URGRvLGG2/wzty57t1ifEICK1asYOH8+Uxcu4FcpxOjqsfDKUs1H4Vi329PxZcfZC4d\n8cauaRzQCnhCCaWJ8HT/DtpJbx52nuM+JYQWwgupSX4oyuW9Pft5VY2kCiZQoAdwSPXmJVcS80V0\nKRU6Kxr6y9LfTmRmMn7NBmpKE/VcRlKFi/vO7WFUlUqMrFKJmwU3JCHv2r6VOa8/z5HDB5ASbmvS\nnEcmT6Vu/UZomoZaTh6jEAKn08mI+x5i0N33lHl94hPTGDm4O9u3byc1JYUqVarw1FNPIaXk5Vde\nYd26dWC109hh4k58MKPwiOscxxQr77tSicdOBXTcofjQQ/hiQqFIahyReZyWNl5SIspoLA9Q/Bjp\nOkuh1DALBR1gk5KhSoVSlrBZKNylVOAbLauU2+InmUt1Pz+34NDi7b+Qk5zNm0SiLyHzntKXN62Z\nPJQRzK3ciX83nE4nH82dzadLP6KosBBFUejdfwgTJj2LlOWvawChKOTl5bP4s9WEhJZOGfOvEMDY\nBx/j0ccex2azYjQaeW7aNFq3bs358+cZO3YsUtMwaNDL6U1jYeEXRz4/arksEen8JPMoQqOmMDFA\n8ccTBavUyJcuXhXJ+At9mT6OHkJhgOLHVplPC4pF8fOERks8S2mDA9QVZmpg4heZR0dRrI2sScl6\nNZ/BVS/1lHx84yaGO/2KU+hKeLs73kw8m0xyp9cJ+Su+gH8Bbrgsi13bt/LYg6MYMfwu9u/bx769\ne7izRxfG3zOAY0cO0vb2O/jq66/LXLdp0yZiKlfj7lFjyx03pko1JkyeSlZWFi1btaJnj+IE+6XL\nlnH8+HH69OpFU5eRCSKIKGEkSOgJRMfXWhb9FH/eV6MYrwaxXebznJZIAS6eEYmssBSgKILAckKE\nZqFiQlBYol2RhYsAdGUCjAAVhYF47CRIO+nSwVdaJgtFBo9Wj3JHj7/OzONu/N05zlD8IBqp+XD8\n4Eqczj/fT+8W/j5MnzKR3b9sYsmihRzYv4/Vq1aSk5HM/fcMpEGjphw5coTExMQy13311dfcO+6R\nqwoQjb7/UYJCwhgyeDBJSUl07doVKSUPPfQQQwYPpiArm6kymK6KLwFCT3M8ScROGg6eUcN4V61E\nE2HhBdcFlmuZpChORnMOxd9MqFp+0C0QPXlcythIlw5iyunEA8Vre6csIFs6OSWtvCxS0Hmb6BYT\nBcC+lFSwFzdBvRwBQs+dmpmjv5SV6LxRccNZyHPeeIFp06bSs0cP97GBAwdSWFTEB3NeY+IT0xg+\nsCsWi4VBAwdiMBjYsGEDM2Y+z4uvv3/N4Neu7VsYM3o0p2Njyc0t7mG2bNkyXn75ZSY9+DCTnJek\nCbOkk2ScvK9GuQs6fNAxQzEx1hVHgMXCO3fcToyvL50++ZK9joJS7gYo7nmnIPAr+RoC0ZGOkxzp\nxOcKn9pxacXHqGeqdgGH1Gge4M/yuZ+w+ngT9pSck7S+GpGU3b75Cx0qYLfmo/O8/qj1LfxziDtz\nms0/rGPzph8xl8QDwsLCeH3Wa/Tp24/9+3Yz+v5HuXf0aGZOn06TJk1IT0/n/Q8+4Oy587zU/+rd\nMQoK8jl+9BAL5r3PwoULyc3N5Xx8PFabjfr16+PjEtQQHu7zt5FHPcw8poSUKioJw8AMLZEx9esx\nsEYNUgoLGP7taqyqViZrY68sKHZNlCBCGPhVK6BfOSbgcdWO3azjwcJ4fAwG+lSvyvC6tTGU7AhS\nCguJkLpyf7uVpJ7tWf/+YN314oYi5IKCfI4c2k/XLovKvNand29effVV3nx/KQs+Xcnbs2byyiuv\nomkalaKiee7lt2ndrjjgIaVk208/sOqbzynIz6N+o6YMGDKSvNxsgoKCqFa9OpMnTWLUqFHExcVR\nr25d7A4HJi4t2l2ygCbCUqq6DkAvFHoIH74uzCbDasXPaiPT6eIjkUO0NLrVszKlk9ddSfQRvu42\nOBahEoGed7VUnlBC3VVM6dLBJ8YCmg+aS3T19u57rT5+6b6F+RlYDJ4csxVxmyid6hQvbQidAZNH\n6c7Ft/DvwbYtP9C5cyc3GV+Eoijc2asnW3/cwNMzXsPPvwJTnplKQvw5FEWhRev2LPp0FRZLsdsg\nNyebb5Z/wu4dWzF5mOnasx916jXCYDTi5eVF9+7dWbxkCdHR0dStUweb3Y7pClGgX7R8hikBZQiw\nrvDAF5XPjp5gTIP6rIw9g9EUwGxHJo/KYp0KKSW7ZAE/yFzeVi8ZB02x8BFp7NTyaXaZi2OTlkuG\nQWNN316lfMZQUqwkJdbMNA5rRbhE6ZZRAIcVBz5hpSV1b2TcUISsKApSFgc9LgY7oNj3dvbsWUDw\n85YfEYpCZMVodDqVqKjKFBUV8cTD99GiTQdee3s+M555lEP7djNyxHACAgLYsHEjvbu0oFPXXmz+\n6SdmzphBs2bNuHvYMAIDAzl+4gQtW7Vi2/odDKI4v9iJxHjVxHuVcE3P6pOxDKxVA0+PCvhE9Gby\nqSWEaAp66SRO2lFEccsmf02HEYUtSh75BggN8+HexCSa40G+gN1aHk3bTixFxpfj9NGNrPlsIprm\n4n1pZZY0ui1sq9R4V82lXosRKOoN9XX/p6CqKk5n2Ya3eXl5pKalkZ2TzZZNGwgNi8Di6YmHhwfR\n0dHs2fULvbs054mnX6BRk+aMGtKLhg3q0ffO7uTm5vLuG88THhmN2WzhwIEDTJgwgUGDB1O7dm3O\nxcXx3HPPcc5RQJr0cnfAcSIxlmONCiGwoFLolBxMS+NUdh7msI7E5p9jePpeKms60qSdHAEuJF9p\nWTQRFlJwsFLk0Cawwv+1d5aBUVzt375m1uLZZKMkQLAEdwhuwSVYcQ3BixWntLQFCrS0uJQWJ7R4\n0OLF3UIJFiwQJUKI7W7W5v2QEkhDedr/W4F2r297Zs/M7M7Z397nnFtY/jyZPVI6fpKSe0IOaSpY\nXL0CQnJ8gXK/ZkmiW4w90XfjUAoK1kvP6I9r3h/FXUnHITGbrnX6/unP459CeF1l2N+ifMUq0tY9\nx/+6u/kdDOrTkTYtm9KtW25S6h82bWLp0qW/7ExnoVKpEEUZLi5q1q5Zg7d3bjKdq1evEjpwIF6F\nfBGR2LZ1C7a2Ly3eTZs2sXL1WjIz0pk69UPatG7Nzl27WLRwIR6enkz/7DP6dOtOf70DDQUnEiQD\nkyyxrJEVQ/XKdE2SJCaZYykhqIhzkRNrMPMsx4QgyFCoXHH3aYbKNrdceWbaHWTJB/AvE8DjJPAt\n34Jy1buisnEiKf4msQ/PI1fYUqJsM+wd3Xkd2ZnJrPk6CIvFQsmKE0m4twF95kOqi44oBZEL5nSK\nlW1Osx6L33pBnv9hsSuSJL0+afVfyNswruNin9ClbQOOHjmCWq0mIyOD6TNmcOTIEWxtbHieno5a\nrSY7O5t+ffsyatQoFAoFBoOBJUuXsm7dOooVL0XLFk0ZOmRI3nkNBgPde/TAs1BRoh/c5btfEtmv\nWr2aH374gY+mTiUxLp59q9czOkeNn6BilTkJAxLDXnHnBIiVDEwyx1BUYYOTp4YLiU8R5E6YTVqc\nNVXQeDcAQJSpiL23npL+/qjuncNFpaB90UKI0y9hMho4tGsH6c+e0PHJZuq5a/Af1ZcZj96jZInc\n3+OL4I0rp1dx8cQaVLaFcHGvQfK9ddiajVQVHIjHSJSko2Xv5ZQo0/Rvekr/N/7IuH7nBPnmjQgG\n9+vEiPffx2wy8cOmTSyYP5/y5ctjMpkI37mTTz/9lF07d1KyZP5Nju+//54v585lxowZtGvbNt8x\ns9lMYK1aTPp4NqtXLCRHr8PHx4fIyEhcNW6YTUZq1QrkwvGTJKUkoxRkIBMpYhQZI/PES1CSLpn5\n3pLCXUmPh6jkimiiVI05OLlWRJLMpCac4GHkIkpVnkxK/FFS4o8DEq7u/tQKGoF/hdb/8/MbDTpE\nmRxRkGE06og4t56fL+0B5Oi1cRQq1hmlrSep8cd5nnoFW7tClK/eFpXKnicPzqPTZuDs6kOlwB74\nFnu7cqz9lwUZ4KtZH3PhzDEmTpzAvHnzKFumDOPHj0etVpOSksLQYcPIzs5m/48/5usnSRLB7dtz\n//59Iq5dy1c5BODUqVNM+XAqPfsNYdU3CyhZsiQ5OTnExsUhCgLlSpZAaWfH+dNnEY0mDFgAgV6C\nK61ENTaCSJSkZ745kUaCI1ukNNSeNSlSYQJKGw0G/TNiotaSlX6Xwv79eXznO7SZ0YgyBSXKNKN+\ny4k4u765IoxksWA06lAo7TCbDSDBuoWtyNHlIFc6IpPZ4FmkHQb9U1ISTpCjTcRsziG451IePzxD\nUtwtJAkKl6hJpcDeODh5/NmP5//Mv1qQAe7cusGyhV9w5uRRtm7ZQpkyL6smR0dH06NnT86dPVug\nX3x8PM1btCBswwYqVy6YLS24fXtq1m3C+CnTuX3zZ9KepeJfuixu7p78fO0yRw/vw2I2U7laTUqX\nrYi7hxfvBQUSF/sEewR0SNQS7CmCkk2kUbTKJ7h61SU+ah3J0eEYzXpEQYZZsuBWtC2FS/VFoXTm\nefIlHt1aSJ2mI15bujwrI5HTB+fmCmpWMhISoqjCYjEiigpABpKJyg1WYmP/MgOWLjuWiBOhIIg4\nqktjyHlGji4JUVQhYaJavVDqNhvz5zyUP4H/uiBLksT2Tev5ZslcVEoFhw8dyreOO3fuXGxsbfMF\nPb1g0aJFrF6zhohr1wocexQdTfv27Tl+8S6CIHD92mUUcjmVqtbE++Z2vpm2kKhsHa7OrtTw9qCY\nszO3H97jgwsRmCQJW0SUCLQV1FwRddzDTMUWe8nRJhB/cwnJKZcBCRVy9KIM/8pTcPWsg9mkI/Hx\nTpLj9tLz/XAcnfOnfpUkicf3TnH+2BKSE25jMRsQBBmSxYIkSYgyBRazARePGpSuPhPhlZloTNR6\n4h5uRqFSo7L1Juv5HQRBRBAVgJHuQzbj5lWwiso/wR8Z12/3HPY3KF22AiPHTeX+3Zv5xBjA0dER\nrVaLXq/PS9b9gqdPnyKXyzlz9mwBQU5PTyf60SOq1zIjCAJly+dP/l6pag0qVa2R99psNnNw3048\nixUjIzOdjMwMitqouWrMIcrVDWOqFo13fR5e+gj31JtMRkNhmYpYycA3UgoJWXEolGoEQcDFIxCl\nzUxO7BvJmcMLMRn1CIKAq7sfaSlPMORkIJfbUaryFFw8AjGbskl4FE7C412UrDCWR7eW4+haPp8Y\nwy/Vd73qY+9cEt8S3QF4nnKFO5enIYpKrpz8jqIl6+JbrAZW/nkEQeC9Hv14+OAebs6qAptqzmr1\nL3slBUlMTESr1ZKQkJC3TPeCc2fPolAoSXuWgm9hP2rXbZh3LKVyd97b3R2PTdOIPRkJkoHUp3Hc\ndVBQxMeD2MRUkMAFGZulNAIcHVA7B2HQJ3P79FC6WuxoI/qhQuSilM1CKRUQEEQ5cqUjvqX6YDRm\nsHlFN/S6LCTJgkKpxEntS0riPSTJhLNrJSrW/QZbe1+yMx7y8OYiFEpXXD1r8TByIb4le+cTY4BC\nxd8j9sEPVKy7HIXSCYvFyMObi3mefAmjPost3/Vi+MdXeNd45/yQX6BS2aDVZmOx5E9bqdFoCPD3\nJywsLF+7xWJhxYoV6HR6vv32W65evZp3TKfTMeXDD7G1s+PmjWtMGTuEwwd2522yZKQ/Z8Pq5Ywb\nEcInU0Zz8fxpxo0I4evZH3H/7k20JgNehX2JkXTUbP8Z7YduByQe3lqONukC03HPK8PkKyj5RPTG\nlHaLrOcvM2nbOxVHodTg7tuGIv4hyGSOpCRGYTRkoVA6U6bmbFw9ayMIInKFI4X9+6Lxqk/0ne8Q\n5SqMOc8wGvInTDHkpCGKcvTaxLw2tVs1vP06ovFuiMrOi/2b81vI6WmxnD44l91hwzmx73OeJT34\nvz8kK/8nVCoVma/Jdta2TRv27t1LfHx8vva4uDj27duHnZ0dEydOJPOVkmN37txh4aJFaLXZfDXr\nY76cOZUH91+W/7p7O5I50yfT+9RjVgYEciNkIl0i7hD+KIWY9GxEB3tsPNzI8vFm9f5rONX/AKNZ\ny52zY2hlVtJFzC3dJBMEaosOTMadhJtLeHXm7eHbCr02k1KVp+LqWQ+jwURywh3kSmeUNp6UqfE5\ntva5paTsnYpTtuYc0lMukRR7GBDQZT1BeiWFpyRJ6LWJiDIVObokAERRQYlyo5AsJkpUGIsxJ4vI\nS5vz9Xl87yQHtk5kz8YRXD8fhiEnf6mot4F30kIGKFzEDzcPLw4dOkTLli3zHXNWq1m4aBF37t6l\nY8eOZGVlsWbNGm7evEmlqtW5EXGFQYMHU7JkSdw0Gs5fuIAoivj4+PBex2CMJhMrl33Ft8vm0aP3\nQBbPm0mN6tVp1rgxScnJfDh2CJmZGTRo0IBhQ4fi7e3NyZMnmT1nDucOzaF0pfbY2Kp5Frub1oJT\nXnasFygEgSCLinNPz+LokptTw2IxYjJkkXZvI5JkRqZ0RlOkPTn6JJ4lniYp5hCPIhcjYcHFIxBv\nv454FmlNctxhvP06osuO49qJEMrW/BJbh8I8jFxIauIpVDYe5OiSyIo9jMGSg0Jmi6OmMtqcX5Yg\ngQAAIABJREFUVIoEDOBexBwMOdkoVfY8uH2YA1sn4l6oGfbO1Ul6+oDvl3WkXLXONGg1BZn8f9cu\ntPL/T/PW7Rke2pVhQ4fi6PgyGEKpyrWa2wUH8/7w4VSoUIEbkZEsW7YMk9lMkcJ+3I2KokHDhtSq\nVYv09HRu375NTk4OnTp1IjAwkKioKPq815IOnXsik8vYue17evbsQZWmjbh06TIDe3dAoZDTo8dQ\n2rZpg06nY+PGjWzZupUvZy4koFInTv44C6XFRDOxSIF7ryTYIhmfkaNNyJux6XWJCCYdjy9OQhBE\nbByK4+rTmOSY/VgsRqIiZqPNfIRC5YKHb0vcfYLQeDciJeEEhf37EfdwCykJxyldfQbZGQ948PNX\nGA0ZyBXORJ4ZhkpQYLAYcbLzRmXriUUy46SpzNUzaylfoxsWi5kfN39AwpNI3H3bIlc4cv3ifs4e\nWUirrl/h59+wwOf4p3hnBVkQBKZ88gUjB/Uk+vFjWrVsSXpGBuvXryc2NhaNRsOzZ89YvHgxKqWS\njh06MO3jj+nVuzeNGzfm6tWraLVajM7O2NjYUL9ePWbPno0gCBw8dIiUpCQUSiVrv13A87Q0Ll++\nTL26dRkYGopKpWLv3r0sXLAgb1oZHBxMQEAAnd97j5VzAvF0d6ZcuWpw8Nxr7z/Xrs/ta7EYuXms\nDyVMWroJGlxEORdMWnY83IyHf1/8q0zl/s9fEVDtM0SZkuTYw1w/NZTi5UchVzqRmniKKg3Xkhx3\nhKhrM7F1KIooyqne5AeeJ18h7voXDJacCRS9eC6Z2ZocyRlBj8rGHZDQ654jSRYObBlP6Rpf4qh+\nsfYWhIdvKyJODeJOxG7a9Vr6TlbyfdcoU64iQc3a0rVbN8aPG0eZMmW4fv068xcsoF27duzdu5dL\nly+z/8ABihQpwrcrVnD+wgW+++47OnboQPjOnWRmZmJjY0NOTg5hGzZQtWpVcnJyuHzpEiaTkQtn\nfyIpKQm9Xs+Vy5dp1bIlwe3acfXqFYYMGUJwu3Z59zNlyhRMZjNbtvzAnWtbCA5uw9mfjmHJev3+\nk4QEv/wuMtJu8ejSR7QRnakvOpItmdmeHUdU1BpK11/Fo8gFmIxZlKo8Bb02nrj73/M85TKCoESh\nzPWbr1T/W25fmkxM1FqePtlHsXIj0Xg35OHFKXjonxOCM4VFJbd0OpZro8hSuWHvVJz0lNMA3Lq6\nnaT4R1So8w3iL5GFHr7NiYlax56No/AuWpl2PZegsilY5+/v5q0RZIvFwtlTxzhz8ggymZymLdpR\nqWqNN0bWVatRm7Wb9jKkf2eWLVuGo6MjtQIDCfD359Lly6xauRJBEDCbzYiiiCAI9OzRgz1799K8\neXOMRiPubm6cO3eOcePGIQgCly9f5pNPPmHp0qVUq1oVgCdPntA/JITZc+bwKDqau3fvEtK/f4F7\nCwgIoGhRP2JiYtm9exdPnjyh39Fj9Dbmj2TKkSwcljIp5FoBSTLz4Oev8dY/5wuxcJ7je0lsKCkq\n+ereBjyDtqGy9UAAnFzKIZPZos+O5vGtedg7leN58lX02bG4+zQl+tZy0lOuUrPZDgRRRlzkAqbh\nRlkx16XIHhljBHeypCTu39+Ym5zJwY07EbtwdqvyihjnYutQGA/fFljMBnZvHE7f0ftxcMrvDmXl\nzWSkP2dP+GaiHz3Ay9uH9p174Ob+Zi+AqdPnsmXjGsZPmIAkSfj6+tKkSRMOHTzIkMGDGT58OJIk\n5eVuqVixIqtXryYhMZFWrVqhVChIS0ujdu3aVP1lHH82fTpJSUmcOH4ctVqNxWJh7759TJ06la7d\nurFs6VISEhJo3apVgfvp1bMnmzdvpnevXkyYMJ4Zn3zKka0HCLXk9+i4LGUjyG1RqDQYDek8PD+e\n4aIHQS9ysAhQWbLjM+Ep9yI+wa/CZO5cmYaDcynsHIthMRuJjVqGwZCNd7HuJD7ejU/xLhTxDyHy\n/AQ8fFvg7hPE8+TLiGl3mIFXXgBVNcGeryQVA5POIdj74OGV62X188XNFCreK0+MX1CoRHfiH21H\nr1NxcNskgnsv+z893z+Tt2INWavNJrRXMPPmfIyHiz2OtiKTPhjIxNEDX+ss/yr+pcvh41uUkSNH\nUqdOHZJTUniWlkbVqlXZER5O6zZtCChdmho1azJr9mz8/f1RKpUU8/MjIz2dTZs3I5PJcHfP9fNd\nsWIFYz/4IE+MAYoUKcLiRYswm82EhYWRlZlZwLXoBQqFnMaNG2FnZ0fp0qVp0DSIz5Qp3JV0mCWJ\n+5KeD82xWGw03L3yKWf3NSM77ie6Cq4FopBqCvbYi3LiH25GlNmQ8GQf9yKm8ej6RLp0qMmYUYMp\n5J6NSgkpiacRBAG5yhm1e3VEmZKs9HvYWyyUfSUs9gVtJQd0T89SvHRj5HIV2qxkVLavr1FmY++L\nTGGHxqshNy5ueuPzsJKfK5fO0bpJNa5fPkWxwh7EPLxJu6Y1OXLwzUmoBUGgVr1GKBRKJk6ciJeX\nF3fv3kVlY0OJEiX4bPp0qlarRukyZQgODubAwYOUKlUKGxsbfHx8OHfuHLfv3KF48eJA7ob2gQMH\nmD9vHmq1GngRBdiODh06IEkSs2bNQi6XvzaJkY2NDZIkMXBgbtmjQcOGctbOTBjPeC6ZyJEs/GTJ\n4EtLIialM+cPtOTS4feQzHoa/SoHhSAIvCc5YUyPBiQMumSeRK3l1rkBqEw/MmbUYAb070tm0l4k\nUxoWixkbe19EUZbn6/w87idaSTYFCgarBTlVsUWfFUP9VpMB0GYm561Rv4pMpkJpo8HLryMxD86S\n8RaEYL8VFvLCudPxdNewYe2qvMEwZPBgQgaEsnHtCvoNfP+N/XXabKpWrUqAvz86vR6FXM6HU6dy\n7949Ppk2jcDAQGJjY1m0eDFfz5tHUJMmaNzcSH32LO8c3377LYMGDeJaRASzZ88ucI1y5cphZ2uL\nja0tZouF8PBwmjRpku89UVFR3L59m0oVK5KdnY29vT1zvv6KNatXs2DVGmJSYrATbcgRzdRouBZR\npgQsRB7qhLPl9Rnq1DIFDx/vxNbeF4tJi0r+hP1HD+T9qEJDQ/niiy/ZvvMnLMU6kaNNRKn6pfqH\nZEbxG9GECkFAFBW07Po1ABrPACKvHHjtezNSI3D1rIMotyUp4dIbn4WVl+j1Oj4Y1pev586lQYMG\nee29evakb7/+VKpSA3eP355tZGdl4ubmRuNGjdC4uuLg4MCO8HBmfv45DRs2ZN/evXh5eXH69Gk+\n/ewz0tPT+eCDDzh54gQms5nU1FR27dpF927dePToEVUqV863Jv2CpkFB3L51i9t37uDt7c3Zc+eo\nW6dOvvesXLkSs8XCo+ho1Go1Xl5ebN21kwVffsWgQwfRGw3Yyx0xyByo1WgNkmQmOyOauFNDChga\nkJv3BeDG2VG4etYlM/knunRqzuTJk/JmnkOHDqFdcDCpCccQRAWizAajIS33BJLptRV9AOSSmZLl\nmuPuFQCAxiuA9NTr2Drk94U26FPJ0Sdj5+iHo0sAz5If4ORSULj/Tv5xC9lgMLB7x2YmThif759Z\npVIxbtxYtv6w9n+eQ+PmQWhoKEuWLmXr1q2MHTeOjIwM1q5ZQ+3atRFFkSJFijD3yy/RaDTcvnOH\nFStWEDpgADvDw/lizhy+/+EH+vTti1wmIykpqcA1dDod2dnZ1KhRg5iYGA4fOcLsOXPIzMxEkiSm\nfTiV9m3aUtKs5PLmndSrVZv1a9chk8kYOGgQB08cw8FRjVFhj0zpSlryeQRBQBBkKJ1LclYquOOb\nKpmIMWhx92mR62OssjBh/Ng8MYZc0R49ehR6bSJREV/g5FqB7PSoXOvYqRSpmHgiFczy9pOgpVyd\n/shkucO6WEBjkLKJf7g1b4dckiSSYg+TlX4Xt0JN0GfH4uj89jjcv+0cPbiPMmXK5BNjgPLly9O8\neXN273jzbMO3SFESEuJp3aYN27dv5+t58zh8+DB+fn7MmD6dQoUKIYoiDRo0YPWqVeTk5DB58mSe\npaWxaOFCwnfsoFevXnTq3Jmt27YR9yvvjBckp6SgVqtxcHAgJyeHYcOGcerUKSRJIi4ujhaNmrB9\nfRhljQpG9+5P66Bm3L9/Hx8fH+YunM/XC+aj1vihQ0KymLBYjAiCDDuHwmQKudWkf80FIRuLIFC5\nwWqKlR+J0ZDOmDGj8y0DqtVqxo8bR/rTH3l0cyka74bEPdiMxWLCwasuh0Ujv46j0EkWrsqM1Gsx\nIa+tRv1Q4h6sR5v1OK/NbNZz/8bXePi2RCazQZcV+5vRsH8n/7iFnJmRjiAKBXwnAUoHBBAfV7BM\ny6tEXLnI7Vs/s2HDBiqUz60usH37dnaEh+PllT9LqiAI9O7dm1mzZvHT0aM4O+duGhQqVIiaNWvS\nuEkTihQpwpKlS1m6ZEne4Hjy5AmjRozEotMTEb6PhpItsUp7wlavZe3atSjlCtQ5Zr4Vi+L2Sz6A\nOIOBT7+ej6e3F4GBgUyaNAWzyYhX0Q5YzAa0mY/hl4/sW/Z99p0MpRQq6goOiIJAimRkjpiMIMjQ\nZVzi2rGOyOUCAQEBBb4DW1tbPDw0JCTdxr/KR9y//iWR5z/At0RPNIXbMu3xbsajoRy26JDYQwbn\nVRLd6r7MCy2KMjoNWMuu9YOJf7QNR9eK6LIeYzZmU7bmF5iMWSQ92U39ZtYk97+XhPhYAgL8X3us\ndOkAHkS/eWzP/+IzqlatysIFC3BwyE3I07VrV3r37l1g/8LPzw9/f39cXFyY9fnnee2jR43Cp1Ah\nlixdSkZGBmfOns2zfiVJ4tChQ8z8aBoGo5Gagj1umLmkkBgxeAgmmYiQY6SN4EwfWQlkgoBkkjgY\nl0G/Hj05dPwY9+/fZ8qHU8nWWihfez43L0zAaEhHZeOGKFPi7FGbGUkX+FT0wUtQIEkSl6RsNvMM\n0c6FOxffRxBEPDzc86UyePk9lSY7IwoPn2BsnYqTFLOfG2dG4FOiO6lKB+bpkwnBFVdBToyUw1JZ\nBsXLtkKteZnYyLd4Leq3HMexve9j51ACpY07z1Mu4eIeiF/pwTyN+RE7BxfcvMoUuP7fzT8uyE7O\nakRRJDo6ukA1hIjr1/ErVuKN/detWsrIESPyxBhyBep1DxdAFAQ8PDzyxPgFLi4udOjQAVdXVw4e\nPEjffv0YMGAAsbGxfDljJkqzxADBjVbiL9apGW4LjkyXJWPMyWGc6JMnxgA+gpLQHCemfDCeHFHE\n0aUMZeusxcbWnZsXJpCji0epUiNJEsmx+7DVFGfJ8xi+sSTjLFOQaM5BJsgIrFuPCePHotFoGDxk\nCBEREQVEOTMzk4SEBIwmGbcvTcVBXRqDPo20pPOYzXosNho+1cYjiHLMAhQrXptO7acXsAjUrkXo\nO3o/P1/8npM/zsbRpTwav86kJp4kKWYvNRsOweNflFnrr8avWAnW//Tja49dv36dcpXrvPYYQNqz\nVPbv2cGxn47miTGAvYPDb250W8xmmjQuWIYgODiYz2fNYtasWQwfPpy+ffvSuFEjli9ewtXTZ5FL\nEgtkRfH8ZfwOMEuslKVxwdaCaNbRz/KyYIIgCLTEmdPpcTRo2ASjyYKLVxvKBQzAaMjAbMom5u5q\nnDSV0WY+4lnadTQexRj+NAoPmQotFnQWExaVgoljhtGqVUuiHz9m4MCBZGZmFlhSiYiIwGKRiI/e\ngY29D/bqsmjT75IQvR2TTMkFucBpwyOUchWCXEnFOv2pETSqwHdQvnpXSpVvxf7NHxD76CIevq2w\ndyrOgxtfkPU8kvdCN/yhupR/Ff+4ICsUCrr2CGHGzJksW7o0b7MsPT2dL774kl79hryx/63ICCaM\nHZmvrXbt2nw4dSpJSUl4eLycYkuSxA+bNhUQ/hc4ODggWSxs3bKFnr168eGUKeifZxBsceIS2bQU\n84t4GcGW+jhwWErFH5sC56si2JFpSKR68x0olLlCnvHsBvrs+zRs/RHR93I34hq1HUfxgCYgiNyJ\n2Mnz1GhsUqPx0WTz3bcr8gbKhAkTmDRxIvXr16dQodwNOIvFwvQZsxBFBSpbN0RRRY72ae4am1KN\nlPMMvSGGoPfmUjygEXKFDQqlXYF7fYEgCFQK7EWpci25cWkzyQmRaDTuNGoZ9taEor4rNAxqyZwZ\nHxK+cycdO3TIaz9z5gynTp9myoxFv9n3ftRt/P398y1PATRu1Iht27bRvFmzfO0xMTHcu3ePWrUK\nuiUqFApUKhWVKlZkzerV9A8JITw8HN3TFMpiQwXRNk+MIXcM9DGrOZwRTS2LHYKsoFDVwp54RXHK\n1pmJIMqRJInYe2soVa4VGo+S6PX3KFqsME3b7cDVowRZmUncvLwVBIFbl8P4eu5MGjXM9f/19PSk\nWdOmzJg5kzmzZyP+UpIsPj6er+cvQqvVYe9UEgQBXeYj5EonFCo39Nnx2Dm60rXvNlS2ztjYOr8x\ngZbKxpEO/VaSEHONm1d2oNdGUqZSHcpWnYeNrdNv9vs7+ccFGWDY6El8OG4ojZo0oWWLFhgMBg4c\nPEj7Tj3o1K3PG/u6uGiIj4uj2Csi6+LiQr++feneowdffvEF1apVIyEhgXnz5nH79m3c3NywWCx5\nDx5yQ6EPHTrEzBkzsLOzY+uWLdQMDERmNKEWZJQSbF77D1rKKOMYkIE5b6PiBUkYUQBx91aSk6PD\nlPMEbWYsH83+hobNgtl7ahCm9NR8fcpW7QTAjpWdGDBhbL5r1q1Th9DQUJo1b06jRo3w9PDk4OFj\nmNFgMsuQi1mo3Wvh4dsSgyGNhIfbyNE+ocewHbh7/zExtXPQENh4+B/qYyU/CoWC5as3M2xANzZv\n3kKVKpW5ezeKG5E3WLh8A87O6t/sq3bVkJCQUGCcdu7cmZWrVvHxxx8zcuRI3N3dOXf+PJMmTUIQ\nRY4fP06JEvlnlVeuXsXOzg4vLy8KFSrEnNmzmTR+PB1x4oKUhb/oUuD6NoKIj2jDY/Prq8wkYCQr\n8w6Pbn2DXhuDUReNg5M9zXv/hK2DK23rm1HKBQymF2u8augxlVWrL3P/+vc0/NW6+meffUbffv2o\nW68eLVu2JCUljWPHjuPs0QAn802Mhkx8S4Xg5FqJZ09P8fTJHtwLBfDegPWIv1Ha6rfwLlwF78JV\n/lCfv4t/fFMPcgfu3EWrWLF2O+4+JfErVZHNu44x6eNZ/3MaEdypO8u/WYHRaMzX7ujkhFxpw6Qp\nUylTtixBTZuiUCoZEBJCamoqk6dMIS0td8f2xWsPDw9q1MjN66DX6zGZTHjbO+EpKIh+zcYYwEPR\niEUQ2SSl5WuXJIkt8izqNmnG86TjVC0vY+igzjRv3pR5M8cQcfEEDSs9w5K5hSeRs7hy+ju0WSl5\n/c1mI7Y2Ba3ukJAQNBoNF69lc/yiCvdiE3DQNMTVowS9RuzGzd2Ge9c+IvrmV/j6FafPqH1/WIyt\n/HmUCijL/uNX6R06AjtnL1p37Mnh0zeoHlj3jf1KliqNs4sr4Tt35muXy+U4OTnzODaRps2bE1C6\nNGPHjmXs2LE4OjqycNEi9u3bh9mcG2ocERHB+PHjGTVyZJ6wZ2Vl4aZ2RYMcD0FB9Gs23YyShXij\njsfk8FDS5zuWIhk5rsihbOXK6J4f5b3gigwZ1AtXZ1tO/NCFhpWe8eTBTdYsmc7K+ZO5fm4fLna5\n92MxG7CxKWjcODo6MnPGDLKysvnpHNx5UpyydVaiz44hsPEwWnWdgz7jFLcvjiQj5RiBjQbw3oB1\nf1iM33beyWxvr2I0Ghk9pDfPUhPp1bMnzk5OHDx0mHPnz7Ns1WZOHjvE9+u+xWIxU7VKFS5fuYKn\npycZGRmk/5JjNjk5mdatW/PpJ5/kLltIEp9//jmxcXGcP3mKJUZvJlhiGCC6U098ucYVLeUwzhxD\n93592LIhjIpKZxobVJiROGxjINFGJC07E4PBgJ+fH/369qV3795cvnyZgYMHo1AoqVunNpUrVeLO\n3SgOHz7CpOlLSM6uw5lDsyldxMAn0z7K93kfPnxIu+D2FCo5DKWNhufJZ0hPuUSXgRvReJb6u7/+\nP5X/era3X3Pn1g0G9+1EUFATgpo0ISUlhfUbwigRUI7QIaNZ8+1ifjq8Dx8fH4oWKcKJkyexs7PD\n0cGBbK0WQRCQJIkpU6bQqWNHAFJSU+nSpQsN69fnYfhBWuhVLLckMVdWGPUr1W/CLKkcsTMSUKEc\nV89fpL3clQCzgieiid2yLGxdXUlISkAQBOrXr8+oUaMoX64cH0+bxrXrN0h7lkKnjh1Rq9UcPHgI\nvcHIF8t3Ym/vSKemFfh+wypKl85vKHw+axbbwo9TqEQoJkM6STG7cPPyo13PJW99Lu838a9Pv/lr\nTCYTRw/t48Ce7ej0WmrUrEer9p0ZO6wf3l4eDAjpj4uLC4cOH2b58uV50UthYWHEx8fj4uKCIAh0\n7tQJFxcXDh48iNliYe2aNSyZv4Dr4T/yXo4dX1sS8ceGcqIdDyU95wQtYydNJDQ0lI0bN7Juw/f4\nuLojymTEpSZRyNuDKZMnU7JkSa5eu8bnn39OtapVGTt2LIG1arF0yRLq16+f9zl+vnGD/v0HsHbn\necwmE0N7NmHokMH07NEdlUrFjchIRo8Zh6NbVXTZWRgM2fgWq0HFmj2wc9D8g0/gz8EqyAVJTnrK\nth/WEXHtAg4OTrRp3wV7ewfGjujPwAGhNGvWlLS0NL5ZsYIrV64w6/PPOXL0KPv378doNOLo6EiJ\n4sVp1rw5TxMT2bN3L3169yYkJIQWTYJonyEn02xkn5ROY8ERF0HOSSmTLCc7Nu/cga+vL+1bt0bj\nWRiTVou9Ws2Fy2cZMWIEXbt0QSaTsXPXLubPm8fKlSu5GxXF8mXL2LVrF05OueuykiQxa/YcYhOS\nWbB8PatWrWbbhsXM+2oulSpVwmAwsG3bduZ+vYBSFTuTnHAXpdKeMlWCKVG2GaL4blvB/2pBTnuW\nypbv13Du9DEUCgVNWwbTvnMPbGzye1WsW7mUqxdOsHzZ0nzToyNHjjB+/HgOHjyIp6cnkiQhCAJn\nz54lNDQUB0dHQkNDGTRwYF5ZndnTZ7Bt23Y0BgspmFDZ2tKqQzAjR43Ki/DTarVUrVaNiKgkzpz8\niXlzPmZX+I58pabS09NpEhTEyBEjOHr0KBs2FKyWO2nyZIr5VyJk8EiuXI9i6ZeTibx+CTs7OyRR\nSeXaQylXvcdbsSP8ZyF31tC2vpmGFV3+04L8c8QVNoWtJCb6IYV8i9C11wCq1aid7z2SJNGhRR3G\njx1N06ZN87UPHDSIlJQUdr5Sdd1kMjH1o484dOgQCoWCzZs25UXvPYqOZtz7I4h5+Ah7o8RTwUTh\nQj4MHT2Sdu3aoVDkbvStW7+e2/ce89nshYx9vz9VKpZhYGhovvvaunUr+/btw2Q206tnT1r9Kvw6\nKyuLBg0bsffoJTRu7qxbv4Gw775Cr9Oj02nxLlyJOs0/ws0rgA5tc9fWd+7Nn73wXeVfmw85LuYx\n/bq1oU6dWgwfMhCdXs8PP2wifOtGVm3clVfoEWD/3h1MGDemgHAFBQVha2dH586dGTlyJGXLlePG\nzz+zYOFCBFGkW9eunDt7ltABA5DJZMjlcj767FN0JiN79u7FaJQ4e+5MPlckAK1Oh0KpRBAEjh3+\nkU4d2ucTYwBnZ2eCgoI4d+4cZcu+3n2sdEAAj2Jz/VOrVfJn9cYdPH+ehjY7C9HOu0BY67s+aHN/\nfGbcnf49fzD/F7ZsXMOyhbMZMGAAXTu25fadO0wcFUqfAcPpP+hlQvoH9++i02YRFBSUr78gCISG\nhjJmzBiGDR9O7169sLWzIzw8nB9//BFPT0+USiXXr1/PE+Rifn6sCttAs+bNyRJEGlarxjfLC+Zz\n0Gq1ed5Px48eYNb0jwu8Jzg4mGmffIK7u/trfeUdHBzw8vbmaWI8Gjd3+vXtQ5/evbh1LxYbW3uO\nnS64ndWhrfqdH99/lHdKkOfMmEKPHt0YPmxYXluzpk0ZPWYMa75dzIgPpuS163VanF4TJioIAhqN\nhm5du3L8xAnCwsJIz8ika88B6HRa4hNisbG1pWOnTnTp0gU7W1t2hIdz79495HIFgXUasGXrVgaE\nhOQ7b1hYGC1atwdys139lgUrCgI5BgPnL1x47fGI69epVL1+vja12gW1uuBOeHKGlGdNvGDvKVkB\nz423kVfv+78uxinJScz74lN2hu+gaNHcgIbAwEBaNG9O2+BgmrZsi29hPwD0Oh0Ojo6vHV9Ojo5o\nXF0JrFmTRYsXo9VqeRQdzc4DZ+nRMYgPp0zhy7lzOXXqFA0aNCAuPp41a9bg5eVFqTIVOX7kAE+f\nPsXT82U4t16vZ9u27Uz/cikAksT/nJ1FvCL6L3j+/DkJ8fEU8n2ZslMURcoHFPllHOc3LtydBJIz\nfv/s/d/COyPImRnpnD97kvlz8+eZEASBIYMHM2LU6HyCXKNWfQ4eOkT5VwJGIHea9vTpU7p27Uqf\nPn2IjIykb/8QQoeNwWg00qVtA4r5FaV5s2acPHGCmNhYnj17Rrly5ShZuhKdu/clpHtbMjIy6NSx\nI2azmS1bt7Jr9x7CtubmgmgY1JIlX02nX79++SzazMxMfty/H4VCiUKpZPeePfnSHJ4+fZqz587x\n0edLftd38mshS9dC2/pmoKA71dtgaZSu6kvpQrmJ1wVBwO2fz3b4VnBgXzhNmwblifELvL29CW7X\njn27tjFkxHgASvmXISnp6WsDqQ4cPEhgrVqEhIQQEpLr21+hSiBFihZj1lfLmTAqlO7du5OTk8MP\nmzbx+PFjypYty507d5k1byTFS/jTq3cfxo8bS7Vq1bh3/z4LFy2iXKVqeUsnDRs3Y/uOHYT075/v\n2vt+/BF7e3ucXdxYtGgxdevUyRN2s9nMrNmzCWrR5rWGhbuTQLo2/5/0f1GM4R0S5OwzconoAAAF\nHUlEQVTsLOxs7bC3ty9wzNPLi/T09HxtfUOH06NjEMWKFSO4XTvkcjn3799n6NCh1KxZk2vXrnHx\n0iXCwjYyffZC7Oxyz7tj/xnGDO3L2vXrcXJ0xMPDg+xsLbYOasZMmIZSpSJs+yFWLZ9Ptx49EUWR\nJs3a8P2Ow3gXyk1MUq9BEOu+W8wHY8cybuxYihYtyo3ISKZNm0aVaoEsXBFGzONHDB/QjU2bNlOp\nUkXu3o0iMjKSBcs34PQG/9Q34WwHvCaZ0Oss6RfciXfIq/L7V/CqCEPWf94afh0Z6c/zWaWv4uXp\nSVLayz9TlY0NIYNG8f7IkSycP5+SJUtiMpnYvWcP69evZ8qUKfz0009s3xHOvfsPWLc5N1KwYZMW\nLF25mYmjQzGZjLi6ulK0aFF+/vkGUz/7koAy5QkoU55iJUqxds03TPvkUzy8vOnSvR9deobkWcXD\nRk8itHd7lAoFnTp1QhRF9u7bx4wZMxk7+VO69gxh9YpFtGrdhmbNmuLk5MSRI0cpXLQ4C5av/83v\n4MXYfVWI3wYj4u/mndnUM5vNNKtXge9WfFNg/XX7jh3s2XeQFeu252u/eSOCmdPGE/skGidnZzIz\nMmjash3pz9NITkqkpH8ZevQdRCn/gjHscTGPOXpoH0ajkdr1GhWosfe/0Om0LFswh+1bwtBmZ+Pm\n7kGfkKH0GTAszx/UYDBw/Oh+Hj28TyGfwjRr2a7A5uRfSbqWVxz3f5vf88P4LcGH378kUdZP/Z/c\n1Dt1/AgL537C7p07CywH9OjZi+79htGyzctIP0mSWL9qGau+WYCjkxMZ6ekU8i1M9Zr1uHPrZyQk\nGge1omOXXjg45o9As1gsnD97ksjrV1G7uNK8dfvXWq1v4vbNn1n89UzOnD4OkkSNwLqM+OBDKler\nmfeepKcJHD6wB212NjVr1aNilep/aCM6OUP61wjyv9bLYsPqb9i5dT3frvgmL3T4RmQkg4cM4Yv5\n3xFYp8Fr+8XFPkGrzaaoXwmUyr+3DJHFYsFgyEGlen2k39vO7506/hlLEP9VQbZYLHQNbkSjBvUY\nOWIESqUSk8nEt999x47wXew8eC7P4+FVDAYDj6MfYGdnj49vwXJKfzVGY262tb/iN/XCWHhX9kTe\nxL/Wy6J3yBCyszNp2y63XJJerychMZFJH33+m2IM/COD9QWiKP6tVu+fjXWJ4a9HFEW+WbONKWOH\n0KBhIwICArh3/x5FihZnZdjO14oxgFKpfO3s7u/it+7rz8DZDlIyBdrWN7P3lOadF+XfyzslyIIg\nMHTkBPqEDOX6tcsoFAoqVwv8SweGFSt/B27uHny3IZzoRw94Ev0Qn8JFKFGyoPvYfwk3x5eivPPN\nBVb+NbxTgvwCewdH6tQvmGbQipV3Hb9iJf5nytn/Em6OkJzx3/FJfiuSC1mxYsXKb/Fi2exNG8f/\nFqyCbMWKlbee/4ooWwXZihUr7wTuTgJKuUCHtup/rTBbBdmKFSvvDM52/25r2SrIVqxYeef4t4qy\nVZCtWLHyTvJvXMKwCrIVK1beWX69hCF3frcLNVgF2YoVK+88L6zltvXN77S1/IdyWQiCkAw8/utu\nx8p/nKKSJLn/3Re1jmsrfzG/e1z/IUG2YsWKFSt/HdYlCytWrFh5S7AKshUrVqy8JVgF2YoVK1be\nEqyCbMWKFStvCVZBtmLFipW3BKsgW7FixcpbglWQrVixYuUtwSrIVqxYsfKWYBVkK1asWHlL+H8C\n03JKFj+X+gAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 4 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"A1khgQHlhJkl","colab_type":"text"},"source":["## 2. Loss functions"]},{"cell_type":"markdown","metadata":{"id":"O1ZYbqXthZvR","colab_type":"text"},"source":["#### Linear classifiers: the coefficients\n","\n","When you call fit with scikit-learn, the logistic regression coefficients are automatically learned from your dataset. In this exercise you will explore how the decision boundary is represented by the coefficients. To do so, you will change the coefficients manually (instead of with fit), and visualize the resulting classifiers."]},{"cell_type":"code","metadata":{"id":"L8znoc7thK_l","colab_type":"code","cellView":"form","colab":{}},"source":["#@title\n","X = np.array([[ 1.78862847,  0.43650985],\n","       [ 0.09649747, -1.8634927 ],\n","       [-0.2773882 , -0.35475898],\n","       [-3.08274148,  2.37299932],\n","       [-3.04381817,  2.52278197],\n","       [-1.31386475,  0.88462238],\n","       [-2.11868196,  4.70957306],\n","       [-2.94996636,  2.59532259],\n","       [-3.54535995,  1.45352268],\n","       [ 0.98236743, -1.10106763],\n","       [-1.18504653, -0.2056499 ],\n","       [-1.51385164,  3.23671627],\n","       [-4.02378514,  2.2870068 ],\n","       [ 0.62524497, -0.16051336],\n","       [-3.76883635,  2.76996928],\n","       [ 0.74505627,  1.97611078],\n","       [-1.24412333, -0.62641691],\n","       [-0.80376609, -2.41908317],\n","       [-0.92379202, -1.02387576],\n","       [ 1.12397796, -0.13191423]])\n","\n","y = np.array([-1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1, -1, -1,\n","       -1, -1, -1])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"tsbGmRxQUL8j","colab_type":"code","outputId":"fad12b4a-071a-42ac-afc3-2d9104eee100","executionInfo":{"status":"ok","timestamp":1557806106886,"user_tz":-480,"elapsed":463,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"source":["model = LogisticRegression()\n","model.fit(X, y)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n","          intercept_scaling=1, max_iter=100, multi_class='warn',\n","          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n","          tol=0.0001, verbose=0, warm_start=False)"]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"code","metadata":{"id":"tQabZT6ZWuOA","colab_type":"code","outputId":"ac052db4-6282-4818-923c-1bd696e4b786","executionInfo":{"status":"ok","timestamp":1557806109253,"user_tz":-480,"elapsed":760,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":286}},"source":["# Set the coefficients\n","model.coef_ = np.array([[0,1]])\n","model.intercept_ = np.array([0])\n","\n","# Plot the data and decision boundary\n","plot_classifier(X,y,model)\n","\n","# Print the number of errors\n","num_err = np.sum(y != model.predict(X))\n","print(\"Number of errors:\", num_err)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Number of errors: 3\n"],"name":"stdout"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADU9JREFUeJzt3GGI5Hd9x/H3xztTaYym9FaQu9Ok\n9NJ42ELSJU0Raoq2XPLg7oFF7iBYJXhgGylVhBRLlPjIhloQrtWTilXQGH0gC57cA40ExAu3ITV4\nFyLb03oXhawxzZOgMe23D2bSna53mX92Z3cv+32/4GD+//ntzJcfe++dndmZVBWSpO3vFVs9gCRp\ncxh8SWrC4EtSEwZfkpow+JLUhMGXpCamBj/JZ5M8meT7l7g+ST6ZZCnJo0lunP2YkqT1GvII/3PA\ngRe5/lZg3/jfUeBf1j+WJGnWpga/qh4Efv4iSw4Bn6+RU8DVSV4/qwElSbOxcwa3sRs4P3F8YXzu\np6sXJjnK6LcArrzyyj+8/vrrZ3D3ktTHww8//LOqmlvL184i+INV1XHgOMD8/HwtLi5u5t1L0ste\nkv9c69fO4q90ngD2ThzvGZ+TJF1GZhH8BeBd47/WuRl4pqp+7ekcSdLWmvqUTpIvAbcAu5JcAD4C\nvBKgqj4FnABuA5aAZ4H3bNSwkqS1mxr8qjoy5foC/npmE0mSNoTvtJWkJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYM\nviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMG\nX5KaMPiS1ITBl6QmDL4kNWHwJamJQcFPciDJ40mWktx1kevfkOSBJI8keTTJbbMfVZK0HlODn2QH\ncAy4FdgPHEmyf9Wyvwfur6obgMPAP896UEnS+gx5hH8TsFRV56rqOeA+4NCqNQW8Znz5tcBPZjei\nJGkWhgR/N3B+4vjC+NykjwK3J7kAnADef7EbSnI0yWKSxeXl5TWMK0laq1m9aHsE+FxV7QFuA76Q\n5Nduu6qOV9V8Vc3Pzc3N6K4lSUMMCf4TwN6J4z3jc5PuAO4HqKrvAq8Cds1iQEnSbAwJ/mlgX5Jr\nk1zB6EXZhVVrfgy8DSDJmxgF3+dsJOkyMjX4VfU8cCdwEniM0V/jnElyT5KD42UfBN6b5HvAl4B3\nV1Vt1NCSpJdu55BFVXWC0Yuxk+funrh8FnjLbEeTJM2S77SVpCYMviQ1YfAlqQmDL0lNGHxJasLg\nS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHw\nJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4\nktSEwZekJgy+JDUxKPhJDiR5PMlSkrsuseadSc4mOZPki7MdU5K0XjunLUiyAzgG/BlwATidZKGq\nzk6s2Qf8HfCWqno6yes2amBJ0toMeYR/E7BUVeeq6jngPuDQqjXvBY5V1dMAVfXkbMeUJK3XkODv\nBs5PHF8Yn5t0HXBdku8kOZXkwMVuKMnRJItJFpeXl9c2sSRpTWb1ou1OYB9wC3AE+EySq1cvqqrj\nVTVfVfNzc3MzumtJ0hBDgv8EsHfieM/43KQLwEJV/aqqfgj8gNEPAEnSZWJI8E8D+5Jcm+QK4DCw\nsGrN1xg9uifJLkZP8Zyb4ZySpHWaGvyqeh64EzgJPAbcX1VnktyT5OB42UngqSRngQeAD1XVUxs1\ntCTppUtVbckdz8/P1+Li4pbctyS9XCV5uKrm1/K1vtNWkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lN\nGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6Qm\nDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1IT\nBl+SmjD4ktSEwZekJgYFP8mBJI8nWUpy14use0eSSjI/uxElSbMwNfhJdgDHgFuB/cCRJPsvsu4q\n4G+Ah2Y9pCRp/YY8wr8JWKqqc1X1HHAfcOgi6z4GfBz4xQznkyTNyJDg7wbOTxxfGJ/7P0luBPZW\n1ddf7IaSHE2ymGRxeXn5JQ8rSVq7db9om+QVwCeAD05bW1XHq2q+qubn5ubWe9eSpJdgSPCfAPZO\nHO8Zn3vBVcCbgW8n+RFwM7DgC7eSdHkZEvzTwL4k1ya5AjgMLLxwZVU9U1W7quqaqroGOAUcrKrF\nDZlYkrQmU4NfVc8DdwIngceA+6vqTJJ7khzc6AElSbOxc8iiqjoBnFh17u5LrL1l/WNJkmbNd9pK\nUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAl\nqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS\n1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpoYFPwkB5I8nmQpyV0Xuf4DSc4meTTJN5O8\ncfajSpLWY2rwk+wAjgG3AvuBI0n2r1r2CDBfVX8AfBX4h1kPKklanyGP8G8ClqrqXFU9B9wHHJpc\nUFUPVNWz48NTwJ7ZjilJWq8hwd8NnJ84vjA+dyl3AN+42BVJjiZZTLK4vLw8fEpJ0rrN9EXbJLcD\n88C9F7u+qo5X1XxVzc/Nzc3yriVJU+wcsOYJYO/E8Z7xuf8nyduBDwNvrapfzmY8SdKsDHmEfxrY\nl+TaJFcAh4GFyQVJbgA+DRysqidnP6Ykab2mBr+qngfuBE4CjwH3V9WZJPckOThedi/wauArSf49\nycIlbk6StEWGPKVDVZ0ATqw6d/fE5bfPeC5J0oz5TltJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh\n8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow\n+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaGBT8JAeSPJ5kKcldF7n+N5J8eXz9Q0mumfWgkqT1mRr8JDuAY8CtwH7gSJL9\nq5bdATxdVb8L/BPw8VkPKklanyGP8G8ClqrqXFU9B9wHHFq15hDwb+PLXwXeliSzG1OStF47B6zZ\nDZyfOL4A/NGl1lTV80meAX4b+NnkoiRHgaPjw18m+f5aht6GdrFqrxpzL1a4FyvcixW/t9YvHBL8\nmamq48BxgCSLVTW/mfd/uXIvVrgXK9yLFe7FiiSLa/3aIU/pPAHsnTjeMz530TVJdgKvBZ5a61CS\npNkbEvzTwL4k1ya5AjgMLKxaswD85fjyXwDfqqqa3ZiSpPWa+pTO+Dn5O4GTwA7gs1V1Jsk9wGJV\nLQD/CnwhyRLwc0Y/FKY5vo65txv3YoV7scK9WOFerFjzXsQH4pLUg++0laQmDL4kNbHhwfdjGVYM\n2IsPJDmb5NEk30zyxq2YczNM24uJde9IUkm27Z/kDdmLJO8cf2+cSfLFzZ5xswz4P/KGJA8keWT8\n/+S2rZhzoyX5bJInL/VepYx8crxPjya5cdANV9WG/WP0Iu9/AL8DXAF8D9i/as1fAZ8aXz4MfHkj\nZ9qqfwP34k+B3xxffl/nvRivuwp4EDgFzG/13Fv4fbEPeAT4rfHx67Z67i3ci+PA+8aX9wM/2uq5\nN2gv/gS4Efj+Ja6/DfgGEOBm4KEht7vRj/D9WIYVU/eiqh6oqmfHh6cYvedhOxryfQHwMUafy/SL\nzRxukw3Zi/cCx6rqaYCqenKTZ9wsQ/aigNeML78W+MkmzrdpqupBRn/xeCmHgM/XyCng6iSvn3a7\nGx38i30sw+5Lramq54EXPpZhuxmyF5PuYPQTfDuauhfjX1H3VtXXN3OwLTDk++I64Lok30lyKsmB\nTZtucw3Zi48Ctye5AJwA3r85o112XmpPgE3+aAUNk+R2YB5461bPshWSvAL4BPDuLR7lcrGT0dM6\ntzD6re/BJL9fVf+1pVNtjSPA56rqH5P8MaP3/7y5qv5nqwd7OdjoR/h+LMOKIXtBkrcDHwYOVtUv\nN2m2zTZtL64C3gx8O8mPGD1HubBNX7gd8n1xAVioql9V1Q+BHzD6AbDdDNmLO4D7Aarqu8CrGH2w\nWjeDerLaRgffj2VYMXUvktwAfJpR7Lfr87QwZS+q6pmq2lVV11TVNYxezzhYVWv+0KjL2JD/I19j\n9OieJLsYPcVzbjOH3CRD9uLHwNsAkryJUfCXN3XKy8MC8K7xX+vcDDxTVT+d9kUb+pRObdzHMrzs\nDNyLe4FXA18Zv27946o6uGVDb5CBe9HCwL04Cfx5krPAfwMfqqpt91vwwL34IPCZJH/L6AXcd2/H\nB4hJvsToh/yu8esVHwFeCVBVn2L0+sVtwBLwLPCeQbe7DfdKknQRvtNWkpow+JLUhMGXpCYMviQ1\nYfAlqQmDL0lNGHxJauJ/Acz2XLpusNoKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"HLhRirwIhczo","colab_type":"text"},"source":["#### What is a loss function?"]},{"cell_type":"code","metadata":{"id":"WLqJmTxYW5R_","colab_type":"code","outputId":"6bae5460-7505-4004-da9b-2806b2b3e39c","executionInfo":{"status":"ok","timestamp":1557753878940,"user_tz":-480,"elapsed":764,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["house = pd.read_csv('/content/boston.csv')\n","house.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>CRIM</th>\n","      <th>ZN</th>\n","      <th>INDUS</th>\n","      <th>CHAS</th>\n","      <th>NX</th>\n","      <th>RM</th>\n","      <th>AGE</th>\n","      <th>DIS</th>\n","      <th>RAD</th>\n","      <th>TAX</th>\n","      <th>PTRATIO</th>\n","      <th>B</th>\n","      <th>LSTAT</th>\n","      <th>MEDV</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.00632</td>\n","      <td>18.0</td>\n","      <td>2.31</td>\n","      <td>0</td>\n","      <td>0.538</td>\n","      <td>6.575</td>\n","      <td>65.2</td>\n","      <td>4.0900</td>\n","      <td>1</td>\n","      <td>296.0</td>\n","      <td>15.3</td>\n","      <td>396.90</td>\n","      <td>4.98</td>\n","      <td>24.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.02731</td>\n","      <td>0.0</td>\n","      <td>7.07</td>\n","      <td>0</td>\n","      <td>0.469</td>\n","      <td>6.421</td>\n","      <td>78.9</td>\n","      <td>4.9671</td>\n","      <td>2</td>\n","      <td>242.0</td>\n","      <td>17.8</td>\n","      <td>396.90</td>\n","      <td>9.14</td>\n","      <td>21.6</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.02729</td>\n","      <td>0.0</td>\n","      <td>7.07</td>\n","      <td>0</td>\n","      <td>0.469</td>\n","      <td>7.185</td>\n","      <td>61.1</td>\n","      <td>4.9671</td>\n","      <td>2</td>\n","      <td>242.0</td>\n","      <td>17.8</td>\n","      <td>392.83</td>\n","      <td>4.03</td>\n","      <td>34.7</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.03237</td>\n","      <td>0.0</td>\n","      <td>2.18</td>\n","      <td>0</td>\n","      <td>0.458</td>\n","      <td>6.998</td>\n","      <td>45.8</td>\n","      <td>6.0622</td>\n","      <td>3</td>\n","      <td>222.0</td>\n","      <td>18.7</td>\n","      <td>394.63</td>\n","      <td>2.94</td>\n","      <td>33.4</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.06905</td>\n","      <td>0.0</td>\n","      <td>2.18</td>\n","      <td>0</td>\n","      <td>0.458</td>\n","      <td>7.147</td>\n","      <td>54.2</td>\n","      <td>6.0622</td>\n","      <td>3</td>\n","      <td>222.0</td>\n","      <td>18.7</td>\n","      <td>396.90</td>\n","      <td>5.33</td>\n","      <td>36.2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["      CRIM    ZN  INDUS  CHAS     NX     RM   AGE     DIS  RAD    TAX  \\\n","0  0.00632  18.0   2.31     0  0.538  6.575  65.2  4.0900    1  296.0   \n","1  0.02731   0.0   7.07     0  0.469  6.421  78.9  4.9671    2  242.0   \n","2  0.02729   0.0   7.07     0  0.469  7.185  61.1  4.9671    2  242.0   \n","3  0.03237   0.0   2.18     0  0.458  6.998  45.8  6.0622    3  222.0   \n","4  0.06905   0.0   2.18     0  0.458  7.147  54.2  6.0622    3  222.0   \n","\n","   PTRATIO       B  LSTAT  MEDV  \n","0     15.3  396.90   4.98  24.0  \n","1     17.8  396.90   9.14  21.6  \n","2     17.8  392.83   4.03  34.7  \n","3     18.7  394.63   2.94  33.4  \n","4     18.7  396.90   5.33  36.2  "]},"metadata":{"tags":[]},"execution_count":62}]},{"cell_type":"code","metadata":{"id":"OsuYoUu9hfjY","colab_type":"code","outputId":"c3b00eb6-d954-4e1c-f7bb-bc42f57807e7","executionInfo":{"status":"ok","timestamp":1557753896188,"user_tz":-480,"elapsed":1030,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["house.shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(506, 14)"]},"metadata":{"tags":[]},"execution_count":63}]},{"cell_type":"code","metadata":{"id":"O14BH4dbXmxQ","colab_type":"code","outputId":"bda21e9b-e39d-4883-bc69-a614ab68f7d6","executionInfo":{"status":"ok","timestamp":1557754038199,"user_tz":-480,"elapsed":755,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["y = house['MEDV'].values\n","X = house.drop('MEDV', axis=1).values\n","print(X.shape)\n","print(y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(506, 13)\n","(506,)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"GuNaXNyWg-q7","colab_type":"text"},"source":["The loss is the square of the difference between the true and predicted y-values (because we want them to be similar)."]},{"cell_type":"code","metadata":{"id":"LhJNNtM9ZJdO","colab_type":"code","colab":{}},"source":["# The squared error, summed over training examples\n","def my_loss(w):\n","    s = 0\n","    for i in range(y.size):\n","        # Get the true and predicted target values for example 'i'\n","        y_i_true = y[i]\n","        y_i_pred = w@X[i]\n","        s = s + (y_i_true - y_i_pred)**2\n","    return s"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"xSWZDwWshiLF","colab_type":"code","colab":{}},"source":["from scipy.optimize import minimize"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Elw99i3mZL4F","colab_type":"code","outputId":"ae71d81a-4143-4421-84cf-d1a9bbcad6ae","executionInfo":{"status":"ok","timestamp":1557754416767,"user_tz":-480,"elapsed":3687,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":85}},"source":["# Returns the w that makes my_loss(w) smallest\n","w_fit = minimize(my_loss, X[0]).x\n","print(w_fit)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[-9.28967722e-02  4.87150034e-02 -4.05553520e-03  2.85397111e+00\n"," -2.86852806e+00  5.92816176e+00 -7.26920487e-03 -9.68510866e-01\n","  1.71156467e-01 -9.39660543e-03 -3.92190208e-01  1.49055350e-02\n"," -4.16303289e-01]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"D8YCgQMLZprl","colab_type":"code","colab":{}},"source":["from sklearn.linear_model import LinearRegression"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"PrM847lyZRZl","colab_type":"code","outputId":"9d74cd6d-6e75-4b25-dacb-2ca18720453d","executionInfo":{"status":"ok","timestamp":1557754472415,"user_tz":-480,"elapsed":940,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":85}},"source":["# Compare with scikit-learn's LinearRegression coefficients\n","lr = LinearRegression(fit_intercept=False).fit(X,y)\n","print(lr.coef_)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[-9.28965170e-02  4.87149552e-02 -4.05997958e-03  2.85399882e+00\n"," -2.86843637e+00  5.92814778e+00 -7.26933458e-03 -9.68514157e-01\n","  1.71151128e-01 -9.39621540e-03 -3.92190926e-01  1.49056102e-02\n"," -4.16304471e-01]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"OnCPIjEyhgjF","colab_type":"text"},"source":["#### Loss function diagrams\n","\n","In this exercise you'll create a plot of the logistic and hinge losses using their mathematical expressions, which are provided to you.\n","\n","*   Evaluate the log_loss() and hinge_loss() functions at the grid points so that they are plotted.\n","\n"]},{"cell_type":"code","metadata":{"id":"yZbowFhkZcAd","colab_type":"code","colab":{}},"source":["# Mathematical functions for logistic and hinge losses\n","def log_loss(raw_model_output):\n","   return np.log(1+np.exp(-raw_model_output))\n","def hinge_loss(raw_model_output):\n","   return np.maximum(0,1-raw_model_output)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"M66W_7X_hbab","colab_type":"code","outputId":"fc538ae0-c661-43b3-c57d-9bb9abcc5c57","executionInfo":{"status":"ok","timestamp":1557756609489,"user_tz":-480,"elapsed":951,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":269}},"source":["# Create a grid of values and plot\n","grid = np.linspace(-2,2,1000)\n","plt.plot(grid, log_loss(grid), label='logistic')\n","plt.plot(grid, hinge_loss(grid), label='hinge')\n","plt.legend()\n","plt.show()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8FWXa//HPlR4IKSSBAAESQDrS\nQpOqFGkWQEBAmhTRFXX1cdUt+ujjrn1VdNVVQEGQIiIi0pEuLUCQLh0SWkhIaKEkuX9/zJEfYEJC\nOMmcnHO9X6+8PMm5M/PNgBeTmXuuW4wxKKWUci9edgdQSinlfFrclVLKDWlxV0opN6TFXSml3JAW\nd6WUckNa3JVSyg1pcVdKKTekxV0ppdyQFnellHJDPnbtOCIiwsTExNi1e6WUKpY2btx4yhgTmdc4\n24p7TEwM8fHxdu1eKaWKJRE5lJ9xellGKaXckBZ3pZRyQ1rclVLKDdl2zV0ppa5cuUJiYiIXL160\nO4rLCQgIIDo6Gl9f3wJ9vxZ3pZRtEhMTKVWqFDExMYiI3XFchjGGlJQUEhMTiY2NLdA28rwsIyIB\nIrJeRLaIyHYReTWHMf4iMk1E9orIOhGJKVAapZRHuXjxIuHh4VrYbyAihIeH39ZvNPm55n4JuMcY\nUx9oAHQWkeY3jBkGnDbGVAPeB94qcCKllEfRwp6z2z0ueRZ3Yznn+NTX8XHj2nwPABMcr2cA7aWw\n/sTOp8C8F+FKRqFsXiml3EG+ZsuIiLeIJAAngUXGmHU3DKkAHAEwxmQC6UB4DtsZKSLxIhKfnJxc\nsMQHlsG6z2DCfXCugNtQSimHoKCgAn/v8OHD2bFjR67vf/XVVxw9ejTf450pX8XdGJNljGkARANN\nRaRuQXZmjPncGBNnjImLjMzz6dmc1e0FfSbC8a0wrgOc2lOw7Sil1G0aO3YstWvXzvX9G4t7XuOd\n6ZbmuRtj0oClQOcb3koCKgKIiA8QAqQ4I2COat8PQ36CS+dgbAc4uLrQdqWU8gzGGJ5//nnq1q1L\nvXr1mDZtGgDZ2dk88cQT1KxZk44dO9K1a1dmzJgBQLt27YiPjycrK4shQ4Zc/d7333+fGTNmEB8f\nz4ABA2jQoAEZGRlXxwPMnz+fRo0aUb9+fdq3b+/0nyfPqZAiEglcMcakiUgg0JE/3jCdDQwG1gAP\nAT8bY268Lu9c0XEwfDFM7g1fPwgPfAJ39i7UXSqlCs+rP25nx9EzTt1m7fLBvHJfnXyNnTlzJgkJ\nCWzZsoVTp07RpEkT2rRpw+rVqzl48CA7duzg5MmT1KpVi0cfffS6701ISCApKYlt27YBkJaWRmho\nKB9//DHvvvsucXFx141PTk5mxIgRrFixgtjYWFJTU53zA18jP2fu5YClIvIrsAHrmvscEXlNRO53\njBkHhIvIXuBZ4EWnJ81J6VgYthCim8LM4bD8HSjkf1OUUu5p1apV9OvXD29vb8qWLUvbtm3ZsGED\nq1atonfv3nh5eREVFcXdd9/9h++tUqUK+/fvZ/To0cyfP5/g4OCb7mvt2rW0adPm6hz20qVLO/3n\nyfPM3RjzK9Awh6+/fM3ri4A9p80lSsPAmTB7NCx9HdIOQvcPwLtgT3UppeyR3zNsVxQWFsaWLVtY\nsGABn332GdOnT2f8+PG2ZnKP3jI+/tDjv9D2Bdg8CSY/BBfT7U6llCpGWrduzbRp08jKyiI5OZkV\nK1bQtGlTWrZsyXfffUd2djYnTpxg2bJlf/jeU6dOkZ2dTa9evXj99dfZtGkTAKVKleLs2bN/GN+8\neXNWrFjBgQMHAArlsoz7tB8Qgbv/CqGV4cenYNy9MGA6hFayO5lSqhjo0aMHa9asoX79+ogIb7/9\nNlFRUfTq1YslS5ZQu3ZtKlasSKNGjQgJCbnue5OSkhg6dCjZ2dkAvPHGGwAMGTKEUaNGERgYyJo1\na66Oj4yM5PPPP6dnz55kZ2dTpkwZFi1a5NSfRwr7vmdu4uLiTKEt1rF/GUwbBL4B0H8alP/DVSWl\nlAvYuXMntWrVsjtGns6dO0dQUBApKSk0bdqU1atXExUVVej7zen4iMhGY0xcLt9ylXtclrlRlXYw\nbAF4+8GXXWH3PLsTKaWKse7du9OgQQNat27NP/7xjyIp7LfLfS7L3KhMLRi+BKb0han9ofNb0Gyk\n3amUUsVQTtfZXZ17nrn/rlRZ62Gn6p1h3vMw/6+QnWV3KqWUKnTuXdwB/EpC30nQbBSs/Q9MHwSX\nL9idSimlCpX7F3cAL2/o8hZ0fhN2/QQTusO5k3anUkqpQuMZxf13zR+HhyfDiR0wtj0k77Y7kVJK\nFQrPKu4ANbvB0J+sfvDjOsKBlXYnUkrZ6ODBg9St+8dGty+//DKLFy+2IZFzeF5xB6jQ2JpJExQF\nX/eALVPtTqSUcjGvvfYaHTp0sDtGgXlmcQcIq2w1HavUHL5/DJa9pU3HlPJQWVlZjBgxgjp16tCp\nUycyMjIYMmTI1da+MTExvPLKKzRq1Ih69eqxa9cuwOru2LFjR+rUqcPw4cOpXLkyp06dAmDSpEk0\nbdqUBg0a8Nhjj5GVVbQz9dx3nnt+BIbCIzOtdgXL/gWnD8J9H4KPn93JlPI88160FuFxpqh60OXN\nPIft2bOHKVOm8MUXX9CnTx++++67P4yJiIhg06ZNfPLJJ7z77ruMHTuWV199lXvuuYeXXnqJ+fPn\nM27cOMB6snTatGmsXr0aX19fnnjiCSZPnsygQYOc+/PdhGcXd7AK+YOfQlisVeDTj1hTJwND7U6m\nlCoisbGxNGjQAIDGjRtz8ODBP4zp2bPn1fdnzpwJWG2Cv//+ewA6d+5MWFgYAEuWLGHjxo00adIE\ngIyMDMqUKVPYP8Z1tLiD1XSs3QtWk7HZo2H8vdB/unXpRilVNPJxhl1Y/P39r7729vYmIyMj1zHe\n3t5kZmbedHvGGAYPHny1gZgdPPeae04a9IOB38PZY9byfUmb7E6klHJhLVu2ZPr06QAsXLiQ06dP\nA9C+fXtmzJjByZPW8zSpqakcOnSoSLNpcb9RbGsYtsjqKPlVN+uhJ6WUysErr7zCwoULqVu3Lt9+\n+y1RUVGUKlWK2rVr8/rrr9OpUyfuvPNOOnbsyLFjx4o0m3u2/HWGcyfhm75wdDN0fsN6AEop5VTF\npeVvbi5duoS3tzc+Pj6sWbOGxx9/nISEBKdt/3Za/uo199wElbGajs0cAfNftGbS3Psvq5WBUkoB\nhw8fpk+fPmRnZ+Pn58cXX3xhd6SrtLjfjF8J6DMRFv7DajqWdgR6fWE1I1NKebw77riDzZs32x0j\nR3rNPS9e3tD5X9DlHfhtnnUd/uwJu1Mp5TbsujTs6m73uGhxz69mI+Hhb6xmY2M7wMmddidSqtgL\nCAggJSVFC/wNjDGkpKQQEBBQ4G3oDdVbdXSzdaP1ykXo+zVUaWt3IqWKrStXrpCYmMjFixftjuJy\nAgICiI6OxtfX97qv5/eGqhb3gkg7DJP7QMoeuP8jaNDf7kRKKQ/htAWyRaSiiCwVkR0isl1Ens5h\nTDsRSReRBMfHywUNXiyEVrIW4K7cEmY9Dkv/pU3HlFIuJT+zZTKB54wxm0SkFLBRRBYZY3bcMG6l\nMaa78yO6qIAQGDAD5vwZlr8Fpw/B/WPAxz/v71VKqUKWZ3E3xhwDjjlenxWRnUAF4Mbi7nl8/OCB\nj6F0DPz8OpxJsq7DB4bZnUwp5eFuabaMiMQADYF1ObzdQkS2iMg8EamTy/ePFJF4EYlPTk6+5bAu\nSQTaPA89v4Aj62BcJ+uBJ6WUslG+i7uIBAHfAc8YY87c8PYmoLIxpj7wETArp20YYz43xsQZY+Ii\nIyMLmtk13dnHajp27qQ1VTKxmN4sVkq5hXwVdxHxxSrsk40xM2983xhzxhhzzvF6LuArIhFOTVoc\nxLSC4YutJ1i/6gY7f7Q7kVLKQ+VntowA44Cdxph/5zImyjEOEWnq2G6KM4MWGxF3wLDFULYuTBsI\na/6jM2mUUkUuP7NlWgIDga0i8nu7s78ClQCMMZ8BDwGPi0gmkAE8bDz5kbOgSBgyB2aOhAV/hdQD\n0PlN8NZWPkqpopGf2TKrAMljzMfAx84K5RZ8A6H3BFj8MvzykbV8X69x4B9kdzKllAfQ3jKFycsL\nOr0OXd+FPQvhq65w9rjdqZRSHkCLe1FoOgL6TYVTe+GL9nBCHxFQShUuLe5Fpfq98Og8yM60FuDe\nt9TuREopN6bFvSiVqw8jlkBIRZj8EGz62u5ESik3pcW9qIVEw6PzIbYNzH4SlvyfTpVUSjmdFnc7\nBARD/+nQaBCsfNdapzXzkt2plFJuRCde28XbF+4bA2ExsOQ1SE+ChydDidJ2J1NKuQE9c7eTCLR+\nzpr/nhQP4zpC6n67Uyml3IAWd1dQ7yEYNBsupFhNx46stzuRUqqY0+LuKiq3sHrS+AfDhPtgxw92\nJ1JKFWNa3F1JRDWrq2TUnTB9MKweozNplFIFosXd1ZSMgMGzofYDsOgf8NNzkJVpdyqlVDGjs2Vc\nkW8gPPQlLImB1R9YTcce+lKbjiml8k3P3F2Vlxd0fBW6vw97l8CXneHMUbtTKaWKCS3uri7uUeg/\nzeoJP7YDHN9mdyKlVDGgxb04uKOj1bLAGBjfGfYutjuRUsrFaXEvLqLqWTNpwirD5D6w8Su7Eyml\nXFixLO4p5zy0D0tIBRg6D6reDT8+DYtfhexsu1MppVxQsSvu87cdp83bS/khIcnuKPYICIZ+06Dx\nEFj1b/huGFy5aHcqpZSLKXbFvUHFUOqUD+HpqQm8NPNXLl7JsjtS0fP2ge4fQIdXYftMmPgAnE+x\nO5VSyoUUu+IeFRLANyOa8Xi7qkxZf4Qen/zC/uRzdscqeiLQ6hlr/vvRzVbTsZR9dqdSSrmIYlfc\nAXy8vXihc02+HNqE4+kZ3PfRKn7c4qFzwOv2hME/QsZpa6rk4XV2J1JKuYBiWdx/d3eNMvz0VGtq\nlgtm9JTN/H3WVs+8TFOpmTWTJjDMajq2babdiZRSNsuzuItIRRFZKiI7RGS7iDydwxgRkTEisldE\nfhWRRoUT94/KhwYydWRzHmtThUlrD9Pr0184lHK+qHbvOsKrWgW+QiOYMRRWva9Nx5TyYPk5c88E\nnjPG1AaaA38Skdo3jOkC3OH4GAl86tSUefD19uKlrrUYOyiOxNMZdB+zirlbjxVlBNdQojQMnAV1\ne8Hi/4U5z2jTMaU8VJ7F3RhzzBizyfH6LLATqHDDsAeAicayFggVkXJOT5uHDrXL8tNTrahaJogn\nJm/ilR+2cSnTwy7T+AZAz7HQ6lnrQacpfeHSWbtTKaWK2C1dcxeRGKAhcONduwrAkWs+T+SP/wAU\nieiwEkx/rAXDWsUyYc0hen+2hsMpF+yIYh8vL+jwirVG676lML6LtUarUspj5Lu4i0gQ8B3wjDHm\nTEF2JiIjRSReROKTk5MLsol88fPx4h/da/PfgY05cOo83T5ayfxtxwttfy6r8WAY8C2cPghj28Ox\nX+1OpJQqIvkq7iLii1XYJxtjcpqKkQRUvObzaMfXrmOM+dwYE2eMiYuMjCxI3ltyb50o5j7VmtiI\nkoyatJFXf9zO5UwPe1y/Wnur6Zh4wZddYM8iuxMppYpAfmbLCDAO2GmM+Xcuw2YDgxyzZpoD6cYY\nl7ijWbF0Cb4d1YIhd8Xw5eqD9P6vB16miaoLw5dA6Vj4pi/Ej7c7kVKqkOXnzL0lMBC4R0QSHB9d\nRWSUiIxyjJkL7Af2Al8ATxRO3ILx9/Hmf++vw6cDGrE/+Rzdxqxkzq8e9tBTcDmr6Vi19jDnz7Do\nZW06ppQbE2PTXOi4uDgTHx9f5Ps9knqB0VM2k3AkjX5NK/Jy9zoE+nkXeQ7bZGXCvL9A/Dio/SD0\n+Mxa1k8pVSyIyEZjTFxe44r1E6oF8ftlmt9709z/8Sp2H/egqYLePtDtPej0OuyYBRPuh/On7E6l\nlHIyjyvuYD309ELnmkx8tCmnL1zm/o9X8c26w9j1W0yRE4G7RkPvCXD8V6snzam9dqdSSjmRRxb3\n37WpHsncp1vTNLY0f/1+K09+s5n0jCt2xyo6dR6EwXOsh5zGdYBDa+xOpJRyEo8u7gBlSgUwYWhT\nXuxSkwXbj9NtzEo2HT5td6yiU7EJDF8EJcJh4v2wdYbdiZRSTuDxxR3Ay0sY1bYq00e1AKDPZ2v4\ndNk+srM95DJN6SowbBFUiLNWdlr5njYdU6qY0+J+jUaVwvjpqdbcWyeKt+bvYvCX60k+6yHrtZYo\nDYNmQb3esOQ1mD0asjzoEpVSbkaL+w1CAn35uH9D/tWjHusPpNLlw5Ws3FN4rRJcio8/9PwC2jwP\nm7+Gyb3hYrrdqZRSBaDFPQciQv9mlZj9ZCtKl/Rl4Lj1vDlvl2e0LhCBe/4O938MB1c6mo4l2p1K\nKXWLtLjfRI2oUvzwp1b0a1qJz5bvo9env7DPU9ZrbTQQBsyA9CPwRXs4mmB3IqXULdDinodAP2/e\n6FmPzx5pzJHTF+g+ZhVT1nvInPiqd8OjC8DLB77sCr8tsDuRUiqftLjnU+e6Ucx/ug2NKofy0syt\njJq0kdPnL9sdq/CVrQ0jlkBENZjyMKz/wu5ESql80OJ+C6JCAvj60Wb8rWstft51ks4frmD1Xg94\ndL9UFAyZC3d0grn/Awv+pk3HlHJxWtxvkZeXMKJNFb5/oiVB/j4MGLuOf83d6f7L+fkHwcPfQNOR\nsOZj+HYwXMmwO5VSKhda3AuoboUQ5oxuzYBmlfh8xX56fvILe0+6+c1WL2/o8jbc+wbs/BEm3Afn\nPGSaqFLFjBb32xDo580/e9Tj84GNOZqWQfePVjJp7SH3vtkqAi2egL5fw/Ft1vJ9yb/ZnUopdQMt\n7k7QqU4UC55pQ5OY0vx91jZGTNxIyjk3f7K11n0w5Ce4cgHGdYSDq+1OpJS6hhZ3JykTbDUg+3u3\nWqz4LZnOH65k+W9ufskiujEMXwxBZeDrB+HX6XYnUko5aHF3Ii8vYXjrKsz6U0tCA30ZPH49L/+w\njYzLbnyzNSwGhi2Eis1g5ghY/o42HVPKBWhxLwS1ywfz4+hWDG0Zw8Q1h+g2ZiUJR9LsjlV4AsPg\nkZlw58Ow9HX44UltOqaUzbS4F5IAX29eua8Ok4c3I+NKFr0+/YV/L/qNK1luOj/cx89aj7Xti5Aw\nCSb1ggw3/gdNKRenxb2QtawWwfxn2nB//fKMWbLHMWXSTddsFYG7X4IHP4VDq2F8Z0g7bHcqpTyS\nFvciEBLoy/t9G/DJgEYknr5AtzGrGL/qgPsuBtKgv3WZ5sxRa33Wo5vtTqSUx9HiXoS61ivHgmfa\ncFfVcF6bs4OB49dxNM1Nn/Ks0ta60ertbzUd2z3P7kRKeRQt7kWsTHAA44c04Y2e9dh8OI17P1jB\n95sT3fPBpzI1ramSkTVgan9Y97ndiZTyGHkWdxEZLyInRWRbLu+3E5F0EUlwfLzs/JjuRUTo17QS\n855uTfWypfjztC386ZtNpLpjl8lSZa2Hnap3gXnPw/yXINuNp4Yq5SLyc+b+FdA5jzErjTENHB+v\n3X4sz1A5vCTTH2vBXzrXYNGOE9z7wQoW7Thhdyzn8ytptSto9jis/QSmD4LLF+xOpZRby7O4G2NW\nAKlFkMUjeXsJT7Srxqw/tSS8pB8jJsbz52kJpF1ws7N4L2/o8iZ0fgt2/QRfdYNzJ+1OpZTbctY1\n9xYiskVE5olIHSdt06PUKR/C7Cdb8VT7O/hxy1E6vr+Cxe54Ft98FDw8GU7udDQd2213IqXckjOK\n+yagsjGmPvARMCu3gSIyUkTiRSQ+OdnN+64UgJ+PF892rH71LH74xHienZZA+gU3e9qzZjcY+hNc\nuWg1HTuwwu5ESrmd2y7uxpgzxphzjtdzAV8Richl7OfGmDhjTFxkZOTt7tpt1a3gOIu/pxo/bDlK\nx/eXs2Snm53FV3A0HStVDr7uCVum2p1IKbdy28VdRKJERByvmzq2mXK72/V0fj5ePNupBj/8qSWl\nS/oxbEI8z053s7P4sMrWAtyVW8D3j8GyN7XpmFJOkp+pkFOANUANEUkUkWEiMkpERjmGPARsE5Et\nwBjgYeOWk7bt8ftZ/Oh7qvFDwlE6fbCcn3e50Vl8YCgM+A7q94dlb8CsxyHTzW4mK2UDsasOx8XF\nmfj4eFv2XVxtTUznf77dwu4TZ+nVKJqX76tNSKCv3bGcwxhY8Q4s/SfEtIa+k6zCr5S6johsNMbE\n5TVOn1AtRupFhzB7dEtG31ONWQlJdPz3cuZvO253LOcQgbZ/gR6fw+G1MK4TnD5kdyqlii0t7sWM\nv483zzmuxUcE+TNq0kYen7SRk2cv2h3NOer3hYHfw7nj1lTJpI12J1KqWNLiXkzVrRDCD0+25C+d\na7Bk10k6vLecaRsOu0ePmtjWMGwR+AbCl92sh56UUrdEi3sx5uvtxRPtqjH/6dbULBfMC99tZcDY\ndRxKOW93tNsXWQOGL4GytWHqAFj7qd2JlCpWtLi7gSqRQUwd0Zx/9qjL1sR07v1gBZ+v2EdmcV/1\nKagMDJ5jPfQ0/0WY94I2HVMqn7S4uwkvL2FAs8oserYtrapF8q+5u+jxyS9sP5pud7Tb41cC+kyE\nFk/Cus9g2iNw2Q1+M1GqkGlxdzNRIQF8Magx/+nfiGPpGdz/8Wrenr+Li1eK8Rmvlzfc+0/o+i78\nNt9a/OOsG831V6oQaHF3QyJCtzvLsfjZtvRoWIFPlu2jy4cr+WXvKbuj3Z6mI+DhKXDqN2v5vpM7\n7U6klMvS4u7GQkv48W7v+nw9rCnZxtB/7Dr+PC2BU+cu2R2t4Gp0hqFzIesSjLsX9i+zO5FSLkmL\nuwdofUckC55pw+h7qjHn16Pc8+4yvll3uPgu0F2+oTWTJrg8TOoFmyfbnUgpl6PF3UME+FoPP817\nug21ywfz1++38tBnv7Dz2Bm7oxVMaEUYtgBiWsEPT8DP/9SmY0pdQ4u7h6lWJogpI5rzXu/6HEy5\nQPePVvHG3J1cuJxpd7RbFxACA2ZAw0dgxdtWZ8nMYnzJSSkn0uLugUSEXo2jWfJsW3o3jua/K/bT\n8d/FdOUnb1+4/2O45+/w6zSrN3zGabtTKWU7Le4eLKykH2/2upMZo1oQ5O/D8InxjJwYz9G0DLuj\n3RoRaPM89BwLiethbEdIPWB3KqVspcVdERdTmjlPteLFLjVZsSeZ9u8t5z9L93Ips5jNjb+zNwyc\nBeeTramSidpSWnkuLe4KsPrUjGpblcXPtqVt9UjeWbCbe99fwdLdJ+2OdmtiWlrL9/kHwVfdYMds\nuxMpZQst7uo60WEl+GxgYyY+2hQvL2HolxsYPiGewykX7I6WfxF3WFMlo+rB9EHwy8c6k0Z5HC3u\nKkdtqkcy/+k2vNSlJr/sO0WH95fz70W/kXG5mFyqKRkBg3+EWvfBwr/B3OchqxjOCFKqgLS4q1z5\n+XjxWNuq/PxcOzrXiWLMkj10cKz+VCz6xvsGQu8JcNdTsOELmDYALp2zO5VSRUKLu8pTVEgAY/o1\nZOrI5gT5+zBq0kYGjV/PvuRiUCi9vKDT/0G392DPQviyC5w5ZncqpQqdFneVb82rhPPTU6145b7a\nJBxO4973V/DajztIv3DF7mh5azIc+k2DlH3WTJoT2+1OpFSh0uKubomPtxdDW8by8/+0o3dcNF/+\ncoC27y5lwi8HueLqi4NU7wSPzgOTBeM7w76f7U6kVKHR4q4KJLKUP2/0vJOfRremVlQwr8zeTpcP\nV7r+1Mly9a2pkiEVYXJv2PS13YmUKhRa3NVtqV0+mG9GNOPzgY3JzMpm6JcbGDx+PXtOnLU7Wu5C\nouHR+RDbFmY/CUv+T6dKKreTZ3EXkfEiclJEtuXyvojIGBHZKyK/ikgj58dUrkxE6FQnioV/bsvf\nu9Vi0+HTdP5wJS//sI3U85ftjpezgGDoPw0aDYKV78J3w7XpmHIr+Tlz/wrofJP3uwB3OD5GArpM\nvYfy8/FieOsqLH/+bvo3rcTkdYdp+85Sxq7c75qtDLx94b4x0P4V2DYDJj4IF1LtTqWUU+RZ3I0x\nK4Cb/Y1/AJhoLGuBUBEp56yAqvgpXdKP/3uwLvOebk3DSmG8/tNO2r+3nFmbk1xvgRARaP0sPDQe\nkjbCuI6Qut/uVErdNmdcc68AHLnm80TH15SHq162FBMfbcrER5sSHODLM9MSuO/jVazck2x3tD+q\n2wsG/QAXUqypkkfW251IqdtSpDdURWSkiMSLSHxysgv+D64KRZvqkcwZ3Yr3+9Yn7cIVBo5bz8Bx\n69iWlG53tOtVbmH1pAkIgQn3wfZZdidSqsCcUdyTgIrXfB7t+NofGGM+N8bEGWPiIiMjnbBrVVx4\neQk9Gkaz5DnrpuvWpHS6f7SKZ6Zu5kiqCzUlC68KwxZbUya/HQyrP9SZNKpYckZxnw0McsyaaQ6k\nG2P0+W6VowBf76s3XUe1rcq8bcdp/95y/m/ODk67ysyakuEwaDbU6QGLXoafntWmY6rYkbwaQInI\nFKAdEAGcAF4BfAGMMZ+JiAAfY82ouQAMNcbkuUpCXFyciY/XxRQ83dG0DN5f9BvfbUqkpJ8Pw1rH\nMqxVLKUCfO2OBtnZsORVWP0B3NHJuunqX8ruVMrDichGY0xcnuPs6u6nxV1da/fxs/x70W4WbD9B\naAlfHmtTlcF3VaaEn4/d0SD+S/jpOShbG/pPh+DydidSHkyLuyqWtiam896i3SzbnUxEkD9/ursq\n/ZpWIsDX295gexZb1+D9g2HAtxBV1948ymNpcVfFWvzBVN5duJu1+1MpFxLA6HvuoHdcNL7eNnbM\nOL4VJveBS2ehz1dQrYN9WZTHym9x194yyiXFxZRm6sgWfDO8GVEhAfz1+620f285321MJMuuB6Gi\n6sGIJRAWYxX5jV/Zk0OpfNAzd+XyjDEs253Muwt3s/3oGapElORPd1fjgQbl8bHjTP7SWfh2COxd\nDK3+DPe8bC0KolQR0Msyyu1kZxsW7jjOh0v2svPYGSqVLsET7arSs1E0fj5FXFyzMmHe8xA/Hur0\nhAc/Bd+Aos2gPJIWd+W2jDEoMUL9AAAS80lEQVQs2XmSj37ew5bEdCqEBjKqXVX6xEXj71OEN16N\ngV/GWHPhKzaHh7+x5sgrVYi0uCu3Z4xh+W/JjFmyh02H04gKDuCxtlWKfnbN9u9h5mMQUgEGzLCe\nclWqkGhxVx7DGMMv+1IYs2QP6w6kEhHkz8g2sfRvVpkg/yKaJ394HUztZ53N95sClZoXzX6Vx9Hi\nrjzSuv0pfPTzXlbtPUWpAB8GNq/M0JaxRJbyL/ydp+yzlu5LT4Qen1qdJpVyMi3uyqNtOZLGf1fs\nY9624/h6e9GrUTQj21QhNqJk4e74QipM7Q+H10CH/4WWz1g945VyEi3uSgEHTp3ni5X7mbExkStZ\n2XSuE8WotlWpXzG08HZ65SL88ARs+w4aD4Gu74G3C7RRUG5Bi7tS1zh59iITfjnI12sOceZiJi2q\nhPNY2yq0rR6JFMaZdXY2LH0dVr4HVdtD76+sdVuVuk1a3JXKwblLmUxdf5ixKw9w/MxFqpcNYmjL\nWHo0rFA4M2w2TYQfn4EytaymYyG6SJm6PVrclbqJy5nZ/LjlKONWHWDHsTOElfClf7NKDGweQ1SI\nkx9G2rsEpg8G/yCrwJe707nbVx5Fi7tS+WCMYf2BVMavPsDCHSfwFqHbneV4tGWsc6/Ln9hu9aO5\nmGZdormjo/O2rTyKFnelbtHhlAtMWHOQ6RuOcPZSJo0rhzG0ZQyd60Q5p4fNmWPwTR+r0Hd9B5oM\nu/1tKo+jxV2pAjp3KZMZ8Uf48peDHEq5QNlgf/o1rcTDTSrd/iWbS+dgxqOwZwHc9RR0eFWbjqlb\nosVdqduUlW1Yuuskk9YdYvlvyXiJ0LFWWR5pXpm7qobj5VXAWTZZmTD/BdgwFmo/CD0+A99A54ZX\nbiu/xV0n3yqVC28voUPtsnSoXZbDKReYvP4Q38YnMn/7cWIjSjKgWSUeahxNaAm/W9ywD3R9F8Ji\nYeHf4cxRq2VByYjC+UGUR9Izd6VuwcUrWczbdoxJaw+z8dBp/H286H5nefo3q0SjSqG3Pmd+xw8w\ncySUKmc1HYuoVjjBldvQyzJKFbKdx84wae0hZm1O4vzlLKqVCaJvXEV6NKpARNAt9LI5sgGmPAwm\ny2obXPmuwgutij0t7koVkXOXMpmz5SjT44+w6XAaPl5C+1pl6NukIm3uiMzfTJvUA1bTsbRD1sIf\n9R4q/OCqWNLirpQN9pw4y/T4I8zclETK+cuUDfanV6No+sRVJCavpmUXUmHaI3BoNdzzD2j9nDYd\nU3+gxV0pG13OzObnXSeZHn+EZbtPkm2gaUxpHmxYgW71yhFSwjfnb8y8BD88CVunQ8OB0P198M5l\nrPJITi3uItIZ+BDwBsYaY9684f0hwDtAkuNLHxtjxt5sm1rclac4ceYiMzYmMnNTIvuSz+Pn7cU9\nNcvwYMMK3F0z8o9LAxoDS/8FK96GKndDnwkQEGJPeOVynFbcRcQb+A3oCCQCG4B+xpgd14wZAsQZ\nY57Mb0At7srTGGPYlnSG7zcnMXvLUU6du0RwgA/d7ixPj4YViKscdv3c+c2T4MenIaK61ZMmtKJ9\n4ZXLcOY896bAXmPMfseGpwIPADtu+l1KqeuICPWiQ6gXHcJfu9Zk9b4UZm1OYtbmJKasP0yF0EAe\naFCebneWo3a5YKThIxASDdMGwtgO0H8alG9g94+hion8nLk/BHQ2xgx3fD4QaHbtWbrjzP0NIBnr\nLP/PxpgjN9uunrkrZTl/KZNFO04wc3MSq/eeIivbEBtRkq71ouhWrzy1vI8g3/S1brg+NB5qdLY7\nsrKRMy/L5Ke4hwPnjDGXROQxoK8x5p4ctjUSGAlQqVKlxocOHbqVn0kpt5dy7hILtp9g7tZjrNmf\ncrXQ96nhy5BDLxCQsh3p8jY0HWF3VGUTZxb3FsD/GmPudXz+EoAx5o1cxnsDqcaYm94B0jN3pW7u\n90L/09ajrNmXgr+5yLiSn3JX1gZO1RtJeI83Ea9CWGBEuTRnFncfrEst7bFmw2wA+htjtl8zppwx\n5pjjdQ/gBWNM85ttV4u7Uvl36twlFmw/zrxfE+lw6EOG+CxgqVdzVtf7J+3qxtCsSml8ndGWWLk8\nZ0+F7Ap8gDUVcrwx5p8i8hoQb4yZLSJvAPcDmUAq8LgxZtfNtqnFXamCSTl3icNz36P+jrf51VRl\n2KXnuBwQzt01ytCxdlna1YikVIDOjXdX+hCTUu5u5xzMd8PJ8A/nP+XfYOr+QFLOX8bXW2heJdwq\n9NXLUCm8hN1JlRNpcVfKEyRuhCl9IesKWX0mscmrDot2nGDRjhMcOHUegCoRJWlbI5J2NcrQLLZ0\n4SwEroqMFnelPMXpg9b6rKn74YH/QP2+ABw4dZ5lu0+ybHcya/encCkzmwBfL1pUCaddjTK0qxFJ\n5fA8+t0ol6PFXSlPknHaetjp4Eq4+2/Q5vnrmo5dvJLFmv0pLN+dzLLdJzmYcgGA2IiStKwWTsuq\nEbSoGn7rC4+oIqfFXSlPk3kZZo+GX6dCgwHQ/QPwyblYH3Sc1S//LZn1B1I5fzkLEahTPpiWVSO4\nq1oETWLCKOGni7W5Gi3uSnkiY2D5W7DsDYhtC30mQmDoTb/lSlY2vyamsXpvCqv3nmLz4TQuZ2Xj\n6y00rBTGXVXDaVktgjujQ/7Y5EwVOS3uSnmyhG9g9lMQXg0GTIfQSvn+1ozLWWw4mMrqfaf4ZW8K\n246mYwz4+XjRIDqUJrFhNIkpTePKYTrl0gZa3JXydAdWwNRHwDcA+k2FCo0KtJn0C1dYdyCFDQdT\nWX/wNNuS0snKNngJ1CoXTJOY0jSNLU2TmNJElrqF5QVVgWhxV0rByV3W8n0XTjmajnW57U1euJzJ\n5sNprDuQyoYDqWw+cpqLV7IBiAkvQcNKYTSsFEqDiqHUjArGz0efnHUmLe5KKcvZE9YC3McSoPOb\n0Owxp27+cmY2246ms+FAKvGHTpNwJI3ks5cA61JO3fLBNKwURoOKVsGPDgtEdPnAAtPirpT6/y5f\ngJkjYNccaP4EdHodCqnpmDGGpLQMEo6kkXA4jYQjaWxNSudSpnV2HxHkT4OKodwZHULdCsHUKR9C\nmVL+WvDzSYu7Uup62Vmw8O+w9hOo2R16fgF+RdOa4EpWNruOnSXhyGk2H7EK/oFT5/m9/EQE+TsK\nfTB1y4dQt0KInuHnQou7Uipn6/4L81+Ecg2s1Z2CytgS49ylTHYeO8O2pHS2H7X+u+fkObKyrZoU\nHOBDnfIh1CkfTI2oUtSMCqZamSAC/Tx7OqYWd6VU7nbNhe+GQckIGDADImvYnQiwnqTdffysVeyP\nprM9KZ2dx89y2XFJRwQqly5BjahS1ChbihpRwdSICiImvCQ+HtLyWIu7UurmkjbBN30h6xL0nQSx\nbexOlKPMrGwOpV7gt+Nn2XX8LL+dOMvuE2c5eOo8jpN8/Ly9qFomiBplg6gaGUSVyCCqlilJTHhJ\nt2uUpsVdKZW3tMNW07GUvXD/R9Cgn92J8u3ilSz2njzH7msK/m/Hz3I0/eLVMSIQHRZIlYjfi35J\nqkSWpFpkEJHF9CZufou7No5QypOFVoJH58P0gTBrFKQdgrYvXNd0zFUF+HpTt4J18/VaFy5ncuDU\nefYln2d/8rmr/11/IJWMK1lXx5Xy96FyRAkqly5JpfASVCpdgsqlS1CxdAnKhwbi7eX6x+Bm9Mxd\nKWU1HZvzDCRMhvr94L4xuTYdK66ysw3Hz1xkf/J59iWfY1/yOQ6lXOBw6gUST1/gStb/r4W+3kJ0\nmFXoK5e2Cn+l8BJUDCtBhdBAggN9bDvr1zN3pVT++fhZveDDYmHp65CeCH2/hsAwu5M5jZeXUD40\nkPKhgbS6I+K697KyDcfSMzjsKPaHUi9cfZ1w+DRnLmZeNz7I34fyoQFXt1chNND6PMT6PCokwPY1\nbfXMXSl1vS3T4Ic/QekqMOBbCKtsdyLbpV+4wqHU8xxJzeBYegZJaRkknc7gaHoGR9Muknr+8nXj\nvQTKBgdcLfRRwY6PEOsjJrxkgfvw6A1VpVTBHVwFUweAty/0mwbRje1O5NIyLmc5Cr31kXQ6g6S0\niySlXeDEmUscT7943fX+x9pU4aWutQq0L70so5QquJhWMGwRTH4IvuoGvcZCre52p3JZgX7eVI20\nZuTkxBjDmYxMjp+5yPEzFykXElDomTxj1r9S6tZFVofhS6BsHZj2CKz91O5ExZaIEFLClxpRpWhb\nPZLqZUsV+j61uCulchcUCYN/tM7a578Ic/9i9ahRLk+Lu1Lq5vxKQO+J0OJJWP9f61r85fN2p1J5\nyFdxF5HOIrJbRPaKyIs5vO8vItMc768TkRhnB1VK2cjLC+79J3R9F/YsgC+7Wn3ilcvKs7iLiDfw\nH6ALUBvoJyK1bxg2DDhtjKkGvA+85eygSikX0HQEPDwFTu2Bse3h5E67E6lc5OfMvSmw1xiz3xhz\nGZgKPHDDmAeACY7XM4D2UhybNiil8lajMwydC1lXYFwn2L/M7kQqB/mZClkBOHLN54lAs9zGGGMy\nRSQdCAdOOSOkUsrFlG8AwxfDN31g4gMQUR1Eb+HlW8OBcNeThbqLIp3nLiIjgZEAlSpVKspdK6Wc\nLbSi1XRsxTtWd0mVf0WwQEp+insSUPGaz6MdX8tpTKKI+AAhQMqNGzLGfA58DtYTqgUJrJRyIQEh\n1nqsyuXk5/eoDcAdIhIrIn7Aw8DsG8bMBgY7Xj8E/Gzs6muglFIq7zN3xzX0J4EFgDcw3hizXURe\nA+KNMbOBccDXIrIXSMX6B0AppZRN8nXN3RgzF5h7w9devub1RaC3c6MppZQqKL29rZRSbkiLu1JK\nuSEt7kop5Ya0uCullBvS4q6UUm7ItmX2RCQZOFTAb4/ANVsbuGoucN1smuvWaK5b4465KhtjIvMa\nZFtxvx0iEp+fNQSLmqvmAtfNprlujea6NZ6cSy/LKKWUG9LirpRSbqi4FvfP7Q6QC1fNBa6bTXPd\nGs11azw2V7G85q6UUurmiuuZu1JKqZsoFsVdRN4RkV0i8quIfC8iobmMu+lC3oWQq7eIbBeRbBHJ\n9c63iBwUka0ikiAi8S6Uq0iPl2OfpUVkkYjscfw3LJdxWY7jlSAiN7aYdlYWl1z4PR+5hohI8jXH\nZ3gR5RovIidFZFsu74uIjHHk/lVEGrlIrnYikn7N8Xo5p3GFkKuiiCwVkR2O/x+fzmFM4R0zY4zL\nfwCdAB/H67eAt3IY4w3sA6oAfsAWoHYh56oF1ACWAXE3GXcQiCjC45VnLjuOl2O/bwMvOl6/mNOf\npeO9c4WcI8+fH3gC+Mzx+mFgWhEcn/zkGgJ8XFR/n67ZbxugEbAtl/e7AvMAAZoD61wkVztgjg3H\nqxzQyPG6FPBbDn+WhXbMisWZuzFmoTEm0/HpWqzVoG6Un4W8nZ1rpzFmd2HuoyDymavIj5fDtYup\nTwAeLIJ95sRVF363688lT8aYFVjrNeTmAWCisawFQkWknAvksoUx5pgxZpPj9VlgJ9Z609cqtGNW\nLIr7DR7F+pfuRjkt5H3jgbSLARaKyEbHOrKuwK7jVdYYc8zx+jhQNpdxASISLyJrRaQw/gHIz89/\n3cLvwO8Lvxem/P659HL8Gj9DRCrm8L4dXPn/wRYiskVE5olInaLeueOSXkNg3Q1vFdoxK9IFsm9G\nRBYDUTm89TdjzA+OMX8DMoHJrpQrH1oZY5JEpAywSER2Oc427M5VKG6W7dpPjDFGRHKbrlXZccyq\nAD+LyFZjzD5nZy2mfgSmGGMuichjWL9d3GNzJle2Cevv0zkR6QrMAu4oqp2LSBDwHfCMMeZMUe3X\nZYq7MabDzd4XkSFAd6C9cVysukF+FvJ2eq58biPJ8d+TIvI91q/et1XcnZCrUI4X3DybiJwQkXLG\nmGOOXz9P5rKN34/ZfhFZhnXW48zi7rSF350sz1zGmGszjMW6j+EKCu3v1O24tqAaY+aKyCciEmGM\nKfSeMyLii1XYJxtjZuYwpNCOWbG4LCMinYG/APcbYy7kMiw/C3kXOREpKSKlfn+NdXM4x7v6Rcyu\n43XtYuqDgT/8liEiYSLi73gdAbQEdjg5h6su/J5nrhuuyd6PdS3XFcwGBjlmgDQH0q+5BGcbEYn6\n/V6JiDTFqnuF/Y80jn2OA3YaY/6dy7DCO2ZFfQe5IB/AXqzrUgmOj99nMJQH5l4zrivWHel9WJcn\nCjtXD6xrZJeAE8CCG3NhzXrY4vjY7iq57Dhejn2GA0uAPcBioLTj63HAWMfru4CtjmO2FRhWSFn+\n8PMDr2GdRAAEAN86/v6tB6oU0THKK9cbjr9LW4ClQM0iyjUFOAZccfz9GgaMAkY53hfgP47cW7nJ\nDLIizvXkNcdrLXBXEeVqhXW/7ddralfXojpm+oSqUkq5oWJxWUYppdSt0eKulFJuSIu7Ukq5IS3u\nSinlhrS4K6WUG9LirpRSbkiLu1JKuSEt7kop5Yb+H43o/qKlpYE2AAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"lYmYoXMKioga","colab_type":"text"},"source":["**Implementing logistic regression**\n","\n","This is very similar to the earlier exercise where you implemented linear regression \"from scratch\" using scipy.optimize.minimize. However, this time we'll minimize the logistic loss and compare with scikit-learn's LogisticRegression (we've set C to a large value to disable regularization; more on this in Chapter 3!).\n","\n","The log_loss() function from the previous exercise is already defined in your environment, and the sklearn breast cancer prediction dataset (first 10 features, standardized) is loaded into the variables X and y."]},{"cell_type":"code","metadata":{"id":"nRUVcQmLleqB","colab_type":"code","colab":{}},"source":["from sklearn.preprocessing import StandardScaler"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"omd2bOMFkg-x","colab_type":"code","outputId":"e9a1a6eb-37fb-4174-b486-418aeb3c50cf","executionInfo":{"status":"ok","timestamp":1557757752937,"user_tz":-480,"elapsed":754,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["cancer = datasets.load_breast_cancer()\n","X_raw = cancer.data # [:,:10] # first 10 features standardized\n","y = cancer.target\n","print(X_raw.shape)\n","print(y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(569, 30)\n","(569,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"JUhUgw5Jlrmg","colab_type":"code","colab":{}},"source":["# The logistic loss, summed over training examples\n","def my_loss(w):\n","    s = 0\n","    for i in range(y.size):\n","        raw_model_output = w@X[i]\n","        s = s + log_loss(raw_model_output * y[i])\n","    return s"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"k115GOTglP4q","colab_type":"code","colab":{}},"source":["# Returns the w that makes my_loss(w) smallest\n","w_fit = minimize(my_loss, X[0]).x\n","print(w_fit)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"jL6TLs9Tk_VR","colab_type":"code","colab":{}},"source":["# Compare with scikit-learn's LogisticRegression\n","lr = LogisticRegression(fit_intercept=False, C=1000000).fit(X,y)\n","print(lr.coef_)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"CvY3lSv2hLLF","colab_type":"text"},"source":["## 3. Logistic regression"]},{"cell_type":"markdown","metadata":{"id":"qsRtTCpthjQM","colab_type":"text"},"source":["#### Logistic regression and regularization\n","\n","In Chapter 1 you used logistic regression on the handwritten digits data set. Here, we'll explore the effect of L2 regularization.\n","\n","The handwritten digits dataset is already loaded, split, and stored in the variables X_train, y_train, X_valid, and y_valid. The variables train_errs and valid_errs are already initialized as empty lists."]},{"cell_type":"code","metadata":{"id":"a_HGGC1EhNCN","colab_type":"code","outputId":"3aba8b88-bc07-440f-b2fe-eb8a2c79d4c3","executionInfo":{"status":"ok","timestamp":1557759249856,"user_tz":-480,"elapsed":741,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["digits = datasets.load_digits()\n","X = digits.data\n","y = digits.target\n","\n","print(X.shape)\n","print(y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(1797, 64)\n","(1797,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Qk4NucSasDut","colab_type":"code","outputId":"f4bdb04b-cc6b-492f-e71c-b5bf87b64db8","executionInfo":{"status":"ok","timestamp":1557759444422,"user_tz":-480,"elapsed":837,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["from sklearn.model_selection import train_test_split\n","X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=450, random_state=42)\n","print(X_train.shape)\n","print(X_valid.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(1347, 64)\n","(450, 64)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"R9HEjILxukRk","colab_type":"text"},"source":["\n","\n","*   Loop over the different values of C_value, creating and fitting a LogisticRegression model each time.\n","\n","*   Save the error on the training set and the validation set for each model.\n","\n","*   Create a plot of the training and testing error as a function of the regularization parameter, C.\n","\n","*   Looking at the plot, what's the best value of C?\n","\n"]},{"cell_type":"code","metadata":{"id":"HMolzDjguuMj","colab_type":"code","colab":{}},"source":["# Train and validaton errors initialized as empty list\n","train_errs = list()\n","valid_errs = list()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZqLu4x2juxAT","colab_type":"code","outputId":"b5e34d91-5713-4cf3-96ca-967e5657229a","executionInfo":{"status":"ok","timestamp":1557760183621,"user_tz":-480,"elapsed":3731,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":518}},"source":["# Loop over values of C\n","for C_value in [0.001, 0.01, 0.1, 1, 10, 100, 1000]:\n","    # Create LogisticRegression object and fit\n","    lr = LogisticRegression(C=C_value)\n","    lr.fit(X_train, y_train)\n","    \n","    # Evaluate error rates and append to lists\n","    train_errs.append( 1.0 - lr.score(X_train, y_train) )\n","    valid_errs.append( 1.0 - lr.score(X_valid, y_valid) )"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"SV6LIjxovgJk","colab_type":"code","outputId":"c0e5ee6b-e702-4b5b-8c20-e3a0789baa88","executionInfo":{"status":"ok","timestamp":1557760227772,"user_tz":-480,"elapsed":1860,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":273}},"source":["C_values = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n","# Plot results\n","plt.semilogx(C_values, train_errs, C_values, valid_errs)\n","plt.legend((\"train\", \"validation\"))\n","plt.show()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX5+PHPyWQjIYRsELJAAklI\n2JcEEZDFFRTBBcWFKgpi3Vpb+23pr7W1frvZWrX+3IqodUfFWuJCcWNRcEkChCVACBAgBEI2Qvb1\nfP+4QwhpIBMykzvL83695pWZe8/MPCcz88yZc849V2mtEUII4Rm8zA5ACCFEz5GkL4QQHkSSvhBC\neBBJ+kII4UEk6QshhAeRpC+EEB5Ekr4QQngQSfpCCOFBJOkLIYQHkaQvhBAexNvsANoLDw/XcXFx\nZochhBAuJSsrq0RrHdFZOadL+nFxcWRmZpodhhBCuBSl1EFbykn3jhBCeBBJ+kII4UEk6QshhAeR\npC+EEB5Ekr4QQngQSfpCCOFBJOkLIYQzOJIFBVkOfxqnm6cvhBAeo7EOdv4Lvn8RCjdDwmWwYKVD\nn1KSvhBC9LTyfMh8GTa/DrVlEJ4Es/4Co29y+FNL0hdCiJ7Q0gL7voSMFyF3DSgFQ6+ECXdB/DTj\ndg+QpC+EEI5UUwZb34SMl6D8AARGwEUPQeodEBzT4+FI0hdCCEco3Gq06revhKY6iJ0IF/8aUuaA\nt69pYUnSF0IIe2mqh53/NpJ9QQb4BMCo+ZC2GAaMMjs6QJK+EEJ034lD1oHZ16CmFMISYOafYfTN\n0Kuv2dGdwaakr5SaCfwdsADLtdZ/brffD3gNGA+UAvO11vlt9g8EcoBHtNaP2yd0IYQwUUsL7F8L\nGcsh9z/GtqRZMGExxE8HL+c8DKrTpK+UsgDPApcBBUCGUipda53TptgioFxrnaCUugl4DJjfZv8T\nwGr7hS2EECapPQFb3zKSfdk+CAiHyQ8aA7N9B5odXadsaelPAPK01vsBlFIrgLkYLfdT5gKPWK+v\nBJ5RSimttVZKXQMcAKrtFrUQQvS0Y9uNg6i2vweNNRCTBtN+AcOvAW8/s6OzmS1JPxo43OZ2AXDB\n2cporZuUUhVAmFKqDvgFxq+En3U/3HOoLoG3b4Zxt8GI68E3wKFPJ4TwAE0NsCvdSPaHvwVvfxg5\nD9LugqgxZkd3Xhw9kPsI8KTWukqd48ADpdQSYAnAwIHn+fPo5BGor4T0++HTX8PYBZB6J4QNOb/H\nE0J4rooCyHwFNr8K1cUQEg+X/wHG3AIBoWZH1y22JP0jQGyb2zHWbR2VKVBKeQPBGAO6FwDzlFJ/\nAfoCLUqpOq31M23vrLVeBiwDSE1N1edTEQaMhnu/gYMbjW/l716Ab56BhEuNb+XEy8DLcl4PLYTw\nAFrDgfVG/tizGnQLJF1h5I8hFzvtwGxX2ZL0M4BEpVQ8RnK/CbilXZl04HbgG2Ae8KXWWgMXnSqg\nlHoEqGqf8O1KKYibYlxOHoWsfxqXt+dD30FGy3/cbS7/TS2EsKO6Ctj6tjEwW7oXeoXCpPuNfBES\nZ3Z0dtdp0rf20d8PrMGYsvmy1nqnUupRIFNrnQ68BLyulMoDyjC+GMzVZwDM+CVM/Rns+tA4BPrz\n38LaPxp9/hMWQ/R4s6MUQpilaKfRqt/2LjRWG/ngmhdg+LXg4292dA6jjAa580hNTdWZmZmOefCi\nHOPbPHuF8SJHjTMWOxp+nVu/yEIIq6YG2P0hfL8cDm0Ci591YHYxRI8zO7puUUplaa1TOy3nUUn/\nlLqTRuLPeBFKco2fc2MXQNoit/w5J4THO1l4uru3qsjo7k1bBGN/4DbdvZL0baE1HNhgJP/dnxgD\nN4mXG63/IZe4zcCNEB5Ja8j/2vh87/rI+HwnXGp8vhMudbuJHbYmfc9ee0cpGDzNuFQcgaxXIOtV\neHOeMUUrbRGMudVtWgJCeIT6Susv+eVQvBv8+8KF9xoDs6GDzY7OdJ7d0u/IqYMxMpbDoW/c4mAM\nITzC8d1Gqz57BTRUwYAxRqt+xPXg08vs6BxOWvrny9vXSPIj5xmHXWcsN0b3t7xhHHadthiGXSMD\nv0I4g+ZG2P2x8TnN/wosvsbEjAl3GbNxeuhsVK5EWvq2qD0B2afm8eZBQJgx3z/1TpdYYEkIt1N5\n7PTAbOVRCB5oLHg27jYIDDc7OlPIQK4jtLQYR+xlLIc9nxjbkmYarf/BM2TgVwhH0hoObrIOzH4I\nLU3GhIu0xcaRs242MNtV0r3jCF5eMGSGcTlx+PTA755PIHSI8eYbczP0CjE7UiHcR30VbHvHOMDy\n+E7wD4YJdxsTLWRtrS6Tln53NdVDzirjyL6C78G7F4y6wRj4dZLTownhkopzrQdTvg31JyFypPG5\nGnmDrKLbAWnp9xRvPxh1o3E5mm09rPs947RpsRcYb9Jhc009EbIQLqO5yfjlnLHc6Er18jGWRUhb\nDLETZGDWDqSl7wi15bDlTch8Ccr2Q2AEjLvdGGgKjjE7OiGcT9Vxo6s06xVjmfQ+MdaB2duhd4TZ\n0bkEjxvIPVnXyPIN+1kybQi9/ZzkB0xLC+z/0ljnI/c/Ritl6JXGdLL4adJqEZ5Nazj8nfHrOGcV\ntDTC4OnGr+OkmWBxks+xi/C47p39xdU8/WUeTS2an89MNjscg5eXcbh3wqVQfhAyXza6fXZ/BGGJ\npwd+/YPNjlSIntNQbZxy8PvlULQd/PoYg7JpiyE80ezo3J7btPQBfvLOVj7efpQvfjqN2FAnHehp\nrIOdHxh9lkcywSfQGA+YcBf0H252dEI4Tkme0eW55U2or4D+I4xEP/IG8OttdnQuz+O6dwCOVdQx\n4/F1TB8awfMLXGCt/MItRmtnx0poqoOBk4x1/pOvloFf4R5amo2uze9fhP1rwcvbmNiQdhcMnChd\nnHbkkUkf4Okv9vLEZ7msWDKRiYPD7BiZA9WUGcs8ZL4E5fnQuz+MX2hc+kSZHJwQ56G6xDi/bOYr\nUHEYgqJOD8wG9Tc7OrfksUm/rrGZS/62nuBePnz4wBQsXi7UkmhpgbzPjSMO934GygtSZhutorgp\n0ioSzk1rKMg03r87P4DmBoi7yOi6HHqVDMw6mMcN5J7i72Nh6axkHnh7C+9lHuamCS60No6XFyRd\nblzKDlj7P98wZjZEJBv9n6NvAr8gsyMV4rSGGtjxvpHsj2aDb5DxKzVtMUQMNTs60Y7btfQBtNbc\n8MI35JdWs/Zn0wny97FTdCZorIUd/zI+UIVbwLe3kfjT7oJ+TjJLSXim0n3GjLQtb0DdCYhIMcak\nRs2XhokJPLZ755RtBSeY88xG7p46mF9emWKHyJxAQZaR/Hf8C5rrjZ/OaYsh+SqwuPAXm3AdLc1G\n12PGi0ZXpJc3JM82unAGTZYuSBN5fNIH+Nl72azaeoTPfjKNuPBAuzymU6guhS2vQcbLUHEIggbA\n+Dtg/O0QFGl2dMIdVZfClteNLscTh6B35OmB2T4DzI5OIEkfgOMnjSmckxPCWXZbp/8L19NRqytl\njtH6HzRJWl2i+wqyjGNKdrxv/LocNMU6rXi2/Lp0Mh47kNtWvz7+3Dsjgb+u2cOmvBImJbjZyRW8\nLDB0pnFp7V99HXb+C/oNN45yHDVfDnwRXdPRONLYBUZjov8ws6MT3eTWLX0wpnBe9uR6An29+eiB\nKXhb3PxEJw01xsFe378Ix7YZh7iPvtk6kyLJ7OiEMyvPN9as3/K6sWhg+NDTM8b8+5gdneiEdO+0\nsXr7Ue55czO/v2YECyYOsutjOy2toSDDupjVv4050/HTjA/x0CtlzrQwtLTAvi+M98neT41jQ5Kv\nMt4n8VOli9CFSNJvQ2vN/GXfkne8irU/m05wLw/ri6wqNgZ+Tx0d2Sf69MBv735mRyfMUFMGW980\nWvblByCw3+mjwIOjzY5OnAdJ+u3sOFLB1c98zZ2T43l4tof2SzY3wd41bdZB8THWQZlwl3HCF2nV\nnamqGAo3w5HNcCTL6C5rqDE7KvtoqjXOMTvwQqNVnzJH1ntycTKQ286I6GDmp8by6qZ8br1gIIMj\nPHBw0+Jt/HRPvgpK9hqtvK1vGWMA/UcaszJG3gC+bjS91Vb1lVC41Zrks+DIFmM6LBhdHhHJxhLZ\n/n3NjdNefPyNM1JFjjQ7EtHDPKalD1BcWc+Mx9dxQXwoLy1Mc8hzuJyGatj2rjEtr2gH+AXD2Fsh\ndRGEJ5gdnWM0NRh1PZJlzE45kgXFewDrZ6HvIIgeB9HjIWocDBgtM6CE05PunbP4x/p9/Gn1bl67\ncwJTk+Q0bK06PIvRDKPrJ2mmMT3UFbW0QOlea+t9s9GSP7bdGNgGCAg3kntrkh8LgW42tVd4BEn6\nZ1Hf1MzlT27A1+LF6h9f5P5TOM9HZZFxhq9T5ysNjj199KUzJ0StoaKgTRfNZqPLpqHS2O/b20jq\nUWNPJ/ngWBnLEG5Bkv45rNl5jLtfz+J3c4Zz+6Q4hz6XS2tugj2fGAfpHNgAFl+jHzjtLohJNT9Z\n1pSdHmQ9NeBafdzY5+UDkSNOd9FEjzdOxeeqv1iE6IQk/XPQWnPr8u/IOXqSdT+bTt8AmbXQqeI9\nRr//1reNlnPkKKPrZ8Q88O2BU1M2VBvL9rZN8uX51p0KwpPO7IePHAHefo6PSwgnIUm/E7uPneTK\nv3/FbRfG8cgcOTetzeqrYNs7xhfA8RxjNsvYBZB6J4QNsc9zNDdC0c4zZ9IU7wLdYuwPjrV20Vj7\n4geMkSNGhceTpG+DX32wnRUZh1nz4EUk9JP1v7tEazi4yej62fWhMec74VKj6yfxMtu7UVpaoGx/\nmy6aLGOgtanO2N8r9MwWfPQ4OaBMiA5I0rdBaVU90x9fx7iBIbx654QeeU63dPKocT7UrH9C5VHo\nO9CY8jn2BxDY7jzFJwvPnElzZAvUVxj7fAKMVnu0NblHjYOQOPPHDoRwAZL0bbT8q/38/uNdvLIw\njRnJ0oLsluZG2P2x0fWT/xVY/GDEdUa3zxHrfPiqY0ZZL2/oP/x06z16vLHAl6wJJMR5kaRvo4am\nFmY+tQEUrHlwKj4yhdM+ju8ykn/2CmiogrCEM2fSRI4An15mRymE25Ck3wVf7Cpi0auZ/Gb2MO6c\nEt+jz+32GmqMA738g82ORAi3ZmvSt6lZq5SaqZTao5TKU0ot7WC/n1LqHev+75RScdbtE5RSW62X\nbKXUtV2tSE+4OLkfFyWG89TnuZRVN5gdjnvxDZCEL4QT6TTpK6UswLPALGAYcLNSqv0ylYuAcq11\nAvAk8Jh1+w4gVWs9BpgJ/EMp5XSdtkopfjN7GNUNzTz5Wa7Z4QghhMPY0tKfAORprfdrrRuAFcDc\ndmXmAq9ar68ELlFKKa11jda6ybrdn9YVrZxPYv8gFlwwkDe/O8ieY5VmhyOEEA5hS9KPBg63uV1g\n3dZhGWuSrwDCAJRSFyildgLbgR+2+RJwOg9emkSQvw//+1EOzjbWIYQQ9uDwqSpa6++01sOBNOCX\nSin/9mWUUkuUUplKqczi4mJHh3RWIYG+PHhpIl/nlfD5ruOmxSGEEI5iS9I/AsS2uR1j3dZhGWuf\nfTBQ2raA1noXUAWMaP8EWutlWutUrXVqRIS5yx0vmDiIhH69+cPHOTQ0tZgaixBC2JstST8DSFRK\nxSulfIGbgPR2ZdKB263X5wFfaq219T7eAEqpQUAykG+XyB3Ex+LFr69KIb+0hlc35ZsdjhBC2FWn\nSd/aB38/sAbYBbyrtd6plHpUKTXHWuwlIEwplQf8FDg1rXMKkK2U2gp8ANyrtS6xdyXsbfrQfswY\nGsHTX+ylpKre7HCEEMJu5OCss8g7XsXMpzZwQ2osf7pOziMqhHBudj04yxMl9OvNDy4cxDsZh8gp\nPGl2OEIIYReS9M/hwUuSCO7lw6Mf7ZQpnEIItyBJ/xyCA3z46WVJfLu/jDU7i8wORwghuk2Sfidu\nnjCQof2D+OMnu6hvajY7HCGE6BZJ+p3wtnjx8OxhHCqr4eWv880ORwghukWSvg2mJIZzaUp/nvly\nL8cr68wORwghzpskfRv96qoUGppbeHzNHrNDEUKI8yZJ30bx4YEsnBTHe1kF7DhSYXY4QghxXiTp\nd8EDlyQSGuDLox/KKpxCCNckSb8L+vj78NDlQ/k+v4xPth8zOxwhhOgySfpdND8tlpQBffjjJ7uo\na5QpnEII1yJJv4ssXoqHZ6dw5EQty7/ab3Y4QgjRJZL0z8OkIeFcMbw/z63bR9FJmcIphHAdkvTP\n06+uHEZTs+Yv/5EpnEII1yFJ/zwNDAvgzinxvL+5gOzDJ8wORwghbCJJvxvuvziB8N5+PConUhdC\nuAhJ+t3Q28+bn18xlKyD5aRnF5odjhBCdEqSfjddPz6G4VF9+PPq3dQ2yBROIYRzk6TfTRYvxW+v\nHs7RijqWbZApnEII5yZJ3w4mxIdy1cgBvLB+H0cras0ORwghzkqSvp0snZVMs9Y8tnq32aEIIcRZ\nSdK3k9jQAJZcNJh/by1k86Fys8MRQogOSdK3o3umD6FfkB+/+zCHlhaZwimEcD6S9O0o0M+bn89M\nJvvwCf699YjZ4QghxH+RpG9n142NZnRMMI/9Zzc1DU1mhyOEEGeQpG9nXl6K31w9jKKT9bywbp/Z\n4QghxBkk6TvA+EGhzBkdxT827KegvMbscIQQopUkfQdZOisZpeDPMoVTCOFEJOk7SFTfXtw9dQgf\nbTtKRn6Z2eEIIQQgSd+h7p42mMg+/jwqUziFEE5Ckr4DBfh6s3RWMtuPVPD+5gKzwxFCCEn6jjZ3\nTBRjB/blL2v2UFUvUziFEOaSpO9gShmrcBZX1vPc2jyzwxFCeDhJ+j1gTGxfrhsbzfKvD3C4TKZw\nCiHMI0m/h/x8ZjIWpfjjJ7vMDkUI4cEk6feQyGB/7pk+hNU7jvHt/lKzwxFCeChJ+j1oydTBRPft\nxaMf5tAsUziFECaQpN+D/H0sLJ2VTM7Rk7yXedjscIQQHsimpK+UmqmU2qOUylNKLe1gv59S6h3r\n/u+UUnHW7ZcppbKUUtutfy+2b/iuZ/aoAaTFhfD4p3uorGs0OxwhhIfpNOkrpSzAs8AsYBhws1Jq\nWLtii4ByrXUC8CTwmHV7CXC11nokcDvwur0Cd1VKKX4zezil1Q0886VM4RRC9CxbWvoTgDyt9X6t\ndQOwApjbrsxc4FXr9ZXAJUoppbXeorUutG7fCfRSSvnZI3BXNjImmHnjYnh54wHyS6rNDkcI4UFs\nSfrRQNsO6ALrtg7LaK2bgAogrF2Z64HNWuv68wvVvfzPFUPxtXjJFE4hRI/qkYFcpdRwjC6fu8+y\nf4lSKlMplVlcXNwTIZmuXx9/7p2RwKc5RWzKKzE7HCGEh7Al6R8BYtvcjrFu67CMUsobCAZKrbdj\ngA+A27TWHZ5KSmu9TGudqrVOjYiI6FoNXNiiKfHEhvbi0Y9yaGpuMTscIYQHsCXpZwCJSql4pZQv\ncBOQ3q5MOsZALcA84EuttVZK9QU+BpZqrTfaK2h34e9j4f/NSmH3sUpWZMgUTiGE43Wa9K199PcD\na4BdwLta651KqUeVUnOsxV4CwpRSecBPgVPTOu8HEoDfKKW2Wi/97F4LFzZzRCQXxIfyxGe5VNTK\nFE4hhGMprZ3ryNDU1FSdmZlpdhg9amdhBbP//9fcOTmeh2e3nw0rhBCdU0plaa1TOysnR+Q6geFR\nwcxPjeXVTfnsL64yOxwhhBuTpO8kHrp8KP4+Fv7wsUzhFEI4jiR9JxER5McDFyfwxe7jbMj1jGmr\nQoieJ0nfiSycHMegsAD+V6ZwCiEcRJK+E/HztvCrK1PYe7yKN787ZHY4Qgg3JEnfyVw2rD+ThoTx\n+Kd7yMgvMzscIYSbkaTvZJRSPHb9KMJ7+3Hri9/xflaB2SEJIdyIJH0nFBsawAf3TiI1LoSH3svm\nsf/spkXOtCWEsANJ+k6qb4Avr945gVsuGMjz6/bxwzeyqK5vMjssIYSLk6TvxHwsXvzhmhE8cvUw\nPt9VxLwXvuHIiVqzwxJCuDBJ+k5OKcXCyfG8vDCNgrIa5j6zkc2Hys0OSwjhoiTpu4jpQ/vxr3sn\nEeBr4aZl37Jqa/vVrYUQonOS9F1IYv8g/n3fZMbE9uXHK7byxKd7ZIBXCNElkvRdTGigL28suoAb\nU2N4+ss87n97M7UNzWaHJYRwEZL0XZCvtxePXT+KX1+Vwuodx7jxH99wrKLO7LCEEC5Akr6LUkqx\n+KLBLL8tlf3FVcx55mu2FZwwOywhhJOTpO/iLknpz/v3TsLH4sWN//iGj7cdNTskIYQTk6TvBpIj\n+7Dq/skMjwrmvrc28/QXe3G2M6IJIZyDJH03Ed7bjzcXX8B1Y6N54rNcfrxiK3WNMsArhDiTt9kB\nCPvx97HwtxtHk9C/N3/5zx4OldWw7Lbx9AvyNzs0IYSTkJa+m1FKce/0BF5YMJ49xyq55pmN7Cys\nMDssIYSTkKTvpmaOiOS9H16IBuY9/w1rdh4zOyQhhBOQpO/GRkQHs+q+ySRFBvHDN7J4ft0+GeAV\nwsNJ0ndz/fr4886SicweFcVj/9nNQ+9lU98kA7xCeCoZyPUA/j4Wnr5pDAkRvXny81wOldbwwg/G\nE97bz+zQhBA9TFr6HkIpxY8vTeSZW8ay/UgFc5/ZyO5jJ80OSwjRwyTpe5jZo6J49+4LaWxu4frn\nNvHFriKzQxJC9CBJ+h5odGxf0u+fQnxEIItfy2T5V/tlgFcIDyFJ30NFBvvz7t0XMnN4JL//eBdL\n399OQ1OL2WEJIRxMkr4HC/D15tlbxvHAxQm8k3mYH7z0HeXVDWaHJYRwIEn6Hs7LS/HQ5UP5+01j\n2HL4BNc8t5G845VmhyWEcBBJ+gKAuWOiWbFkItX1zVz77CbW5xabHZIQwgEk6YtW4waGsOr+ycSE\nBnDHK9/zz40HZIBXCDcjSV+cIbpvL1b+8EIuTu7PIx/m8PCqHTQ2ywCvEO5Ckr74L4F+3vzjB+O5\ne9pg3vj2EAtf+Z6KmkazwxJC2IEkfdEhi5fil7NS+Ou8UXx/oIxrn9vI/uIqs8MSQnSTJH1xTjek\nxvLWXRM5UdvINc9uZGNeidkhCSG6QZK+6FRaXCir7ptMZLA/t738PW9+d9DskIQQ50mSvrBJbGgA\n798ziamJ4fzqgx08kr6TJhngFcLl2JT0lVIzlVJ7lFJ5SqmlHez3U0q9Y93/nVIqzro9TCm1VilV\npZR6xr6hi54W5O/D8tvTWDQlnn9uyufOVzM5WScDvEK4kk6TvlLKAjwLzAKGATcrpYa1K7YIKNda\nJwBPAo9Zt9cBDwM/s1vEwlQWL8XDs4fxp+tGsimvhOue28TB0mqzwxJC2MiWlv4EIE9rvV9r3QCs\nAOa2KzMXeNV6fSVwiVJKaa2rtdZfYyR/4UZunjCQ1xddQElVPdc8u5Hv9peaHZIQwga2JP1o4HCb\n2wXWbR2W0Vo3ARVAmD0CFM7rwiFh/PveyYQE+rLgpe94N+Nw53cSQpjKKQZylVJLlFKZSqnM4mJZ\n88WVxIUH8sG9k5k4OIyfv7+NP36yi+YWWbpBCGdlS9I/AsS2uR1j3dZhGaWUNxAM2Px7X2u9TGud\nqrVOjYiIsPVuwkkE9/LhlYVp3HbhIJZt2M+S1zKpqm8yOywhRAdsSfoZQKJSKl4p5QvcBKS3K5MO\n3G69Pg/4UstKXR7F2+LFo3NH8Ojc4azLLeb65zZxuKzG7LCEEO10mvStffT3A2uAXcC7WuudSqlH\nlVJzrMVeAsKUUnnAT4HWaZ1KqXzgCWChUqqgg5k/wo3cdmEc/7wjjcKKWq55diOZ+WVmhySEaEM5\nW4M8NTVVZ2Zmmh2G6Ka841UsfjWDwhN1PHR5EteNiyEiyM/ssIRwW0qpLK11aqflJOkLRzlR08AD\nb2/hq70leCmYnBDO1aOjmDkikj7+PmaHJ4RbkaQvnEZuUSXpWwtZlX2Ew2W1+Hp7MWNoBHNGR3NJ\nSj/8fSxmhyiEy5OkL5yO1pqth0+Qnl3IR9uOUlxZT6CvhSuGR3L1mCimJITjY3GKWcRCuBxJ+sKp\nNbdovt1fSvrWQlbvOMrJuiZCA325cmQkc0ZHkzooBC8vZXaYQrgMSfrCZdQ3NbN+TzHp2YV8vquI\nusYWooL9mT06ijmjoxge1Qel5AtAiHORpC9cUnV9E5/lFJGeXciG3GKaWjSDIwKZY/0CGBzR2+wQ\nhXBKkvSFyyuvbmD1jmOs2nqE7/PL0BpGRgczZ3QUs0cPYEBwL7NDFMJpSNIXbuVYRR0fbSskPbuQ\nbQUVKAUT4kKZMyaKK0cMICTQ1+wQhTCVJH3htg6UVJO+tZD07CPsK67G20sxNSmCOaOjuGxYfwL9\nvM0OUYgeJ0lfuD2tNTlHT5K+tZAPswsprKjD38eLS1L6M3d0FNOGRuDnLccACM8gSV94lJYWTdah\nctK3FvLx9qOUVTfQx9+bmSMimTsmmomDw7DIFFDhxtwq6Tc2NlJQUEBdnZyAy178/f2JiYnBx8f9\nlkNobG5hY14J6dmFrNlxjOqGZiKC/Lhq5ADmjIlibGxfmQIq3I5bJf0DBw4QFBREWFiYfFjtQGtN\naWkplZWVxMfHmx2OQ9U1NvPl7uOkby3kyz3HaWhqYWBoAFePHsCc0dEMjQwyO0Qh7MLWpO8SI151\ndXXExcVJwrcTpRRhYWF4wlnK/H0sXDlyAFeOHMDJukbW7DhGenYhz6/bx7Nr9zG0fxBzxhjHAMSG\nBpgdrhAO5xJJH5CEb2ee+P/s4+/DDamx3JAaS3FlPZ9sP0p6diF/XbOHv67Zw9iBfZkzOoqrRg2g\nX5C/2eEK4RCyupWNTpw4wXPPPdfl+1155ZWcOHHCARGJ7ogI8uP2SXG8f88kvvr5DH4xM5nahmZ+\n92EOE//4BQuWGyd6r6htNDtUIezKJfr0d+3aRUpKikkRGfLz85k9ezY7duw4Y3tTUxPe3i7zg+kM\nzvB/dTZ7iypJzy5k1dZCDpXV4GvxYvrQCOaMieKS5P708pUpoMI5uVWfvjNYunQp+/btY8yYMfj4\n+ODv709ISAi7d+8mNzeXa67Lmi/bAAAOVElEQVS5hsOHD1NXV8ePf/xjlixZAkBcXByZmZlUVVUx\na9YspkyZwqZNm4iOjmbVqlX06iVLCTiTxP5BPHT5UH56WRLZBRWkby3ko22FfJpTRKCvhWFRfRgY\nGkhcWACDwq1/QwMJDnC/WVDCPUlL30ZtW/rr1q3jqquuYseOHa2zX8rKyggNDaW2tpa0tDTWr19P\nWFjYGUk/ISGBzMxMxowZw4033sicOXNYsGCBaXVyhv+rK2hu0Xy3v5RPdhwlt6iKg6XVFJ2sP6NM\n3wAfBoUFMig0wPgiCAskLjyAgaGBhPf29cgxFNGz3Lal/7sPd5JTeNKujzksqg+/vXp4l+4zYcKE\nM6Y7Pv3003zwwQcAHD58mL179xIWFnbGfeLj4xkzZgwA48ePJz8/v3uBix5h8VJMSghnUkJ467ba\nhmYOldWQX1rNwdJqDpbWcLC0hs2HyvloWyEtbdpSgb4W4wvh1JeB9e+gsAAi+/jLeQNEj3K5pO8s\nAgMDW6+vW7eOzz//nG+++YaAgACmT5/e4YFkfn6nTwxusViora3tkViF/fXytTA0MqjDef4NTS0U\nlNdYvwiqybf+3VNUyee7imhsPv2N4OvtxaDQgA6/EKL79sJbziQm7Mzlkn5XW+T2EhQURGVlZYf7\nKioqCAkJISAggN27d/Ptt9/2cHTCmfh6ezE4oneHa/83t2gKT9QaXwhlxi+E/JJqDpXV8HVeCXWN\nLa1lvb0UMSG9GNjmy8D4G0BMSICcW1icF5dL+mYJCwtj8uTJjBgxgl69etG/f//WfTNnzuSFF14g\nJSWFoUOHMnHiRBMjFc7M4qWIDQ0gNjSAKYSfsU9rzfHKeuOLwNptlF9aw6HSGrYcKqeyrqm1rFIQ\nFdyLgaEBxIWf/kIYGGr8SpCVRsXZyECuB5P/q+vQWlNe09g6fpBfWs2h1i+HGkqrG84oHxHkZ+02\nOj3TyBhklplG7sptB3KF8ERKKUIDfQkN9GXswJD/2l9Z19g6mNz2C2FjXgnvbz5zfKlvgA/x4YFc\nODiMqUkRjB8Ugo+MHXgMSfpCuIEgfx9GRAczIjr4v/bVNVpnGlnHDvJLq9lzrJJlG/bz3Lp99Pbz\n5sIhYUxLimBaUoSsQeTmJOkL4eb8fSwk9Q8iqf+ZM40q6xrZtK+U9bnFbMgt5rOcIgAGhwcyNSmC\nqUnhTBwcRoCvpAl3Iq+mEB4qyN+HK4ZHcsXwSLTWHCipbv0CWJFxiH9uysfX4kVafAhTEyOYNjSC\nof2D5EAzFydJXwiBUqp1mukdk+Opa2wmM7+c9bnH2ZBbwp9W7+ZPq3fTv48fUxMjmJoUwUWJ4fQN\nkBPSuxpJ+kKI/+LvY2FKYjhTEsP51VVwtKKWr3JLWL+3mE9zingvqwAvBaNi+jLVOhYwOiZYDiZz\nAfIKOUjv3saBOYWFhcybN6/DMtOnT6ez8wE/9dRT1NTUtN6WpZqFGQYE9+LGtFievWUcmx++jH/d\nO4kHLk5EKXjmy71c//wmxv3vZ9z35mbeyTjE0Qo52txZSUvfwaKioli5cuV53/+pp55iwYIFBAQY\nMyo++eQTe4UmxHmxeCnGDQxh3MAQfnJZEidqGvg6r4QNucWszy3m4+1HAUjq37t1LCAtLlSOIHYS\nkvRttHTpUmJjY7nvvvsAeOSRR/D29mbt2rWUl5fT2NjI73//e+bOnXvG/dquzllbW8sdd9xBdnY2\nycnJZ6y9c88995CRkUFtbS3z5s3jd7/7HU8//TSFhYXMmDGD8PBw1q5d27pqZ3h4OE888QQvv/wy\nAIsXL+bBBx8kPz9flnAWPapvgC+zR0Uxe1QUWmtyi6paxwJe++Ygy78+gL+PFxMHh7WOBwyJCJQB\nYbNorZ3qMn78eN1eTk7Of23raZs3b9ZTp05tvZ2SkqIPHTqkKyoqtNZaFxcX6yFDhuiWlhattdaB\ngYFaa60PHDighw8frrXW+m9/+5u+4447tNZaZ2dna4vFojMyMrTWWpeWlmqttW5qatLTpk3T2dnZ\nWmutBw0apIuLi1uf99TtzMxMPWLECF1VVaUrKyv1sGHD9ObNm/WBAwe0xWLRW7Zs0VprfcMNN+jX\nX3+9wzo5w/9VuLfq+kb95a4i/dtVO/SMv67Vg37xkR70i4/0pD99oZe+v02v3n5UV9Q2mB2mWwAy\ntQ051vVa+quXwrHt9n3MyJEw68/nLDJ27FiOHz9OYWEhxcXFhISEEBkZyU9+8hM2bNiAl5cXR44c\noaioiMjIyA4fY8OGDfzoRz8CYNSoUYwaNap137vvvsuyZctoamri6NGj5OTknLG/va+//pprr722\ndbXP6667jq+++oo5c+bIEs7CaQT4ejMjuR8zkvsBcLispnVa6IfZhbz9/SEsXorxA0OYmhTO1KQI\nRkQFy3LTDuR6Sd9EN9xwAytXruTYsWPMnz+fN998k+LiYrKysvDx8SEuLq7DJZU7c+DAAR5//HEy\nMjIICQlh4cKF5/U4p8gSzsJZxYYGsGDiIBZMHERjcwubD5azYa8xFvD4p7k8/mkuYYG+TEkMZ1pS\nBBclRhAR5Nf5AwubuV7S76RF7kjz58/nrrvuoqSkhPXr1/Puu+/Sr18/fHx8WLt2LQcPHjzn/adO\nncpbb73FxRdfzI4dO9i2bRsAJ0+eJDAwkODgYIqKili9ejXTp08HTi/pHB5+5oqMF110EQsXLmTp\n0qVorfnggw94/fXXHVJvIRzBx+LFBYPDuGBwGP9zRTIlVfV8tbeYDbnGoPCqrYUADI/qYxwhnGis\nE+TrLZMOu8P1kr6Jhg8fTmVlJdHR0QwYMIBbb72Vq6++mpEjR5KamkpycvI573/PPfdwxx13kJKS\nQkpKCuPHjwdg9OjRjB07luTkZGJjY5k8eXLrfZYsWcLMmTOJiopi7dq1rdvHjRvHwoULmTBhAmAM\n5I4dO1a6coTLCu/tx7VjY7h2bAwtLZqcoydZb50R9OKG/Ty/bh+BvhYuHBLOtKERTEuMYGCYrBPU\nVbK0sgeT/6twFafWCTo1LbSg3OiyjAsLYHRsXyxuMhNozMC+3HZh3Hnd165LKyulZgJ/ByzAcq31\nn9vt9wNeA8YDpcB8rXW+dd8vgUVAM/AjrfWaLtRDCCE6XCfo1BfA5kPlZodnN316Of5cB50mfaWU\nBXgWuAwoADKUUula65w2xRYB5VrrBKXUTcBjwHyl1DDgJmA4EAV8rpRK0lo327siQgjP0HadoIWT\n480Ox+XYMiIyAcjTWu/XWjcAK4C57crMBV61Xl8JXKKMIy/mAiu01vVa6wNAnvXxhBBCmMCWpB8N\nHG5zu8C6rcMyWusmoAIIs/G+NnG2sQdXJ/9PITyTU8x9UkotUUplKqUyi4uL/2u/v78/paWlkqjs\nRGtNaWkp/v7+ZocihOhhtgzkHgFi29yOsW7rqEyBUsobCMYY0LXlvmitlwHLwJi9035/TEwMBQUF\ndPSFIM6Pv78/MTExZochhOhhtiT9DCBRKRWPkbBvAm5pVyYduB34BpgHfKm11kqpdOAtpdQTGAO5\nicD3XQ3Sx8eH+HgZsBFCiO7qNOlrrZuUUvcDazCmbL6std6plHoUY4GfdOAl4HWlVB5QhvHFgLXc\nu0AO0ATcJzN3hBDCPC5xcJYQQohzs/XgLKcYyBVCCNEznK6lr5SqAPa22RSMMQW0o9unrrfdFg6U\nnOfTt3+urpTpaLstsZ/tenfqca44bdnvTHXpzmvS0b6u3Hbl91f72+3r4uj317nKuPP7q6NtPVWX\nRK11cKelbFl0vycvwDJbb5+63m6bTScSsOW5u1Kmo+22xH6OOp13PWypy7n2O1NduvOadPX95E7v\nr87q4uj3lz3r4krvL2evi9baKbt3PuzC7Q/PUsZez92VMh1ttyX2c13vjs4e51z7naku3XlNOtrn\nKe+v9rdduS6u9P7qaJsz1cX5une6SymVqW0YzHB27lIPkLo4I3epB0hdusoZW/rdtczsAOzEXeoB\nUhdn5C71AKlLl7hdS18IIcTZuWNLXwghxFlI0hdCCA8iSV8IITyIxyR9pVSKUuoFpdRKpdQ9ZsfT\nHUqpa5RSLyql3lFKXW52PN2hlBqslHpJKbXS7Fi6SikVqJR61fpa3Gp2PN3hyq9De272+bB/3urO\nARo9dQFeBo4DO9ptnwnswTgj11IbH8sLeMNN6hICvOQmdVlp9vusq3UCfgBcbb3+jtmx2+P1cZbX\nwU51MfXzYee62C1vmf4PsPGfNBUY1/afhLHi5z5gMOALZAPDgJHAR+0u/az3mQOsBm5x9bpY7/c3\nYJyb1MUpkk0X6/RLYIy1zFtmx96dujjb62Cnupj6+bBXXeydt2xZT990WusNSqm4dptbz90LoJRa\nAczVWv8JmH2Wx0kH0pVSHwNvOS7is7NHXaznH/4zsFprvdmxEZ+dvV4XZ9KVOmGc/jMG2IoTdpV2\nsS45PRtd13SlLkqpXTjB5+Nsuvq62DtvOd0btQu6dP5dpdR0pdTTSql/AJ84Orgu6uq5hB8ALgXm\nKaV+6MjAzkNXX5cwpdQLwFil1C8dHdx5Olud/gVcr5R6Hvstb+BoHdbFRV6H9s72ujjz5+Nszva6\n2D1vuURL3x601uuAdSaHYRda66eBp82Owx601qWAq3wwz6C1rgbuMDsOe3Dl16E9N/t8rMPOecuV\nW/o2nX/XRUhdnJs71Unq4px6rC6unPRbz92rlPLFOEVjuskxnS+pi3NzpzpJXZxTz9XF7JFsG0e7\n3waOAo0YfV2LrNuvBHIxRr1/ZXacUhfXrYs71knq4pwXs+siC64JIYQHceXuHSGEEF0kSV8IITyI\nJH0hhPAgkvSFEMKDSNIXQggPIklfCCE8iCR9IYTwIJL0hRDCg0jSF0IID/J/cG69LeQsjvgAAAAA\nSUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"zeSk_U4mv6uE","colab_type":"text"},"source":["Summary:\n","\n","Too much regularization (small C) doesn't work well - due to underfitting - and too little regularization (large C) doesn't work well either - due to overfitting. "]},{"cell_type":"markdown","metadata":{"id":"n0qnfFuWhl7T","colab_type":"text"},"source":["#### Logistic regression and probabilities\n","\n","In this exercise we'll perform feature selection on the movie review sentiment data set using L1 regularization. The features and targets are already loaded for you in X_train and y_train.\n","\n","We'll search for the best value of C using scikit-learn's GridSearchCV(), which was covered in the prerequisite course.\n","\n","\n","\n","*   Instantiate a logistic regression object that uses L1 regularization.\n","*   Find the value of C that minimizes cross-validation error.\n","*   Print out the number of selected features for this value of C.\n","\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"BKBUk4Arho-L","colab_type":"code","colab":{}},"source":["# Specify L1 regularization\n","lr = LogisticRegression(penalty='l1')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Md7B24hOx2DZ","colab_type":"code","colab":{}},"source":["# Instantiate the GridSearchCV object and run the search\n","searcher = GridSearchCV(lr, {'C':[0.001, 0.01, 0.1, 1, 10]})\n","searcher.fit(X_train, y_train)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"JVp30Vs9x3hx","colab_type":"code","colab":{}},"source":["# Report the best parameters\n","print(\"Best CV params\", searcher.best_params_)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"fuNWsOnyx5Ap","colab_type":"code","colab":{}},"source":["# Find the number of nonzero coefficients (selected features)\n","best_lr = searcher.best_estimator_\n","coefs = best_lr.coef_\n","print(\"Total number of features:\", coefs.size)\n","print(\"Number of selected features:\", np.count_nonzero(coefs))"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kuuOL14HyE3p","colab_type":"text"},"source":["**Identifying the most positive and negative words**\n","\n","In this exercise we'll try to interpret the coefficients of a logistic regression fit on the movie review sentiment dataset. The model object is already instantiated and fit for you in the variable lr.\n","\n","In addition, the words corresponding to the different features are loaded into the variable vocab. For example, since vocab[100] is \"think\", that means feature 100 corresponds to the number of times the word \"think\" appeared in that movie review."]},{"cell_type":"code","metadata":{"id":"0vFlTjbZynKp","colab_type":"code","cellView":"form","colab":{}},"source":["#@title\n","vocab = np.array(['the', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this',\n","       'that', 'was', 'as', 'for', 'with', 'movie', 'but', 'film', 'on',\n","       'not', 'you', 'he', 'are', 'his', 'have', 'be', 'one', '!', 'all',\n","       'at', 'by', 'an', 'who', 'they', 'from', 'so', 'like', 'there',\n","       'her', 'or', 'just', 'about', 'if', 'has', 'out', 'what', '?',\n","       'some', 'good', 'more', 'when', 'she', 'very', 'even', 'my', 'no',\n","       'up', 'time', 'would', 'which', 'only', 'story', 'really', 'their',\n","       'see', 'had', 'can', 'were', 'me', 'we', 'than', 'well', 'much',\n","       'been', 'get', 'people', 'will', 'bad', 'other', 'also', 'into',\n","       'do', 'because', 'great', 'first', 'how', 'him', 'most', \"don't\",\n","       'its', 'made', 'then', 'them', 'way', 'make', 'could', 'too',\n","       'any', 'after', 'movies', 'think', 'characters', 'character',\n","       'watch', 'films', 'two', 'many', 'seen', 'being', 'acting',\n","       'never', 'plot', 'little', 'where', 'love', 'best', 'life', 'did',\n","       'show', 'know', 'does', 'ever', 'here', 'better', 'man', 'your',\n","       'still', 'end', 'over', 'off', 'these', 'say', 'scene', 'why',\n","       'while', 'scenes', 'such', 'go', 'something', 'should', 'through',\n","       'back', \"i'm\", 'those', 'watching', 'real', 'though', 'now',\n","       \"doesn't\", 'thing', 'years', 'actors', 'director', 'another',\n","       \"didn't\", 'before', 'nothing', 'new', 'funny', 'actually', 'work',\n","       'makes', 'find', 'look', 'old', 'few', 'going', 'same', 'again',\n","       'lot', 'part', 'every', 'cast', 'us', 'world', 'quite', 'want',\n","       'things', 'pretty', 'young', 'seems', 'around', 'got', 'down',\n","       'however', \"can't\", 'fact', 'horror', 'take', 'enough', 'both',\n","       'may', 'give', 'original', 'between', 'big', 'own', \"i've\",\n","       'thought', 'series', 'without', 'right', 'times', 'long', 'always',\n","       'gets', 'action', 'comedy', \"isn't\", 'family', 'come', 'point',\n","       'role', 'saw', 'interesting', 'almost', 'least', 'whole', 'must',\n","       'bit', 'music', 'script', 'done', 'guy', 'anything', 'minutes',\n","       'last', 'since', 'might', 'performance', 'far', 'feel', 'probably',\n","       'am', 'woman', 'kind', 'girl', 'away', 'yet', 'rather', 'worst',\n","       'sure', 'fun', 'anyone', 'making', 'each', 'played', 'tv', 'found',\n","       'having', 'day', 'although', 'especially', 'our', 'course',\n","       'believe', 'comes', 'trying', 'goes', 'hard', 'looks', 'different',\n","       'place', 'book', 'actor', 'put', 'maybe', 'money', 'someone',\n","       'ending', 'reason', 'let', 'everything', \"wasn't\", 'sense', 'once',\n","       'shows', 'screen', 'dvd', 'true', 'set', 'worth', 'job', 'main',\n","       'looking', 'watched', 'everyone', 'together', 'three', 'plays',\n","       'john', 'later', 'said', 'play', 'instead', 'audience', 'seem',\n","       'beautiful', 'takes', 'effects', 'himself', 'version', 'during',\n","       'left', 'night', 'house', 'seeing', 'wife', 'special', 'excellent',\n","       'father', 'american', 'idea', 'else', 'nice', 'shot', 'simply',\n","       'year', 'read', \"you're\", 'black', 'high', 'less', 'war', 'star',\n","       'completely', 'help', 'fan', 'poor', 'death', 'second', 'men',\n","       'hollywood', 'either', 'mind', 'used', 'given', 'home', 'kids',\n","       'try', 'performances', 'women', 'enjoy', 'classic', 'boring',\n","       'short', 'wrong', 'need', 'rest', 'use', 'friends', 'until',\n","       'along', 'dead', 'truly', 'half', 'production', 'line', 'tell',\n","       'couple', 'remember', 'next', 'start', 'stupid', 'perhaps', 'came',\n","       'recommend', 'moments', 'awful', 'wonderful', 'episode', 'mean',\n","       'understand', 'terrible', 'full', 'camera', 'getting', 'stars',\n","       'playing', 'video', 'keep', 'sex', 'doing', 'others', 'early',\n","       'often', 'small', 'definitely', 'gives', 'person', 'school',\n","       'face', 'perfect', 'name', 'itself', 'become', 'human', 'lines',\n","       'yes', 'finally', 'dialogue', 'lost', 'felt', 'case', 'piece',\n","       'top', 'liked', 'supposed', 'children', \"couldn't\", 'title',\n","       'absolutely', 'head', 'budget', 'mother', 'live', 'written', 'boy',\n","       'picture', 'against', 'cinema', 'went', 'worse', 'certainly',\n","       'entire', 'sort', 'style', 'waste', 'problem', 'mr', 'hope',\n","       'entertaining', 'overall', 'friend', 'killer', 'evil', 'several',\n","       'loved', 'fans', 'oh', 'beginning', 'white', 'lives', 'care',\n","       'becomes', 'direction', 'example', 'already', 'based', 'drama',\n","       'despite', 'seemed', 'dark', 'throughout', 'unfortunately',\n","       'wanted', \"i'd\", 'final', 'history', 'amazing', 'turn', 'fine',\n","       'laugh', 'michael', 'son', 'humor', 'guess', 'totally', 'lead',\n","       'sound', 'guys', 'writing', \"you'll\", 'wants', 'low', 'works',\n","       'tries', 'called', 'under', 'past', 'viewer', 'quality', 'child',\n","       'days', 'behind', 'game', 'turns', \"they're\", 'enjoyed', 'today',\n","       'able', 'act', 'town', 'favorite', 'kill', 'flick', 'starts',\n","       'gave', 'actress', 'sometimes', 'eyes', 'etc', 'side', 'horrible',\n","       'girls', 'genre', 'soon', \"won't\", 'car', 'brilliant', 'parts',\n","       'art', 'heart', 'themselves', 'expect', 'kid', 'stuff', 'stories',\n","       'thinking', 'city', 'obviously', 'directed', 'late', 'myself',\n","       'blood', 'decent', 'feeling', 'run', 's', 'highly', 'god',\n","       'except', 'close', 'fight', 'hand', 'anyway', 'roles', 'daughter',\n","       'killed', 'moment', 'says', 'heard', 'leave', 'matter', 'took',\n","       'cannot', 'police', 'happens', 'brother', 'hour', 'violence',\n","       'happened', 'strong', 'particularly', 'james', 'extremely',\n","       'involved', 'chance', 'writer', 'obvious', 'experience',\n","       \"wouldn't\", 'lack', 'including', 'told', 'alone', 'attempt',\n","       'murder', 'living', 'happen', 'please', 'age', 'wonder',\n","       'complete', 'ago', 'voice', 'group', 'score', 'david', 'coming',\n","       'interest', 'save', 'none', 'ok', 'crap', 'type', 'looked',\n","       'simple', 'slow', 'possible', 'number', 'seriously', 'hell',\n","       'gore', 'exactly', 'shown', 'king', 'hero', 'annoying', 'song',\n","       'husband', 'sad', 'whose', 'career', 'yourself', 'cinematography',\n","       'taken', 'musical', 'ends', 'usually', 'serious', \"i'll\", 'stop',\n","       'hours', 'scary', 'released', 'across', 'hilarious', 'running',\n","       'reality', 'relationship', 'usual', 'ridiculous', 'known', 'hit',\n","       'opening', 'somewhat', 'started', 'opinion', 'jokes', 'novel',\n","       'cool', 'change', 'robert', 'wish', 'ones', 'body', 'finds',\n","       'order', 'saying', 'english', 'huge', 'cut', 'shots', 'episodes',\n","       'mostly', 'taking', 'female', 'talking', 'strange', 'major',\n","       'view', 'power', 'happy', 'documentary', 'apparently', 'rating',\n","       'disappointed', 'level', 'talent', 'call', 'country', 'jack',\n","       'due', 'events', 'room', 'important', 'songs', 'basically',\n","       'clearly', 'knows', 'knew', 'supporting', 'attention',\n","       'television', 'future', 'turned', 'paul', 'easily', 'problems',\n","       \"aren't\", 'silly', 'british', 'word', 'tells', 'earth', 'words',\n","       'local', 'single', 'light', 'four', 'cheap', 'sequence', 'bring',\n","       'entertainment', 'thriller', 'beyond', 'george', 'miss', 'modern',\n","       'whether', 'b', 'predictable', 'falls', 'five', 'sets', 'similar',\n","       'review', 'richard', 'needs', 'upon', 'lady', 'enjoyable',\n","       'appears', 'romantic', 'comic', 'giving', 'mystery', 'talk',\n","       'rock', 'message', 'animation', 'within', 'theater', 'sequel',\n","       'bunch', 'mention', 'herself', 'feels', 'nearly', 'points',\n","       'theme', 'lee', 'above', 'dull', 'add', 'york', 'ways', 'moving',\n","       'storyline', 'ten', 'surprised', \"haven't\", 'lots', 'team',\n","       'begins', 'middle', 'using', 'fantastic', 'actual', 'sister',\n","       'effort', 'viewers', 'named', 'among', 'elements', 'easy', 'stay',\n","       'tom', 'comments', 'avoid', 'showing', 'typical', 'release',\n","       'clear', 'hate', 'tried', 'peter', 'sorry', 'french', 'dialog',\n","       'editing', 'tale', 'certain', 'season', 'soundtrack', 'buy',\n","       'fall', 'general', 'near', 'means', 'famous', 'check', 'parents',\n","       'red', 'somehow', 'material', 'oscar', 'period', 'form',\n","       'straight', 'weak', 'working', 'doubt', 'class', 'leads', 'filmed',\n","       'gone', 'kept', 'greatest', 'figure', 'viewing', 'feature',\n","       'disney', 'eye', 'realistic', 'brought', 'imagine', 'hear',\n","       'atmosphere', 'fast', 'particular', 'suspense', 'lame', 'move',\n","       'whatever', 'sequences', 'america', 'follow', 'indeed', 'die',\n","       'crime', 'learn', 'eventually', 'reviews', 'wait', 'forget',\n","       'deal', 'zombie', 'dance', 'space', 'okay', 'premise', 'surprise',\n","       'believable', 'nature', 'possibly', \"you've\", 'third', 'decided',\n","       'subject', 'expected', 'de', 'japanese', 'dr', 'became', 'truth',\n","       'imdb', 'sexual', 'average', 'stand', 'difficult', 'screenplay',\n","       'romance', 't', 'sit', 'poorly', 'rent', 'joe', 'nor', 'leaves',\n","       'question', 'stage', 'note', 'begin', 'killing', 'needed',\n","       'reading', 'unless', 'baby', 'superb', 'directors', 'society',\n","       'otherwise', 'shame', 'meet', 'street', 'situation', 'meets',\n","       'memorable', 'dog', 'credits', 'earlier', 'forced', 'weird',\n","       'minute', 'older', 'laughs', 'realize', 'emotional', 'jane',\n","       'beauty', 'writers', 'comment', 'footage', 'write', 'ask', 'badly',\n","       'interested', 'dramatic', 'sounds', 'whom', 'hot', 'keeps',\n","       'features', 'directing', 'mess', 'development', 'crazy', 'quickly',\n","       'male', 'mark', 'towards', 'creepy', 'free', 'monster',\n","       'perfectly', 'result', 'total', 'plus', 'previous', 'brings',\n","       'unique', 'plenty', 'worked', 'cheesy', 'effect', 'personal',\n","       'incredibly', 'hands', 'bill', 'fantasy', 'return', 'dream',\n","       'apart', 'deep', 'setting', 'admit', 'open', 'appear',\n","       'background', 'christmas', 'leading', 'doctor', 'casting',\n","       'hardly', 'meant', 'ben', 'potential', 'boys', 'powerful',\n","       'business', 'masterpiece', 'fails', 'battle', 'joke', 'create',\n","       'various', 'forward', 'fire', 'inside', 'outside', 'portrayed',\n","       'girlfriend', 'ideas', 'twist', 'william', 'missing', 'nudity',\n","       'reasons', 'villain', 'dumb', 'political', 'deserves', 'match',\n","       'secret', 'expecting', 'air', 'fairly', 'present', 'gay',\n","       'fighting', 'unlike', 'married', 'manages', 'break', 'scott',\n","       'success', 'attempts', 'western', 'spoilers', 'acted', 'remake',\n","       'pay', 'box', 'recently', 'rich', 'front', 'cute', 'further',\n","       'cop', 'sadly', 'era', 'copy', 'agree', 'talented', 'telling',\n","       'sci-fi', 'filmmakers', 'following', 'public', 'crew',\n","       'incredible', 'missed', 'wasted', 'pure', 'plain', 'brothers',\n","       'odd', 'caught', 'flat', 'social', 'mentioned', 'pace', 'ended',\n","       'decides', 'considering', 'waiting', 'members', 'mary', 'large',\n","       'list', 'sweet', 'revenge', 'popular', 'uses', 'hold', 'slightly',\n","       'office', 'compared', 'neither', 'wrote', 'escape', 'sees',\n","       'suddenly', 'e', 'party', 'spirit', 'tension', 'convincing',\n","       'created', 'fear', 'spent', 'rate', 'cause', 'entirely', 'island',\n","       'cartoon', 'intelligent', 'credit', 'clever', \"we're\", 'choice',\n","       'water', 'kills', 'bored', 'familiar', 'moves', 'tony', 'laughing',\n","       'gun', 'visual', 'successful', 'ultimately', 'basic', 'band',\n","       'trouble', 'la', 'cat', 'zombies', 'concept', 'value', 'biggest',\n","       'positive', 'singing', 'consider', 'exciting', 'dancing',\n","       'company', 'died', 'recent', 'state', 'language', 'effective',\n","       'science', 'cover', 'spend', 'portrayal', 'violent', 'cold',\n","       'appreciate', 'produced', 'speak', 'pointless', 'former', 'studio',\n","       'amusing', 'adult', 'common', 'filled', 'planet', 'younger',\n","       'books', 'store', 'walk', 'follows', 'focus', 'solid', 'bizarre',\n","       'italian', 'impressive', 'animated', 'impossible', 'german',\n","       'amount', 'tone', 'century', 'adventure', 'conclusion',\n","       'producers', \"weren't\", 'showed', 'recommended', 'fit', 'van',\n","       'depth', 'runs', 'situations', 'chemistry', 'prison', 'project',\n","       'jim', 'respect', 'awesome', 'control', 'accent', 'hair',\n","       'considered', 'win', 'won', 'smith', 'force', 'decide', 'college',\n","       'trip', 'disturbing', 'somewhere', 'mad', 'failed', 'dad',\n","       'changed', 'leaving', 'questions', 'longer', 'steve', 'audiences',\n","       'barely', 'aside', 'honest', 'slasher', 'sick', 'ghost',\n","       'shooting', 'trash', 'images', 'thanks', 'charming', 'starring',\n","       'generally', 'pathetic', 'literally', 'west', 'values', 'culture',\n","       'likes', 'fake', 'surprisingly', 'touch', 'magic', 'involving',\n","       'yeah', 'alive', 'immediately', 'stewart', 'harry', 'south',\n","       'frank', 'garbage', 'natural', 'utterly', 'camp', 'bought',\n","       'honestly', 'adaptation', 'sam', 'london', 'aspect', 'pictures',\n","       'ability', 'detective', 'nobody', 'glad', 'explain', 'fair',\n","       'computer', 'appearance', 'genius', 'sitting', 'cult', 'attack',\n","       'master', 'meaning', 'personally', 'stick', 'normal', 'u', 'army',\n","       'appeal', 'knowing', 'tough', 'remains', 'humour', 'military',\n","       'nowhere', 'journey', 'added', 'charlie', 'rare', 'thinks',\n","       'thank', 'dreams', 'c', 'purpose', 'touching', 'unbelievable',\n","       'comedies', 'week', 'catch', 'taste', 'terms', 'chase',\n","       'beautifully', 'channel', 'walking', 'sexy', 'batman', 'terrific',\n","       'twists', 'silent', 'fiction', 'kelly', 'wow', 'standard',\n","       'equally', 'naked', 'mood', 'subtle', 'mistake', 'managed',\n","       'complex', 'laughable', 'pick', 'themes', 'road', 'chris',\n","       'lovely', 'narrative', 'wild', 'disappointing', 'likely', 'brain',\n","       'costumes', 'thus', 'plan', 'issues', 'excuse', 'club',\n","       'outstanding', 'soldiers', 'painful', 'producer', 'date',\n","       'justice', 'surely', 'constantly', 'christopher', 'government',\n","       'fully', 'self', 'edge', 'law', 'boss', 'presented', 'victim',\n","       'cinematic', 'contains', 'door', 'places', 'central', 'details',\n","       'innocent', 'presence', 'climax', 'everybody', 'slowly',\n","       'marriage', 'besides', 'pass', 'ride', 'manner', 'charles',\n","       'hoping', 'animals', 'historical', 'charm', 'stunning', 'finish',\n","       'vampire', 'd', 'photography', 'impression', \"you'd\", 'henry',\n","       'loves', 'gang', 'spoiler', 'thrown', 'mysterious',\n","       'disappointment', 'expectations', 'bottom', 'hey', 'shoot',\n","       'allen', 'exception', 'paris', 'woods', 'minor', 'makers', 'soul',\n","       'festival', 'critics', 'aspects', 'stands', 'loud', 'indian',\n","       'suppose', 'scenery', 'train', 'bother', 'cry', 'church', 'color',\n","       'feelings', 'sent', 'heavy', 'support', 'emotion', 'bruce',\n","       'opportunity', 'award', 'hotel', 'brief', 'element', 'filming',\n","       'mainly', 'ahead', 'fascinating', 'forever', 'blue', 'rated',\n","       'acts', 'pieces', 'emotions', 'available', 'intended', 'twice',\n","       'dies', 'building', 'names', 'throw', 'track', 'compelling',\n","       'serial', 'happening', 'changes', 'drawn', 'hurt', 'don', 'puts',\n","       'green', 'falling', 'student', 'smart', 'jerry', 'likable',\n","       'offer', 'speaking', 'tired', 'suggest', 'include', 'bed', 'pain',\n","       'difference', 'victims', 'confused', 'adults', 'lover', 'followed',\n","       'impact', 'f', 'billy', 'arthur', 'approach', 'giant', 'motion',\n","       \"hasn't\", 'appeared', 'boyfriend', 'page', 'developed', 'bar',\n","       'image', 'park', 'jones', 'gorgeous', 'actresses', 'confusing',\n","       'trailer', 'laughed', 'notice', 'system', 'summer', 'share',\n","       'lacks', 'fresh', 'numbers', 'alien', 'fellow', 'event', 'grade',\n","       'supposedly', 'flaws', 'martin', 'murders', 'moral', 'zero',\n","       'content', 'noir', 'million', 'al', 'opera', 'mom', 'gem', 'ii',\n","       'relationships', 'tragedy', 'answer', 'mediocre', 'helps',\n","       'funniest', 'drive', 'merely', 'lighting', 'proves', 'wondering',\n","       'agent', 'students', 'born', 'random', 'mix', 'ray', 'j',\n","       'delivers', 'finding', 'hospital', 'creative', 'christian',\n","       'putting', 'key', 'standards', 'race', 'damn', 'drug', 'holes',\n","       'childhood', 'shock', 'imagination', 'impressed', 'absolute',\n","       'lived', 'negative', 'attractive', 'land', 'davis', 'paid', 'alan',\n","       'thoroughly', 'extreme', 'rape', 'flicks', 'provides', 'ms',\n","       'becoming', 'latter', 'addition', 'seemingly', 'seconds', 'folks',\n","       'reminded', 'flying', 'fell', 'brian', 'ugly', 'detail', 'offers',\n","       'thats', 'porn', 'tragic', 'faces', 'collection', 'afraid',\n","       'intense', 'lord', 'affair', 'six', 'spot', 'ship', 'williams',\n","       'stuck', 'lose', 'seven', 'industry', 'forgotten', 'hidden',\n","       'queen', 'soldier', 'count', 'nasty', 'beat', 'stone', 'apartment',\n","       'cliché', 'adds', 'jackson', 'held', 'castle', 'rented',\n","       \"shouldn't\", 'fashion', 'design', 'angry', 'artistic', 'uncle',\n","       'therefore', 'turning', 'americans', 'ground', 'filmmaker', 'area',\n","       'pull', 'information', 'games', 'states', 'anymore', 'bond',\n","       'location', 'super', 'listen', 'shocking', 'describe',\n","       'personality', 'jason', 'lets', 'step', 'danny', 'favourite',\n","       'scientist', 'deliver', 'asks', 'animal', 'ready', 'onto', 'fox',\n","       'creature', 'picked', 'wooden', 'inspired', 'chinese', 'stephen',\n","       'intelligence', 'clothes', 'news', 'grace', 'led', 'dirty',\n","       'redeeming', 'compare', 'thin', 'allowed', 'criminal', 'member',\n","       'carry', 'helped', 'artist', 'tears', 'wonderfully', 'drugs',\n","       'struggle', 'moved', 'teenage', 'captain', 'necessary',\n","       'desperate', 'includes', 'trust', 'deeply', 'whatsoever', 'wars',\n","       'willing', 'treat', 'began', 'andy', 'martial', 'food', 'direct',\n","       'g', 'ed', 'commentary', 'quick', 'nightmare', 'plane', 'theatre',\n","       'heaven', 'disaster', 'station', 'wall', 'professional', 'humans',\n","       'accident', 'douglas', 'sleep', 'phone', 'cgi', 'dying', 'worthy',\n","       'sky', 'introduced', 'superior', 'energy', 'johnny', 'rarely',\n","       'teacher', 'sight', 'warning', 'comedic', 'r', 'independent',\n","       'anybody', 'double', 'eddie', 'actions', 'unusual', 'roll',\n","       'realized', 'epic', 'remarkable', 'apparent', 'powers', 'mouth',\n","       'allow', 'mental', 'wearing', 'returns', 'continue', 'unnecessary',\n","       'tim', 'physical', 'witch', 'keaton', 'provide', 'technical',\n","       'pleasure', 'arts', 'absurd', 'normally', 'suicide', 'desire',\n","       'anywhere', 'england', 'engaging', 'superman', 'devil', 'ford',\n","       'taylor', 'fred', 'limited', 'skip', 'surprising', 'bloody', 'jr',\n","       'adam', 'scared', 'heroes', 'hitler', 'memory', 'process',\n","       'wedding', 'brutal', 'watchable', 'suspect', 'accept', 'joan',\n","       'ring', 'prince', 'media', 'machine', 'russian', 'intriguing',\n","       'legend', 'jeff', 'somebody', 'search', 'suit', 'wanting',\n","       'finished', 'reminds', 'holds', 'according', 'build', 'cops',\n","       'torture', 'academy', 'vision', 'pacing', 'hated', 'nicely',\n","       'passion', 'shakespeare', 'asked', 'religious', 'exist', 'cage',\n","       'dick', 'extra', 'grand', 'joy', 'bits', 'pilot', 'faith',\n","       'growing', 'clichés', 'nick', 'ladies', 'price', 'constant',\n","       'tarzan', 'blame', 'smile', 'originally', 'explanation', 'lies',\n","       'dangerous', 'kevin', 'instance', 'sat', 'community', 'jesus',\n","       'japan', 'moon', 'freddy', 'deserve', 'river', 'heroine', 'met',\n","       'toward', 'unknown', 'capture', 'gotten', 'higher', 'accurate',\n","       'players', 'winning', 'dressed', 'quiet', 'whilst', 'explained',\n","       'friendship', 'fail', 'teen', \"hadn't\", 'knowledge', 'drunk',\n","       'player', 'heads', 'kate', 'lovers', 'field', 'starting',\n","       'humanity', 'mike', 'guns', 'record', 'creating', 'sucks',\n","       'officer', 'memories', 'vhs', 'judge', 'jump', 'villains', 'cars',\n","       'pop', 'horse', 'finest', 'fights', 'issue', 'featuring', 'radio',\n","       'responsible', 'lacking', 'morgan', 'floor', 'saved', 'deserved',\n","       'jimmy', 'kinda', 'understanding', 'lynch', 'pulled', 'jean',\n","       'keeping', 'delightful', 'european', 'rubbish', 'results',\n","       'low-budget', 'treated', 'hopes', 'gene', 'monsters', 'manage',\n","       'loving', 'washington', 'gary', 'terribly', 'months', 'bland',\n","       'eat', 'santa', 'fate', 'simon', 'partner', 'included', 'mixed',\n","       'screaming', 'broken', 'sign', 'mine', 'numerous', 'singer',\n","       'hits', 'cable', 'witty', 'spanish', 'forces', 'whenever', 'empty',\n","       'author', 'fat', 'p', 'conflict', 'youth', 'streets', 'ball',\n","       'private', 'loose', 'loss', 'concerned', 'eric', 'reviewers',\n","       'brown', 'vs', 'pretentious', 'skills', 'werewolf', 'wind',\n","       'unfunny', 'ann', 'talents', 'naturally', 'ordinary', 'noticed',\n","       'psychological', 'discover', 'opposite', 'bob', 'regular',\n","       'realism', 'saving', 'perspective', 'albert', 'prove', 'finale',\n","       'bigger', 'morning', 'dated', 'driving', 'sean', 'anthony',\n","       'mission', 'gold', 'soap', 'cuts', 'portray', 'loses', 'blonde',\n","       'locations', 'length', 'discovered', 'bright', \"we've\", 'grant',\n","       'aware', 'film-making', 'continues', 'kong', 'humorous', 'below',\n","       'wood', 'satire', 'survive', 'dealing', 'international', 'visit',\n","       'magnificent', 'gags', 'dan', 'howard', 'calls', 'behavior',\n","       'breaks', 'owner', 'candy', 'shallow', 'debut', 'nonsense',\n","       'murdered', 'opens', 'curious', 'jennifer', 'visually', 'trek',\n","       'golden', 'connection', 'context', 'captured', 'essentially',\n","       'sing', 'shop', 'miles', 'deals', 'bank', 'advice', 'm', 'frankly',\n","       'occasionally', 'cameo', 'revealed', 'corny', 'received', 'blind',\n","       'harris', \"they've\", 'traditional', 'lesson', 'genuine', 'efforts',\n","       'learned', 'segment', 'window', 'current', 'gangster', 'visuals',\n","       'versions', 'scream', 'mrs', 'luke', 'develop', 'o', 'identity',\n","       'rob', 'allows', 'existence', 'national', 'program', 'sucked',\n","       'welles', 'genuinely', 'comparison', 'luck', 'decade',\n","       'references', 'unexpected', 'stock', 'anna', 'president',\n","       'formula', 'remembered', 'village', 'proved', 'meanwhile', 'favor',\n","       'ages', 'lake', 'grew', 'study', 'brilliantly', 'sea', 'vampires',\n","       'sheer', 'robin', 'unable', 'wise', 'ice', 'reach', 'sake',\n","       'ultimate', 'drew', 'board', 'steal', 'sudden', 'awards',\n","       'stereotypes', 'strength', 'logic', 'awkward', 'leader', 'boat',\n","       'passed', 'desert', 'plots', 'bet', 'cross', 'spectacular',\n","       'barbara', 'failure', 'sinatra', 'parker', 'h', 'halloween',\n","       'buddy', 'types', 'killers', 'reaction', 'sheriff', 'evening',\n","       'delivered', 'crappy', 'bear', 'gonna', 'steven', 'creates',\n","       'technology', 'fault', 'utter', 'rose', 'edited', 'discovers',\n","       'pair', 'thomas', 'laughter', 'lucky', 'dreadful', 'insane',\n","       'reviewer', 'flashbacks', 'families', 'majority', 'relief', 'w',\n","       'standing', 'graphic', 'emotionally', 'painfully', 'freedom',\n","       'ran', 'woody', 'caused', 'parody', 'gratuitous', 'decision',\n","       'site', 'entertained', 'religion', 'meeting', 'travel', 'anime',\n","       'individual', 'protagonist', 'attitude', 'wayne', 'underrated',\n","       'nevertheless', 'speed', 'victor', 'foreign', 'endless', 'seasons',\n","       'relate', 'costs', 'foot', 'feet', 'practically', 'gordon',\n","       'pleasant', 'daniel', 'test', 'hill', 'treatment', 'france',\n","       'cinderella', 'vehicle', 'twenty', 'combination', 'marie',\n","       'described', 'classics', 'v', 'alex', 'hunter', 'victoria',\n","       'native', 'gory', 'jackie', 'ancient', 'joseph', 'irritating',\n","       'levels', 'stopped', 'eating', 'l', 'overly', 'portrays', 'tape',\n","       'assume', 'rules', 'commercial', 'hearing', 'executed', 'product',\n","       'haunting', 'broadway', 'believes', 'excited', 'asking', 'range',\n","       'model', 'theaters', 'proper', 'wide', 'portraying', 'roy',\n","       'chosen', 'wit', 'chief', 'sell', 'uk', 'angel', 'rescue', 'moore',\n","       'extras', 'ruined', 'contrived', 'generation', 'capable', 'recall',\n","       'embarrassing', 'largely', 'matt', 'unrealistic', 'marry', 'clean',\n","       'produce', 'round', 'depressing', 'fill', 'nancy', 'center',\n","       'handsome', 'winner', 'choose', 'anne', 'germany', 'rise', 'cares',\n","       'facts', 'exploitation', 'matters', 'suffering', 'theatrical',\n","       'learns', 'ryan', 'priest', 'post', 'sequels', 'kick', 'involves',\n","       'clue', 'patrick', 'contrast', 'sympathetic', 'grow', 'fame',\n","       'appealing', 'research', 'strongly', 'louis', 'built', 'asian',\n","       'excitement', 'cash', 'evidence', 'dry', 'anderson', 'disgusting',\n","       'teenager', 'correct', 'lewis', 'appropriate', 'claim', 'chick',\n","       'vote', 'pity', 'heck', 'talks', 'losing', 'destroy', 'oliver',\n","       'walter', 'thoughts', 'canadian', 'lousy', 'circumstances',\n","       'costume', 'promise', 'tedious', 'crowd', 'voices', 'safe',\n","       'saturday', 'europe', 'substance', 'sports', 'training',\n","       'bringing', 'frame', 'haunted', 'bomb', 'fool', 'fly', 'naive',\n","       'captures', 'hanging', 'football', 'bodies', 'teenagers', 'till',\n","       'amateurish', 'mask', 'max', 'tend', 'convinced', 'satisfying',\n","       'obsessed', 'welcome', 'psycho', 'virtually', 'hopefully',\n","       'horribly', 'asleep', 'walks', 'hunt', 'scare', 'spoil', 'cost',\n","       'powell', 'danger', 'relatively', 'insult', 'grown', 'trilogy',\n","       'steals', 'africa', 'murphy', 'fits', 'universal', 'creatures',\n","       'reporter', 'unlikely', 'baseball', 'che', 'robot', 'tiny',\n","       'dubbed', 'continuity', 'seat', 'influence', 'market', 'depicted',\n","       'flesh', 'remain', 'hall', 'texas', 'united', 'kim', 'initial',\n","       'fu', 'unfortunate', 'offensive', 'category', 'send', 'russell',\n","       'fare', 'north', 'factor', 'cowboy', 'soft', 'believed', 'ass',\n","       'lawyer', 'politics', 'lower', 'qualities', 'provided', 'target',\n","       'drag', 'witness', 'contemporary', 'columbo', 'touches',\n","       'presents', 'refreshing', 'hide', 'rental', 'holding', 'promising',\n","       'professor', 'viewed', 'australian', 'display', 'source',\n","       'amateur', 'nominated', 'structure', 'cartoons', 'shocked',\n","       'hitchcock', 'makeup', 'forgettable', 'claims', 'movement',\n","       'weekend', 'sisters', 'caine', 'plans', 'edward', 'deaths',\n","       'surreal', 'closer', 'clark', 'handled', 'ruin', 'roger',\n","       'accents', 'surprises', 'angles', 'repeated', 'serves', 'chose',\n","       'adventures', 'eight', 'speaks', 'supernatural', 'brave',\n","       'previously', 'peace', 'pacino', 'warm', 'hat', 'over-the-top',\n","       'emma', 'deadly', 'granted', 'degree', 'latest', 'suffers',\n","       'weeks', 'whoever', 'cant', 'rain', 'skin', 'accidentally',\n","       'experiences', 'highlight', 'lesbian', 'covered', 'service',\n","       'enemy', 'dollars', 'mistakes', 'pile', 'routine', 'paper',\n","       'treasure', 'mgm', 'harsh', 'uninteresting', 'alice', 'realizes',\n","       'colors', 'make-up', 'combined', 'sympathy', 'speech', 'twisted',\n","       'aliens', 'friday', 'veteran', 'wilson', 'propaganda', 'universe',\n","       'convince', 'atrocious', 'dogs', 'designed', 'struggling', 'ted',\n","       'draw', 'path', 'frightening', \"would've\", 'mainstream', 'section',\n","       'recognize', 'invisible', 'lugosi', 'nude', 'walked', 'enter',\n","       'pitt', 'print', 'committed', 'x', 'aka', 'focused', 'guilty',\n","       'blah', 'princess', 'false', 'terror', 'occasional', 'security',\n","       'directly', 'narration', 'sarah', 'technically', 'com', 'offered',\n","       'theory', 'breaking', 'magical', 'massive', 'suspenseful', 'kinds',\n","       'variety', 'experienced', 'featured', 'driver', 'donald', 'prior',\n","       'irish', 'core', 'subtitles', 'reputation', 'explains', 'fairy',\n","       'birth', 'sorts', 'gritty'], dtype='<U14')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"2X1w4jn4yxUC","colab_type":"code","outputId":"9b48f13e-293d-4d90-e46f-ff4ee8d956e9","executionInfo":{"status":"ok","timestamp":1557761044892,"user_tz":-480,"elapsed":1031,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["vocab.size # 2500"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["2500"]},"metadata":{"tags":[]},"execution_count":102}]},{"cell_type":"code","metadata":{"id":"eXuC3AgFzKhR","colab_type":"code","colab":{}},"source":["lr = LogisticRegression()\n","lr.fit( no data )"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"7ZRuBGSrzNSI","colab_type":"text"},"source":["\n","\n","*   Find the words corresponding to the 5 largest coefficients.\n","*   Find the words corresponding to the 5 smallest coefficients.\n","\n"]},{"cell_type":"code","metadata":{"id":"35LdQoWszS3p","colab_type":"code","colab":{}},"source":["# Get the indices of the sorted cofficients\n","inds_ascending = np.argsort(lr.coef_.flatten()) \n","inds_descending = inds_ascending[::-1]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Y6gqsAudc7tn","colab_type":"code","colab":{}},"source":["# Print the most positive words\n","print(\"Most positive words: \", end=\"\")\n","for i in range(5):\n","    print(vocab[inds_descending[i]], end=\", \")\n","print(\"\\n\")\n","\n","# Most positive words: favorite, superb, noir, knowing, loved,"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"bJV3qP0zc_SL","colab_type":"code","colab":{}},"source":["# Print most negative words\n","print(\"Most negative words: \", end=\"\")\n","for i in range(5):\n","    print(vocab[inds_ascending[i]], end=\", \")\n","print(\"\\n\")\n","\n","# Most negative words: disappointing, waste, worst, boring, lame,"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-ZHMCxEudQ4M","colab_type":"text"},"source":["**Regularization and proabilities**\n","\n","In this exercise, you will observe the effects of changing the regularization stength on the predicted probabilities.\n","\n","A 2D binary classification dataset is already loaded into the environment as X and y.\n","\n","\n","\n","1.   Compute the maximum predicted probability.\n","2.   Create a model with C=0.1 and examine how the plot and probabilities change.\n","\n"]},{"cell_type":"code","metadata":{"id":"gNraNuAidsj6","colab_type":"code","cellView":"form","outputId":"d6988efd-5146-438d-87ee-2ff9b741a2e0","executionInfo":{"status":"ok","timestamp":1557806137697,"user_tz":-480,"elapsed":738,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["#@title\n","X = np.array([[ 1.78862847,  0.43650985],\n","       [ 0.09649747, -1.8634927 ],\n","       [-0.2773882 , -0.35475898],\n","       [-3.08274148,  2.37299932],\n","       [-3.04381817,  2.52278197],\n","       [-1.31386475,  0.88462238],\n","       [-2.11868196,  4.70957306],\n","       [-2.94996636,  2.59532259],\n","       [-3.54535995,  1.45352268],\n","       [ 0.98236743, -1.10106763],\n","       [-1.18504653, -0.2056499 ],\n","       [-1.51385164,  3.23671627],\n","       [-4.02378514,  2.2870068 ],\n","       [ 0.62524497, -0.16051336],\n","       [-3.76883635,  2.76996928],\n","       [ 0.74505627,  1.97611078],\n","       [-1.24412333, -0.62641691],\n","       [-0.80376609, -2.41908317],\n","       [-0.92379202, -1.02387576],\n","       [ 1.12397796, -0.13191423]])\n","\n","y = np.array([-1, -1, -1,  1,  1, -1,  1,  1,  1, -1, -1,  1,  1, -1,  1, -1, -1,\n","       -1, -1, -1])\n","\n","print(\"X array: \", X.shape)\n","print(\"y array: \", y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["X array:  (20, 2)\n","y array:  (20,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"pcQCnPbyd-a-","colab_type":"code","outputId":"64a2abc4-ef1f-4046-e486-279988d3b4ef","executionInfo":{"status":"ok","timestamp":1557806139113,"user_tz":-480,"elapsed":787,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":663}},"source":["# Set the regularization strength\n","model = LogisticRegression(C=1)\n","\n","# Fit and plot\n","model.fit(X,y)\n","plot_classifier(X,y,model,proba=True)\n","\n","# Predict probabilities on training points\n","prob = model.predict_proba(X)\n","print(\"Maximum predicted probability\", prob)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Maximum predicted probability [[0.9209835  0.0790165 ]\n"," [0.95866275 0.04133725]\n"," [0.84074059 0.15925941]\n"," [0.07667962 0.92332038]\n"," [0.07026958 0.92973042]\n"," [0.48558394 0.51441606]\n"," [0.023877   0.976123  ]\n"," [0.07068669 0.92931331]\n"," [0.1125408  0.8874592 ]\n"," [0.95855226 0.04144774]\n"," [0.71378564 0.28621436]\n"," [0.10913863 0.89086137]\n"," [0.04431288 0.95568712]\n"," [0.89400692 0.10599308]\n"," [0.03608222 0.96391778]\n"," [0.61928355 0.38071645]\n"," [0.77089251 0.22910749]\n"," [0.9511639  0.0488361 ]\n"," [0.85283928 0.14716072]\n"," [0.92092304 0.07907696]]\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n"],"name":"stderr"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADU9JREFUeJzt3GGI5Hd9x/H3xztTaYym9FaQu9Ok\n9NJ42ELSJU0Raoq2XPLg7oFF7iBYJXhgGylVhBRLlPjIhloQrtWTilXQGH0gC57cA40ExAu3ITV4\nFyLb03oXhawxzZOgMe23D2bSna53mX92Z3cv+32/4GD+//ntzJcfe++dndmZVBWSpO3vFVs9gCRp\ncxh8SWrC4EtSEwZfkpow+JLUhMGXpCamBj/JZ5M8meT7l7g+ST6ZZCnJo0lunP2YkqT1GvII/3PA\ngRe5/lZg3/jfUeBf1j+WJGnWpga/qh4Efv4iSw4Bn6+RU8DVSV4/qwElSbOxcwa3sRs4P3F8YXzu\np6sXJjnK6LcArrzyyj+8/vrrZ3D3ktTHww8//LOqmlvL184i+INV1XHgOMD8/HwtLi5u5t1L0ste\nkv9c69fO4q90ngD2ThzvGZ+TJF1GZhH8BeBd47/WuRl4pqp+7ekcSdLWmvqUTpIvAbcAu5JcAD4C\nvBKgqj4FnABuA5aAZ4H3bNSwkqS1mxr8qjoy5foC/npmE0mSNoTvtJWkJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYM\nviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMG\nX5KaMPiS1ITBl6QmDL4kNWHwJamJQcFPciDJ40mWktx1kevfkOSBJI8keTTJbbMfVZK0HlODn2QH\ncAy4FdgPHEmyf9Wyvwfur6obgMPAP896UEnS+gx5hH8TsFRV56rqOeA+4NCqNQW8Znz5tcBPZjei\nJGkWhgR/N3B+4vjC+NykjwK3J7kAnADef7EbSnI0yWKSxeXl5TWMK0laq1m9aHsE+FxV7QFuA76Q\n5Nduu6qOV9V8Vc3Pzc3N6K4lSUMMCf4TwN6J4z3jc5PuAO4HqKrvAq8Cds1iQEnSbAwJ/mlgX5Jr\nk1zB6EXZhVVrfgy8DSDJmxgF3+dsJOkyMjX4VfU8cCdwEniM0V/jnElyT5KD42UfBN6b5HvAl4B3\nV1Vt1NCSpJdu55BFVXWC0Yuxk+funrh8FnjLbEeTJM2S77SVpCYMviQ1YfAlqQmDL0lNGHxJasLg\nS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHw\nJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4\nktSEwZekJgy+JDUxKPhJDiR5PMlSkrsuseadSc4mOZPki7MdU5K0XjunLUiyAzgG/BlwATidZKGq\nzk6s2Qf8HfCWqno6yes2amBJ0toMeYR/E7BUVeeq6jngPuDQqjXvBY5V1dMAVfXkbMeUJK3XkODv\nBs5PHF8Yn5t0HXBdku8kOZXkwMVuKMnRJItJFpeXl9c2sSRpTWb1ou1OYB9wC3AE+EySq1cvqqrj\nVTVfVfNzc3MzumtJ0hBDgv8EsHfieM/43KQLwEJV/aqqfgj8gNEPAEnSZWJI8E8D+5Jcm+QK4DCw\nsGrN1xg9uifJLkZP8Zyb4ZySpHWaGvyqeh64EzgJPAbcX1VnktyT5OB42UngqSRngQeAD1XVUxs1\ntCTppUtVbckdz8/P1+Li4pbctyS9XCV5uKrm1/K1vtNWkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lN\nGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6Qm\nDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1IT\nBl+SmjD4ktSEwZekJgYFP8mBJI8nWUpy14use0eSSjI/uxElSbMwNfhJdgDHgFuB/cCRJPsvsu4q\n4G+Ah2Y9pCRp/YY8wr8JWKqqc1X1HHAfcOgi6z4GfBz4xQznkyTNyJDg7wbOTxxfGJ/7P0luBPZW\n1ddf7IaSHE2ymGRxeXn5JQ8rSVq7db9om+QVwCeAD05bW1XHq2q+qubn5ubWe9eSpJdgSPCfAPZO\nHO8Zn3vBVcCbgW8n+RFwM7DgC7eSdHkZEvzTwL4k1ya5AjgMLLxwZVU9U1W7quqaqroGOAUcrKrF\nDZlYkrQmU4NfVc8DdwIngceA+6vqTJJ7khzc6AElSbOxc8iiqjoBnFh17u5LrL1l/WNJkmbNd9pK\nUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAl\nqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS\n1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpoYFPwkB5I8nmQpyV0Xuf4DSc4meTTJN5O8\ncfajSpLWY2rwk+wAjgG3AvuBI0n2r1r2CDBfVX8AfBX4h1kPKklanyGP8G8ClqrqXFU9B9wHHJpc\nUFUPVNWz48NTwJ7ZjilJWq8hwd8NnJ84vjA+dyl3AN+42BVJjiZZTLK4vLw8fEpJ0rrN9EXbJLcD\n88C9F7u+qo5X1XxVzc/Nzc3yriVJU+wcsOYJYO/E8Z7xuf8nyduBDwNvrapfzmY8SdKsDHmEfxrY\nl+TaJFcAh4GFyQVJbgA+DRysqidnP6Ykab2mBr+qngfuBE4CjwH3V9WZJPckOThedi/wauArSf49\nycIlbk6StEWGPKVDVZ0ATqw6d/fE5bfPeC5J0oz5TltJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh\n8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow\n+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaGBT8JAeSPJ5kKcldF7n+N5J8eXz9Q0mumfWgkqT1mRr8JDuAY8CtwH7gSJL9\nq5bdATxdVb8L/BPw8VkPKklanyGP8G8ClqrqXFU9B9wHHFq15hDwb+PLXwXeliSzG1OStF47B6zZ\nDZyfOL4A/NGl1lTV80meAX4b+NnkoiRHgaPjw18m+f5aht6GdrFqrxpzL1a4FyvcixW/t9YvHBL8\nmamq48BxgCSLVTW/mfd/uXIvVrgXK9yLFe7FiiSLa/3aIU/pPAHsnTjeMz530TVJdgKvBZ5a61CS\npNkbEvzTwL4k1ya5AjgMLKxaswD85fjyXwDfqqqa3ZiSpPWa+pTO+Dn5O4GTwA7gs1V1Jsk9wGJV\nLQD/CnwhyRLwc0Y/FKY5vo65txv3YoV7scK9WOFerFjzXsQH4pLUg++0laQmDL4kNbHhwfdjGVYM\n2IsPJDmb5NEk30zyxq2YczNM24uJde9IUkm27Z/kDdmLJO8cf2+cSfLFzZ5xswz4P/KGJA8keWT8\n/+S2rZhzoyX5bJInL/VepYx8crxPjya5cdANV9WG/WP0Iu9/AL8DXAF8D9i/as1fAZ8aXz4MfHkj\nZ9qqfwP34k+B3xxffl/nvRivuwp4EDgFzG/13Fv4fbEPeAT4rfHx67Z67i3ci+PA+8aX9wM/2uq5\nN2gv/gS4Efj+Ja6/DfgGEOBm4KEht7vRj/D9WIYVU/eiqh6oqmfHh6cYvedhOxryfQHwMUafy/SL\nzRxukw3Zi/cCx6rqaYCqenKTZ9wsQ/aigNeML78W+MkmzrdpqupBRn/xeCmHgM/XyCng6iSvn3a7\nGx38i30sw+5Lramq54EXPpZhuxmyF5PuYPQTfDuauhfjX1H3VtXXN3OwLTDk++I64Lok30lyKsmB\nTZtucw3Zi48Ctye5AJwA3r85o112XmpPgE3+aAUNk+R2YB5461bPshWSvAL4BPDuLR7lcrGT0dM6\ntzD6re/BJL9fVf+1pVNtjSPA56rqH5P8MaP3/7y5qv5nqwd7OdjoR/h+LMOKIXtBkrcDHwYOVtUv\nN2m2zTZtL64C3gx8O8mPGD1HubBNX7gd8n1xAVioql9V1Q+BHzD6AbDdDNmLO4D7Aarqu8CrGH2w\nWjeDerLaRgffj2VYMXUvktwAfJpR7Lfr87QwZS+q6pmq2lVV11TVNYxezzhYVWv+0KjL2JD/I19j\n9OieJLsYPcVzbjOH3CRD9uLHwNsAkryJUfCXN3XKy8MC8K7xX+vcDDxTVT+d9kUb+pRObdzHMrzs\nDNyLe4FXA18Zv27946o6uGVDb5CBe9HCwL04Cfx5krPAfwMfqqpt91vwwL34IPCZJH/L6AXcd2/H\nB4hJvsToh/yu8esVHwFeCVBVn2L0+sVtwBLwLPCeQbe7DfdKknQRvtNWkpow+JLUhMGXpCYMviQ1\nYfAlqQmDL0lNGHxJauJ/Acz2XLpusNoKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"ycGPAmm8fCsS","colab_type":"code","outputId":"63df6411-bc1e-4121-d65b-4e9369042d1e","executionInfo":{"status":"ok","timestamp":1557806366954,"user_tz":-480,"elapsed":785,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":340}},"source":["# Set the regularization strength\n","model = LogisticRegression(C=0.1)\n","\n","# Fit and plot\n","model.fit(X,y)\n","plot_classifier(X,y,model,proba=True)\n","\n","# Predict probabilities on training points\n","prob = model.predict_proba(X)\n","print(\"Maximum predicted probability\", np.amax(prob))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Maximum predicted probability 0.8990965658995651\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n"],"name":"stderr"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADU9JREFUeJzt3GGI5Hd9x/H3xztTaYym9FaQu9Ok\n9NJ42ELSJU0Raoq2XPLg7oFF7iBYJXhgGylVhBRLlPjIhloQrtWTilXQGH0gC57cA40ExAu3ITV4\nFyLb03oXhawxzZOgMe23D2bSna53mX92Z3cv+32/4GD+//ntzJcfe++dndmZVBWSpO3vFVs9gCRp\ncxh8SWrC4EtSEwZfkpow+JLUhMGXpCamBj/JZ5M8meT7l7g+ST6ZZCnJo0lunP2YkqT1GvII/3PA\ngRe5/lZg3/jfUeBf1j+WJGnWpga/qh4Efv4iSw4Bn6+RU8DVSV4/qwElSbOxcwa3sRs4P3F8YXzu\np6sXJjnK6LcArrzyyj+8/vrrZ3D3ktTHww8//LOqmlvL184i+INV1XHgOMD8/HwtLi5u5t1L0ste\nkv9c69fO4q90ngD2ThzvGZ+TJF1GZhH8BeBd47/WuRl4pqp+7ekcSdLWmvqUTpIvAbcAu5JcAD4C\nvBKgqj4FnABuA5aAZ4H3bNSwkqS1mxr8qjoy5foC/npmE0mSNoTvtJWkJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYM\nviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMG\nX5KaMPiS1ITBl6QmDL4kNWHwJamJQcFPciDJ40mWktx1kevfkOSBJI8keTTJbbMfVZK0HlODn2QH\ncAy4FdgPHEmyf9Wyvwfur6obgMPAP896UEnS+gx5hH8TsFRV56rqOeA+4NCqNQW8Znz5tcBPZjei\nJGkWhgR/N3B+4vjC+NykjwK3J7kAnADef7EbSnI0yWKSxeXl5TWMK0laq1m9aHsE+FxV7QFuA76Q\n5Nduu6qOV9V8Vc3Pzc3N6K4lSUMMCf4TwN6J4z3jc5PuAO4HqKrvAq8Cds1iQEnSbAwJ/mlgX5Jr\nk1zB6EXZhVVrfgy8DSDJmxgF3+dsJOkyMjX4VfU8cCdwEniM0V/jnElyT5KD42UfBN6b5HvAl4B3\nV1Vt1NCSpJdu55BFVXWC0Yuxk+funrh8FnjLbEeTJM2S77SVpCYMviQ1YfAlqQmDL0lNGHxJasLg\nS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHw\nJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4\nktSEwZekJgy+JDUxKPhJDiR5PMlSkrsuseadSc4mOZPki7MdU5K0XjunLUiyAzgG/BlwATidZKGq\nzk6s2Qf8HfCWqno6yes2amBJ0toMeYR/E7BUVeeq6jngPuDQqjXvBY5V1dMAVfXkbMeUJK3XkODv\nBs5PHF8Yn5t0HXBdku8kOZXkwMVuKMnRJItJFpeXl9c2sSRpTWb1ou1OYB9wC3AE+EySq1cvqqrj\nVTVfVfNzc3MzumtJ0hBDgv8EsHfieM/43KQLwEJV/aqqfgj8gNEPAEnSZWJI8E8D+5Jcm+QK4DCw\nsGrN1xg9uifJLkZP8Zyb4ZySpHWaGvyqeh64EzgJPAbcX1VnktyT5OB42UngqSRngQeAD1XVUxs1\ntCTppUtVbckdz8/P1+Li4pbctyS9XCV5uKrm1/K1vtNWkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lN\nGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6Qm\nDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1IT\nBl+SmjD4ktSEwZekJgYFP8mBJI8nWUpy14use0eSSjI/uxElSbMwNfhJdgDHgFuB/cCRJPsvsu4q\n4G+Ah2Y9pCRp/YY8wr8JWKqqc1X1HHAfcOgi6z4GfBz4xQznkyTNyJDg7wbOTxxfGJ/7P0luBPZW\n1ddf7IaSHE2ymGRxeXn5JQ8rSVq7db9om+QVwCeAD05bW1XHq2q+qubn5ubWe9eSpJdgSPCfAPZO\nHO8Zn3vBVcCbgW8n+RFwM7DgC7eSdHkZEvzTwL4k1ya5AjgMLLxwZVU9U1W7quqaqroGOAUcrKrF\nDZlYkrQmU4NfVc8DdwIngceA+6vqTJJ7khzc6AElSbOxc8iiqjoBnFh17u5LrL1l/WNJkmbNd9pK\nUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAl\nqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS\n1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpoYFPwkB5I8nmQpyV0Xuf4DSc4meTTJN5O8\ncfajSpLWY2rwk+wAjgG3AvuBI0n2r1r2CDBfVX8AfBX4h1kPKklanyGP8G8ClqrqXFU9B9wHHJpc\nUFUPVNWz48NTwJ7ZjilJWq8hwd8NnJ84vjA+dyl3AN+42BVJjiZZTLK4vLw8fEpJ0rrN9EXbJLcD\n88C9F7u+qo5X1XxVzc/Nzc3yriVJU+wcsOYJYO/E8Z7xuf8nyduBDwNvrapfzmY8SdKsDHmEfxrY\nl+TaJFcAh4GFyQVJbgA+DRysqidnP6Ykab2mBr+qngfuBE4CjwH3V9WZJPckOThedi/wauArSf49\nycIlbk6StEWGPKVDVZ0ATqw6d/fE5bfPeC5J0oz5TltJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh\n8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow\n+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaGBT8JAeSPJ5kKcldF7n+N5J8eXz9Q0mumfWgkqT1mRr8JDuAY8CtwH7gSJL9\nq5bdATxdVb8L/BPw8VkPKklanyGP8G8ClqrqXFU9B9wHHFq15hDwb+PLXwXeliSzG1OStF47B6zZ\nDZyfOL4A/NGl1lTV80meAX4b+NnkoiRHgaPjw18m+f5aht6GdrFqrxpzL1a4FyvcixW/t9YvHBL8\nmamq48BxgCSLVTW/mfd/uXIvVrgXK9yLFe7FiiSLa/3aIU/pPAHsnTjeMz530TVJdgKvBZ5a61CS\npNkbEvzTwL4k1ya5AjgMLKxaswD85fjyXwDfqqqa3ZiSpPWa+pTO+Dn5O4GTwA7gs1V1Jsk9wGJV\nLQD/CnwhyRLwc0Y/FKY5vo65txv3YoV7scK9WOFerFjzXsQH4pLUg++0laQmDL4kNbHhwfdjGVYM\n2IsPJDmb5NEk30zyxq2YczNM24uJde9IUkm27Z/kDdmLJO8cf2+cSfLFzZ5xswz4P/KGJA8keWT8\n/+S2rZhzoyX5bJInL/VepYx8crxPjya5cdANV9WG/WP0Iu9/AL8DXAF8D9i/as1fAZ8aXz4MfHkj\nZ9qqfwP34k+B3xxffl/nvRivuwp4EDgFzG/13Fv4fbEPeAT4rfHx67Z67i3ci+PA+8aX9wM/2uq5\nN2gv/gS4Efj+Ja6/DfgGEOBm4KEht7vRj/D9WIYVU/eiqh6oqmfHh6cYvedhOxryfQHwMUafy/SL\nzRxukw3Zi/cCx6rqaYCqenKTZ9wsQ/aigNeML78W+MkmzrdpqupBRn/xeCmHgM/XyCng6iSvn3a7\nGx38i30sw+5Lramq54EXPpZhuxmyF5PuYPQTfDuauhfjX1H3VtXXN3OwLTDk++I64Lok30lyKsmB\nTZtucw3Zi48Ctye5AJwA3r85o112XmpPgE3+aAUNk+R2YB5461bPshWSvAL4BPDuLR7lcrGT0dM6\ntzD6re/BJL9fVf+1pVNtjSPA56rqH5P8MaP3/7y5qv5nqwd7OdjoR/h+LMOKIXtBkrcDHwYOVtUv\nN2m2zTZtL64C3gx8O8mPGD1HubBNX7gd8n1xAVioql9V1Q+BHzD6AbDdDNmLO4D7Aarqu8CrGH2w\nWjeDerLaRgffj2VYMXUvktwAfJpR7Lfr87QwZS+q6pmq2lVV11TVNYxezzhYVWv+0KjL2JD/I19j\n9OieJLsYPcVzbjOH3CRD9uLHwNsAkryJUfCXN3XKy8MC8K7xX+vcDDxTVT+d9kUb+pRObdzHMrzs\nDNyLe4FXA18Zv27946o6uGVDb5CBe9HCwL04Cfx5krPAfwMfqqpt91vwwL34IPCZJH/L6AXcd2/H\nB4hJvsToh/yu8esVHwFeCVBVn2L0+sVtwBLwLPCeQbe7DfdKknQRvtNWkpow+JLUhMGXpCYMviQ1\nYfAlqQmDL0lNGHxJauJ/Acz2XLpusNoKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"uKliae3fftcj","colab_type":"text"},"source":["As you probably noticed, smaller values of C lead to less confident predictions. That's because smaller C means more regularization, which in turn mean smaller coefficients, which means raw model outputs closer to zero, and thus, probabiities closer to 0.5 after the raw model output is squashed through the sigmoid function.\n","\n","smaller C > more regularization > smaller coefficients > raw outputs closer to zero"]},{"cell_type":"markdown","metadata":{"id":"Yl--Svr6gXIM","colab_type":"text"},"source":["**Visualizing easy and difficult examples**\n","\n","In this exercise, you'll visualize the examples that the logistic regression model is most, and least, confident about by looking at the largest, and smallest, predicted probabilities.\n","\n","The handwritten digits dataset is already loaded into the variables X and y. The show_digit function takes in an integer index and plots the corresponding image, with some extra information displayed above the image."]},{"cell_type":"code","metadata":{"id":"3-J_sPMZgbOS","colab_type":"code","outputId":"42defdbf-534b-4d43-eb0d-b9c64b310e8e","executionInfo":{"status":"ok","timestamp":1557807165185,"user_tz":-480,"elapsed":755,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["digits = datasets.load_digits()\n","X = digits.data\n","y = digits.target\n","print(\"X shape \", X.shape)\n","print(\"y shape\", y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["X shape  (1797, 64)\n","y shape (1797,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"cats-g0tgMNe","colab_type":"code","colab":{}},"source":["def show_digit(i, lr=None):\n","    plt.imshow(np.reshape(X[i], (8,8)), cmap='gray', vmin = 0, vmax = 16, interpolation=None)\n","    plt.xticks(())\n","    plt.yticks(())\n","    if lr is None:\n","        plt.title(\"class label = %d\" % y[i])\n","    else:\n","        pred = lr.predict(X[i][None])\n","        pred_prob = lr.predict_proba(X[i][None])[0,pred]\n","        plt.title(\"label=%d, prediction=%d, proba=%.2f\" % (y[i], pred, pred_prob))\n","    plt.show()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"-igPI8T1i0TK","colab_type":"code","outputId":"b27f85e6-f2eb-46e8-c3eb-cd69366013b7","executionInfo":{"status":"ok","timestamp":1557807412686,"user_tz":-480,"elapsed":1410,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"source":["lr = LogisticRegression()\n","lr.fit(X,y)\n","\n","# Get predicted probabilities\n","proba = lr.predict_proba(X)\n","\n","# Sort the example indices by their maximum probability\n","proba_inds = np.argsort(np.max(proba,axis=1))\n","proba_inds"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["array([1660, 1628, 1658, ...,  565, 1051,  646])"]},"metadata":{"tags":[]},"execution_count":23}]},{"cell_type":"code","metadata":{"id":"nkHChmn3jrPs","colab_type":"code","outputId":"fd54ee6f-22d9-4e65-9175-ad2129c119c2","executionInfo":{"status":"ok","timestamp":1557807419344,"user_tz":-480,"elapsed":789,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":268}},"source":["# Show the most confident (least ambiguous) digit\n","show_digit(proba_inds[-1], lr)"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAO4AAAD7CAYAAABt9agKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAC89JREFUeJzt3XuMXFUdwPHvoS0vha5FMCmUNrwR\nlQ0+8A8xG0AhJEoxKpFAaACBqCEVJTwkZjWIgCI24CMRoUoUHxiJwRASkfIUH0h5VMDwWCxCCRZa\nWFFUOP5xTuEy7HRntsvt/rbfT7LJzt47c+49c78zd24hk3LOSIpls429AZL6Z7hSQIYrBWS4UkCG\nKwVkuFJAGxRuSmkkpXRwj+vmlNJuExxnwvedilJKQymlxxq3V6SUhibwOAeklB6Y1I0LZLodF/2Y\ntu+4qTg/pbS6/pyfUkobe7vGknPeJ+e8bLz1Og/UnPPNOec9X9eNe+02LEgp3ZBSej6ldH+vL9xR\npZQ+k1L6U0rphZTS0h7W/2xKaVVK6dmU0mUppS0ayyZt7qZtuMCJwEJgX+AdwIeAkyZ7kJTSzMl+\nzCnuSuBOYDvgC8BVKaXtJ3uQKTSvjwPnAJeNt2JK6RDgDOAgYD6wC/ClxiqTN3c55wn/ACPAwfX3\n9wC/A9YATwCXAJs31s3AKcDDwD+ArwGbNZYfB9wHPANcB8zvuO9ufW7bbcCJjdvHA7f3eN9h4Crg\np8BzwJ+BfTv2+3TgbuAFYCYwF/gF8BTwCHBKY/2tgKV13/4CnAY81mUeZwBnAQ/Vse8A5gE31Xn4\nJzAKHAkMdTzO3sCy+hysAD7cWLYU+Bbw6/q4vwd27XNO96j7u03jbzcDJ/dxvJxZ5+AZ4HJgy7ps\nCHiszusq4Ir6908CDwJPA78C5vZyTAG7Ar8FVtdlPwIGNuBYPwdYOs46PwbObdw+CFg1GXP3mrEm\nMdx3Au+tB/ECSoSLOyb5BmAOsDPwV+CEuuzw+uTsXe9/NnDbWOFSXtHWdPtp3GctsH/j9ruA5/oI\n97/AR4FZwOcpMc5q7PdySlBbUc5c7gC+CGxOeaV9GDikrn9efZLm1PvcS/dwTwPuAfYEEuWMYbvO\neWge7PX3WXUOz6rbcCAl0D0b4a6mvMDOrAfyTxqPdfd65vXbdZ0jgPs65uoS4OI+jpd76xzMAW4F\nzmnsy/+A84Et6rweSIluv/q3i4GbejymdgM+UO+3PeWF75uN+16znv29ZoLh3gUc2bj95rqN223o\n3L1u4Y6xbDHwy45JPrRx+1PA9fX3a4HjG8s2A56nvut2HrA9btuLwF6N27vXx0k9hnt7x/Y8ARzQ\n2O/jGsv3B/7W8RhnApfX3x/u2PcT6R7uA8DhXbZrfeEeQHmnap7FXAkMN8K9tLHsMOD+Puf0GDrO\nWoCvjHdAd+znyR3b8FBjX/5DfQeuf/s+cEHj9hspL6gLxjumxhh7IXDnBhzrvYT7UMf2zKrbuGBD\n567zZ9I+R6SU9gC+QXln25ryqn5Hx2orG78/Sjm9hPJ5YElK6cLmQwI71vUmYhTYtnF7W2A01xnr\nwcvbmnN+qV4FnjvWcsr2z00prWn8bQblXZZ6v85972Ye5QDo11xgZc75pY5xdmzcXtX4/XlKCP3o\nnFPq7ef6eIxuxwDAUznnfzduz6V8TAEg5zyaUlpN2aeR9T1eSuktwBLKC9o2lBffZ/rYzokY65iD\nMj+TMXcvm8yLU98B7gd2zzlvSzll67yKO6/x+86UD/5QJv+knPNA42ernPNtnYOklM5KKY12+2ms\nuoJymrnOvvVvvXp5W1NKmwE7NbYXyivpOiuBRzq2f5uc82F1+RNj7Hs3Kymfz/r1ODCvbmtznL/3\ncuf6T1Ld5vW7dbUVwC4ppW0ad53wvPLqYwBePafUZfMb2/gGymlnc5+6Pd659fHeXo/Ho2kcjyml\na9ezv9f2sT9NYx1zT+acVzM5c/eKiZ46jHGK9wfKZ7wE7EU55bul4zTveuBNlMm+n3rxiHL+fy+w\nT709G/hYt1PEHrftZMrn7B0pr8IrePVp2giwqMt9hymnZB+hnDmcWtef1bnf9fYMyjvD6ZTPZjOA\ntwHvrsvPB26s+74T5fPk+j7j3k05tU+UK+LrPuOuAj7YuN8Qr5wqb045JT+Dcoo2RHk136suX0r9\nPNl53z7n9Xbg68CW9XlbA2zfeMw8zvFyT52DOcAt1Is5Y20PcDDlYt8g5bPqkj6OqZ8B36vPxY6U\nz9MT2d+ZdV+/ClxRf5/ZZd1D63P0VmCAcnHsvF7mru/tmsRw318nbpRyivjlMSZ53RXA1cCFwIzG\n8mPqk/os5V3nsg0MNwEXUK5GPl1/T42D/OWDuku4zavKdwL7jbXfjb/NpXymXEU5Jbu9MTdbAz+s\nT1QvV5XPplwMew74I7BTfuXF6In6OB/vPNiBfSgvEGvrOEc0li1lcsJdQLly/S/Ki3PzBewY4NZx\njpd1V5XXAD8Atl7f9tR9fqg+h9esm4vxjqk6F3dQjsflwOcmuL/DdZzmz3BdtnN9/J0b658KPEk5\nji8Htuhl7vr9WXcgb1JSSu8DPp1z/kSX5cOUF4qjW92w4FJKlwI/zzlf12X5COWq729a3bBpaKr8\nI3ercs63UE7TNIlyzids7G3YVEzn/3JKmrY2yVNlKTrfcaWADFcKqK+LUymlaXlevcMOO7Q63rx5\n88ZfaZK8+OKLrY314IMPtjbW6Ojo+CsFlXMe938/3SSvKnc66qijWh3voosuam2stWvXtjbWwoUL\nWxtr2bJlrY01FXmqLAVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVk\nuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkB9fenXdP0mg5GRkVbHmz9/fmtj\nLVmypLWxFi1a1NpYg4ODrY0F7R4jvXyTge+4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJA\nhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCG\nKwU0c2NvQDcDAwPTciyAG2+8sbWxhoeHWxurTW1+3QlMvXn0HVcKyHClgAxXCshwpYAMVwrIcKWA\nDFcKyHClgAxXCshwpYAMVwrIcKWADFcKyHClgAxXCshwpYAMVwrIcKWADFcKyHClgAxXCshwpYAM\nVwrIcKWADFcKyHClgAxXCmjKfnfQ4OBga2PNnj27tbEAhoaGWh2vLSMjI62N1ebxMRX5jisFZLhS\nQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJA\nhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQH4FCXDXXXe1NtZ01uZzNjAw0NpYU5HvuFJAhisF\nZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVk\nuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFNGW/gqRNa9as2dib8LpZsGBBa2MtXLiwtbEWL17c\n2lhTke+4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJA\nhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwWUcs69r5xS7ytvoMHBwbaGYtmy\nZa2NBbB8+fLWxmrzK0ja1ObxAe1+TU3OOY23ju+4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVk\nuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4UkCGKwVkuFJAhisFZLhSQIYrBWS4\nUkCGKwVkuFJAMzf2BnTT5vfrXH311a2NBXDssce2Ntajjz7a2liLFi1qbaw2v8tnKvIdVwrIcKWA\nDFcKyHClgAxXCshwpYAMVwrIcKWADFcKyHClgAxXCshwpYAMVwrIcKWADFcKyHClgAxXCshwpYAM\nVwrIcKWADFcKyHClgAxXCshwpYAMVwrIcKWAUs6595VTegpo7zstpE3P/Jzz9uOt1Fe4kqYGT5Wl\ngAxXCshwpYAMVwrIcKWADFcKyHClgAxXCshwpYD+D/eyf/HcCxbHAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"Vve--36Ljtae","colab_type":"code","outputId":"36c1fc07-ec9b-488a-dc4e-86b09fc8ac38","executionInfo":{"status":"ok","timestamp":1557807425923,"user_tz":-480,"elapsed":728,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":268}},"source":["# Show the least confident (most ambiguous) digit\n","show_digit(proba_inds[0], lr)"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAO4AAAD7CAYAAABt9agKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADGNJREFUeJzt3X2MZWV9wPHvj12ovI+LIF1ZlwoI\nSO2ytmij3XaW4ktIdRdsJSKkRNEaTQi1Gl7SNhslWCQtQ6zaRiu00Qi+AYIhNlBHwDcacIuCoCwv\nLsKWFXaWXUhjq0//OM+sZy9zZ+Zeh7Pzm/1+kk3mzjn3Ps8593zvPefuZm+UUpCUyx67egKSBme4\nUkKGKyVkuFJChislZLhSQr9WuBHxUEScNMt1S0QcOeQ4Q993PoqI0Yh4pHX77ogYHeJxVkXEfXM6\nuUQW2nExiAX/jhsRe0XED9uhzDellONKKeMzrdd7oJZSbi2lHP2cTq7/XP6ozueiXTF+VyJiSURc\nExFPR8TDEXH6NOv+ZUQ8EBFPRcSjEXFZRCxuLX91RNweEdsi4q6I+INh57XgwwU+AGx+rh68/cTs\nLiJiT+By4LvP4RjzZb9+DPg58ELgbcAnIuK4Put+BXhFKeUA4LeBFcA50LwAANcDlwIjwEeA6yPi\n+cNMas7CjYhXRsS3I2IiIh6LiH+MiL16Vju5viL9LCIujYg9Wvd/e31n3BIRX4uI5XMwp98CzgA+\nPOD91kXEFyPi6vrqeGdErGgtfygizouIu4CnI2JxRCyNiC9FxOaIeDAizmmtv3dEXFm37R7ghJ7x\ndlxyRMSiiLgwIjbUse+IiGURcUtd/b8iYntEnDbFKfexETFen4O7I+JNrWVXRsTHIuKr9XG/GxFH\nDLJfWv4K+Hfg3kHuVLfzgoi4p+6LKyLieXXZaEQ8UvfrJuCK+vt3RsT9EfFkRHwlIpb2POyUx1RE\nHBER/xERT9Rln42IkQHnuy/wZuBvSinbSym30cR55lTrl1I2lFImJu8O/BKYPEN6NbCplPKFUsov\nSimfoXlDOXWQObUHG/oP8BBwUv35d4HfBxYDhwM/BM5trVuArwNLgBcDPwLOrsvWAPcDx9b7/zXw\nrZ77Hll/Ph+Y6PenZ343AKcAo8AjA2zXOuB/gT8F9gTeDzwI7Nna7vXAMmBvmhfAO4C/BfYCXgI8\nALy+rv93wK1125cBP2jPp2c/fgD4PnA0zZO/Ajiodz/U2zu2q87zfuDCOocTgW3A0XX5lcATwCvr\nPv4scFXrse6aZr9+vLXe8vrc7Vcf86IBj5cf1H2wBPjm5P3rtvwfcAnwG3W/ngj8DHhF/d1HgVtm\neUwdCby23u9g4BZgrOfY6Le9N9R1VgLP9GzD+4Hrp9nG04Gn6tw2Ayvq7/8EuKdn3R8Dlw3V3lyF\nO8Wyc4FrenbyG1q33wPcXH++EXhHa9kewDPA8qkO2FnO7RTgxt4DfIBwv9Mzn8eAVa3tfntr+auA\nn/Q8xgXAFfXnB3q2/V30D/c+YE2feU0X7ipgE7BHa/nngHWtcD/VWnYycO8Qz/l1wGmtxxw03Hf3\nzGFDa1t+DjyvtfxfgI+0bu9H84J6+EzH1BRjrwW+N+C2rqJ5l2z/7p3A+CzuexTwIeDQevsgmheF\nt9K8yP45zTvyPw/6HJRS5vRU+aURcUNEbIqIp4CLgRf0rLax9fPDwORpz3Lg8nqKNwE8SfNu86Ih\n57IvzTXEOTOtO40dcy2l/BJ4pDXfnZbTzH/p5PzrNlxIc11EvV/vtvezDNgwxHyXAhvrXNvjtPfh\nptbPz9CEMGsR8UZg/1LK1UPMb1K/YwBgcynlf1q3l9LaV6WU7TRnDe1tmvLxIuKFEXFVRPy0Ho+f\n4dnH40y2Awf0/O4AmjOZaZVSfgzcDXy83n6C5szyfcB/A28AbqI5rgY2lx9OfYLmmueo0lycX0gT\nX9uy1s8vBh6tP28E/qKUMtL6s3cp5Vu9g9Trv+39/tTVjqI5Xb+1Xi99GfjN+qJy+Cy3Z8dc63XT\nYa35QvNqP2kj8GDP/PcvpZxclz82xbb3sxEY5trzUWBZ+3ODOs5PZ3Pnek3cb7/+U13tj4Hfq/tx\nE3AacG5EXDfAPPsdA7DzPp3cph2fddQX5IN6tqnf411cH+/l9Xg8g9bxGBE3TrO9N9bVfgQsjoij\nWmOsoAlyNhbTei5LKd8opZxQSllCc518DHD7LB9rZ8O8Tfc5xbud5hov6oTuA27rOc27GXg+zc6+\nF3hXXXYKzbXPcfX2gcCf9TtFnMW8FgOHtv6cSvOEHgosas39rD73X0dzSnZqfaz31fX37N3uensR\ncCdwHs212SKaTxVPqMsvAb5Rt/0wmuvJ6a5x76J58Qngd/jVNe4m4HWt+43yq1PlvWhOyc+nORUb\npXlnOKZMcVrLgJcP9T779+zXq4HLgCWtxywzHC/fr/tgCXAbcHG/+QAn0VwnHk9zrXr5AMfU54FP\n1ufiRTTX0wNtb32cq2guOfYFXgNspR6nU6x7NnBI/fllNIH/Q2v5yvrcHACMAd8cur05DPcP647b\nTvNBzAen2Mnn1IPrCeDvqRHV5WfWJ/UpmnedTw8b7hTz3OmgqAf5joO6T7hfrAfmNuB7NB/zP2u7\nW79bWp/gTcAW4DutfbMP8G801zj30MTZL9xFNB/OPVjH/k/gsLrs3TTv3hPAW6bYruNoXiC21nFO\naS27kl8z3Cn2U+9jnjndwVi384I6twngX4F9pptP3eYNNJdPN0zui5mOqbov7qjH43qaT8KHCXcJ\ncC3wNPAT4PTWslXA9tbtK2hOg5+u23opO1+zf64+N1vrsXXIsPs+6gPuVqL5i+/3llLe2mf5OpoX\nijM6nVhyEfEp4AullK/1Wf4Qzae+N3U6sQVovvwld6dK8/dxt+3qeSw0pZSzd/Ucdhe7w7+ckhac\n3fJUWcrOd1wpIcOVEhrow6mIWJDn1UccMey/tR/Otm0z/sObOfP44493NpbmRiml9x8uPctA17gL\nNdxrr7220/HGx8c7G2tsbKyzsTQ3ZhOup8pSQoYrJWS4UkKGKyVkuFJChislZLhSQoYrJWS4UkKG\nKyVkuFJChislZLhSQoYrJWS4UkKGKyVkuFJChislZLhSQoYrJWS4UkKGKyVkuFJChislNG//Q/Q1\na9Z0NVTn/yH6ypUrOxtrYmKis7G6/I/e165d29lYAOvXr+9sLP9DdGmBMlwpIcOVEjJcKSHDlRIy\nXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJc\nKSHDlRIyXCkhw5USMlwpocW7egL9rF69urOxtm7d2tlY0O3XWXT5tSAjIyOdjdXlPpyPfMeVEjJc\nKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwp\nIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRKat98d1OV3wxx44IGdjQUwMTHR2Vhdbtva\ntWs7G2t35zuulJDhSgkZrpSQ4UoJGa6UkOFKCRmulJDhSgkZrpSQ4UoJGa6UkOFKCRmulJDhSgkZ\nrpSQ4UoJGa6UkOFKCRmulJDhSgkZrpSQ4UoJGa6UkOFKCRmulJDhSglFKWX2K0fMfuVEzjrrrE7H\nGxsb62ys8fHxzsbyK0jmRiklZlrHd1wpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIy\nXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEvIr\nSICRkZFOx9uyZUtnY61evbqzsbr8upOFzK8gkRYow5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwp\nIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkh\nw5US8itIgLGxsU7HO/744zsba3R0tLOxNDf8ChJpgTJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHD\nlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOVEjJcKSHDlRIyXCkhw5USMlwpIcOV\nEjJcKSHDlRJavKsnMB90/f0669ev73Q8LTy+40oJGa6UkOFKCRmulJDhSgkZrpSQ4UoJGa6UkOFK\nCRmulJDhSgkZrpSQ4UoJGa6UkOFKCRmulJDhSgkZrpSQ4UoJGa6UkOFKCRmulJDhSgkZrpSQ4UoJ\nGa6UUJRSZr9yxGbg4eduOtJub3kp5eCZVhooXEnzg6fKUkKGKyVkuFJChislZLhSQoYrJWS4UkKG\nKyVkuFJC/w82wk1ItCuA6QAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"F6v0W-mihpjT","colab_type":"text"},"source":["#### Multi-class logistic regression\n","\n","In this exercise, you'll fit the two types of multi-class logistic regression, one-vs-rest and softmax/multinomial, on the handwritten digits data set and compare the results. The handwritten digits dataset is already loaded and split into `X_train, y_train, X_test`, and `y_test`."]},{"cell_type":"code","metadata":{"id":"MFWD7aWrhsgb","colab_type":"code","outputId":"d3fe5cf7-d61c-45ac-c8ae-482a1f5ab0ea","executionInfo":{"status":"ok","timestamp":1557807902848,"user_tz":-480,"elapsed":718,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=450, random_state=42)\n","print(\"X_train shape\", X_train.shape)\n","print(\"X_test shape\", X_test.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["X_train shape (1347, 64)\n","X_test shape (450, 64)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"6EKAu7ivlj3j","colab_type":"code","outputId":"11f2da39-34f0-44e6-c20e-03d5b243fb0a","executionInfo":{"status":"ok","timestamp":1557807974632,"user_tz":-480,"elapsed":936,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"source":["# Fit one-vs-rest logistic regression classifier\n","lr_ovr = LogisticRegression()\n","lr_ovr.fit(X_train, y_train)\n","\n","print(\"OVR training accuracy:\", lr_ovr.score(X_train, y_train))\n","print(\"OVR test accuracy    :\", lr_ovr.score(X_test, y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n","  \"this warning.\", FutureWarning)\n"],"name":"stderr"},{"output_type":"stream","text":["OVR training accuracy: 0.9948032665181886\n","OVR test accuracy    : 0.9666666666666667\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"LKJXaYq8lyBk","colab_type":"code","outputId":"2750d9ef-1a7a-45c7-dd5b-aee15a8bce6f","executionInfo":{"status":"ok","timestamp":1557807979201,"user_tz":-480,"elapsed":765,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":105}},"source":["# Fit softmax classifier\n","lr_mn = LogisticRegression(multi_class=\"multinomial\", solver=\"lbfgs\")\n","lr_mn.fit(X_train, y_train)\n","\n","print(\"Softmax training accuracy:\", lr_mn.score(X_train, y_train))\n","print(\"Softmax test accuracy    :\", lr_mn.score(X_test, y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Softmax training accuracy: 1.0\n","Softmax test accuracy    : 0.9733333333333334\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n","  \"of iterations.\", ConvergenceWarning)\n"],"name":"stderr"}]},{"cell_type":"markdown","metadata":{"id":"NO6_4s7z_7Dq","colab_type":"text"},"source":["**Visualizing multi-class logistic regression**\n","\n","In this exercise we'll continue with the two types of multi-class logistic regression, but on a toy 2D data set specifically designed to break the one-vs-rest scheme.\n","\n","The data set is loaded into X_train and y_train. The two logistic regression objects,lr_mn and lr_ovr, are already instantiated (with C=100), fit, and plotted.\n","\n","Notice that lr_ovr never predicts the dark blue class... yikes! Let's explore why this happens by plotting one of the binary classifiers that it's using behind the scenes."]},{"cell_type":"code","metadata":{"id":"MU-uhMns_5Y6","colab_type":"code","outputId":"c6d2acf1-f246-4bbe-d1f1-775e908bead7","executionInfo":{"status":"ok","timestamp":1557814847975,"user_tz":-480,"elapsed":1169,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":357}},"source":["# Print training accuracies\n","print(\"Softmax     training accuracy:\", lr_mn.score(X_train, y_train))\n","print(\"One-vs-rest training accuracy:\", lr_ovr.score(X_train, y_train))\n","\n","# Create the binary classifier (class 1 vs. rest)\n","lr_class_1 = LogisticRegression(C=100)\n","lr_class_1.fit(X_train, y_train==1)\n","\n","# Plot the binary classifier (class 1 vs. rest)\n","plot_classifier(X_train, y_train==1, lr_class_1)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Softmax     training accuracy: 1.0\n","One-vs-rest training accuracy: 0.9948032665181886\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n","  FutureWarning)\n"],"name":"stderr"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADU9JREFUeJzt3GGI5Hd9x/H3xztTaYym9FaQu9Ok\n9NJ42ELSJU0Raoq2XPLg7oFF7iBYJXhgGylVhBRLlPjIhloQrtWTilXQGH0gC57cA40ExAu3ITV4\nFyLb03oXhawxzZOgMe23D2bSna53mX92Z3cv+32/4GD+//ntzJcfe++dndmZVBWSpO3vFVs9gCRp\ncxh8SWrC4EtSEwZfkpow+JLUhMGXpCamBj/JZ5M8meT7l7g+ST6ZZCnJo0lunP2YkqT1GvII/3PA\ngRe5/lZg3/jfUeBf1j+WJGnWpga/qh4Efv4iSw4Bn6+RU8DVSV4/qwElSbOxcwa3sRs4P3F8YXzu\np6sXJjnK6LcArrzyyj+8/vrrZ3D3ktTHww8//LOqmlvL184i+INV1XHgOMD8/HwtLi5u5t1L0ste\nkv9c69fO4q90ngD2ThzvGZ+TJF1GZhH8BeBd47/WuRl4pqp+7ekcSdLWmvqUTpIvAbcAu5JcAD4C\nvBKgqj4FnABuA5aAZ4H3bNSwkqS1mxr8qjoy5foC/npmE0mSNoTvtJWkJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYM\nviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMG\nX5KaMPiS1ITBl6QmDL4kNWHwJamJQcFPciDJ40mWktx1kevfkOSBJI8keTTJbbMfVZK0HlODn2QH\ncAy4FdgPHEmyf9Wyvwfur6obgMPAP896UEnS+gx5hH8TsFRV56rqOeA+4NCqNQW8Znz5tcBPZjei\nJGkWhgR/N3B+4vjC+NykjwK3J7kAnADef7EbSnI0yWKSxeXl5TWMK0laq1m9aHsE+FxV7QFuA76Q\n5Nduu6qOV9V8Vc3Pzc3N6K4lSUMMCf4TwN6J4z3jc5PuAO4HqKrvAq8Cds1iQEnSbAwJ/mlgX5Jr\nk1zB6EXZhVVrfgy8DSDJmxgF3+dsJOkyMjX4VfU8cCdwEniM0V/jnElyT5KD42UfBN6b5HvAl4B3\nV1Vt1NCSpJdu55BFVXWC0Yuxk+funrh8FnjLbEeTJM2S77SVpCYMviQ1YfAlqQmDL0lNGHxJasLg\nS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHw\nJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4\nktSEwZekJgy+JDUxKPhJDiR5PMlSkrsuseadSc4mOZPki7MdU5K0XjunLUiyAzgG/BlwATidZKGq\nzk6s2Qf8HfCWqno6yes2amBJ0toMeYR/E7BUVeeq6jngPuDQqjXvBY5V1dMAVfXkbMeUJK3XkODv\nBs5PHF8Yn5t0HXBdku8kOZXkwMVuKMnRJItJFpeXl9c2sSRpTWb1ou1OYB9wC3AE+EySq1cvqqrj\nVTVfVfNzc3MzumtJ0hBDgv8EsHfieM/43KQLwEJV/aqqfgj8gNEPAEnSZWJI8E8D+5Jcm+QK4DCw\nsGrN1xg9uifJLkZP8Zyb4ZySpHWaGvyqeh64EzgJPAbcX1VnktyT5OB42UngqSRngQeAD1XVUxs1\ntCTppUtVbckdz8/P1+Li4pbctyS9XCV5uKrm1/K1vtNWkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lN\nGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6Qm\nDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1IT\nBl+SmjD4ktSEwZekJgYFP8mBJI8nWUpy14use0eSSjI/uxElSbMwNfhJdgDHgFuB/cCRJPsvsu4q\n4G+Ah2Y9pCRp/YY8wr8JWKqqc1X1HHAfcOgi6z4GfBz4xQznkyTNyJDg7wbOTxxfGJ/7P0luBPZW\n1ddf7IaSHE2ymGRxeXn5JQ8rSVq7db9om+QVwCeAD05bW1XHq2q+qubn5ubWe9eSpJdgSPCfAPZO\nHO8Zn3vBVcCbgW8n+RFwM7DgC7eSdHkZEvzTwL4k1ya5AjgMLLxwZVU9U1W7quqaqroGOAUcrKrF\nDZlYkrQmU4NfVc8DdwIngceA+6vqTJJ7khzc6AElSbOxc8iiqjoBnFh17u5LrL1l/WNJkmbNd9pK\nUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAl\nqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS\n1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpoYFPwkB5I8nmQpyV0Xuf4DSc4meTTJN5O8\ncfajSpLWY2rwk+wAjgG3AvuBI0n2r1r2CDBfVX8AfBX4h1kPKklanyGP8G8ClqrqXFU9B9wHHJpc\nUFUPVNWz48NTwJ7ZjilJWq8hwd8NnJ84vjA+dyl3AN+42BVJjiZZTLK4vLw8fEpJ0rrN9EXbJLcD\n88C9F7u+qo5X1XxVzc/Nzc3yriVJU+wcsOYJYO/E8Z7xuf8nyduBDwNvrapfzmY8SdKsDHmEfxrY\nl+TaJFcAh4GFyQVJbgA+DRysqidnP6Ykab2mBr+qngfuBE4CjwH3V9WZJPckOThedi/wauArSf49\nycIlbk6StEWGPKVDVZ0ATqw6d/fE5bfPeC5J0oz5TltJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh\n8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow\n+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaGBT8JAeSPJ5kKcldF7n+N5J8eXz9Q0mumfWgkqT1mRr8JDuAY8CtwH7gSJL9\nq5bdATxdVb8L/BPw8VkPKklanyGP8G8ClqrqXFU9B9wHHFq15hDwb+PLXwXeliSzG1OStF47B6zZ\nDZyfOL4A/NGl1lTV80meAX4b+NnkoiRHgaPjw18m+f5aht6GdrFqrxpzL1a4FyvcixW/t9YvHBL8\nmamq48BxgCSLVTW/mfd/uXIvVrgXK9yLFe7FiiSLa/3aIU/pPAHsnTjeMz530TVJdgKvBZ5a61CS\npNkbEvzTwL4k1ya5AjgMLKxaswD85fjyXwDfqqqa3ZiSpPWa+pTO+Dn5O4GTwA7gs1V1Jsk9wGJV\nLQD/CnwhyRLwc0Y/FKY5vo65txv3YoV7scK9WOFerFjzXsQH4pLUg++0laQmDL4kNbHhwfdjGVYM\n2IsPJDmb5NEk30zyxq2YczNM24uJde9IUkm27Z/kDdmLJO8cf2+cSfLFzZ5xswz4P/KGJA8keWT8\n/+S2rZhzoyX5bJInL/VepYx8crxPjya5cdANV9WG/WP0Iu9/AL8DXAF8D9i/as1fAZ8aXz4MfHkj\nZ9qqfwP34k+B3xxffl/nvRivuwp4EDgFzG/13Fv4fbEPeAT4rfHx67Z67i3ci+PA+8aX9wM/2uq5\nN2gv/gS4Efj+Ja6/DfgGEOBm4KEht7vRj/D9WIYVU/eiqh6oqmfHh6cYvedhOxryfQHwMUafy/SL\nzRxukw3Zi/cCx6rqaYCqenKTZ9wsQ/aigNeML78W+MkmzrdpqupBRn/xeCmHgM/XyCng6iSvn3a7\nGx38i30sw+5Lramq54EXPpZhuxmyF5PuYPQTfDuauhfjX1H3VtXXN3OwLTDk++I64Lok30lyKsmB\nTZtucw3Zi48Ctye5AJwA3r85o112XmpPgE3+aAUNk+R2YB5461bPshWSvAL4BPDuLR7lcrGT0dM6\ntzD6re/BJL9fVf+1pVNtjSPA56rqH5P8MaP3/7y5qv5nqwd7OdjoR/h+LMOKIXtBkrcDHwYOVtUv\nN2m2zTZtL64C3gx8O8mPGD1HubBNX7gd8n1xAVioql9V1Q+BHzD6AbDdDNmLO4D7Aarqu8CrGH2w\nWjeDerLaRgffj2VYMXUvktwAfJpR7Lfr87QwZS+q6pmq2lVV11TVNYxezzhYVWv+0KjL2JD/I19j\n9OieJLsYPcVzbjOH3CRD9uLHwNsAkryJUfCXN3XKy8MC8K7xX+vcDDxTVT+d9kUb+pRObdzHMrzs\nDNyLe4FXA18Zv27946o6uGVDb5CBe9HCwL04Cfx5krPAfwMfqqpt91vwwL34IPCZJH/L6AXcd2/H\nB4hJvsToh/yu8esVHwFeCVBVn2L0+sVtwBLwLPCeQbe7DfdKknQRvtNWkpow+JLUhMGXpCYMviQ1\nYfAlqQmDL0lNGHxJauJ/Acz2XLpusNoKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"hYUPnIC7AG8D","colab_type":"text"},"source":["**One-vs-rest SVM**\n","\n","Instead of using LinearSVC, we'll now use scikit-learn's SVC object, which is a non-linear \"kernel\" SVM (much more on what this means in Chapter 4!). Again, your task is to create a plot of the binary classifier for class 1 vs. rest."]},{"cell_type":"code","metadata":{"id":"Bdam352QAMDZ","colab_type":"code","colab":{}},"source":["# We'll use SVC instead of LinearSVC from now on\n","from sklearn.svm import SVC"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"42RVOx0cAagq","colab_type":"code","outputId":"8c1fdd0c-0664-4a80-878b-18548d17d8d2","executionInfo":{"status":"ok","timestamp":1557814961000,"user_tz":-480,"elapsed":1051,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":323}},"source":["# Create/plot the binary classifier (class 1 vs. rest)\n","svm_class_1 = SVC()\n","svm_class_1.fit(X_train, y_train==1)\n","plot_classifier(X_train, y_train==1, svm_class_1)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n","  \"avoid this warning.\", FutureWarning)\n"],"name":"stderr"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADU9JREFUeJzt3GGI5Hd9x/H3xztTaYym9FaQu9Ok\n9NJ42ELSJU0Raoq2XPLg7oFF7iBYJXhgGylVhBRLlPjIhloQrtWTilXQGH0gC57cA40ExAu3ITV4\nFyLb03oXhawxzZOgMe23D2bSna53mX92Z3cv+32/4GD+//ntzJcfe++dndmZVBWSpO3vFVs9gCRp\ncxh8SWrC4EtSEwZfkpow+JLUhMGXpCamBj/JZ5M8meT7l7g+ST6ZZCnJo0lunP2YkqT1GvII/3PA\ngRe5/lZg3/jfUeBf1j+WJGnWpga/qh4Efv4iSw4Bn6+RU8DVSV4/qwElSbOxcwa3sRs4P3F8YXzu\np6sXJjnK6LcArrzyyj+8/vrrZ3D3ktTHww8//LOqmlvL184i+INV1XHgOMD8/HwtLi5u5t1L0ste\nkv9c69fO4q90ngD2ThzvGZ+TJF1GZhH8BeBd47/WuRl4pqp+7ekcSdLWmvqUTpIvAbcAu5JcAD4C\nvBKgqj4FnABuA5aAZ4H3bNSwkqS1mxr8qjoy5foC/npmE0mSNoTvtJWkJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYM\nviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMG\nX5KaMPiS1ITBl6QmDL4kNWHwJamJQcFPciDJ40mWktx1kevfkOSBJI8keTTJbbMfVZK0HlODn2QH\ncAy4FdgPHEmyf9Wyvwfur6obgMPAP896UEnS+gx5hH8TsFRV56rqOeA+4NCqNQW8Znz5tcBPZjei\nJGkWhgR/N3B+4vjC+NykjwK3J7kAnADef7EbSnI0yWKSxeXl5TWMK0laq1m9aHsE+FxV7QFuA76Q\n5Nduu6qOV9V8Vc3Pzc3N6K4lSUMMCf4TwN6J4z3jc5PuAO4HqKrvAq8Cds1iQEnSbAwJ/mlgX5Jr\nk1zB6EXZhVVrfgy8DSDJmxgF3+dsJOkyMjX4VfU8cCdwEniM0V/jnElyT5KD42UfBN6b5HvAl4B3\nV1Vt1NCSpJdu55BFVXWC0Yuxk+funrh8FnjLbEeTJM2S77SVpCYMviQ1YfAlqQmDL0lNGHxJasLg\nS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHw\nJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4\nktSEwZekJgy+JDUxKPhJDiR5PMlSkrsuseadSc4mOZPki7MdU5K0XjunLUiyAzgG/BlwATidZKGq\nzk6s2Qf8HfCWqno6yes2amBJ0toMeYR/E7BUVeeq6jngPuDQqjXvBY5V1dMAVfXkbMeUJK3XkODv\nBs5PHF8Yn5t0HXBdku8kOZXkwMVuKMnRJItJFpeXl9c2sSRpTWb1ou1OYB9wC3AE+EySq1cvqqrj\nVTVfVfNzc3MzumtJ0hBDgv8EsHfieM/43KQLwEJV/aqqfgj8gNEPAEnSZWJI8E8D+5Jcm+QK4DCw\nsGrN1xg9uifJLkZP8Zyb4ZySpHWaGvyqeh64EzgJPAbcX1VnktyT5OB42UngqSRngQeAD1XVUxs1\ntCTppUtVbckdz8/P1+Li4pbctyS9XCV5uKrm1/K1vtNWkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lN\nGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6Qm\nDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1IT\nBl+SmjD4ktSEwZekJgYFP8mBJI8nWUpy14use0eSSjI/uxElSbMwNfhJdgDHgFuB/cCRJPsvsu4q\n4G+Ah2Y9pCRp/YY8wr8JWKqqc1X1HHAfcOgi6z4GfBz4xQznkyTNyJDg7wbOTxxfGJ/7P0luBPZW\n1ddf7IaSHE2ymGRxeXn5JQ8rSVq7db9om+QVwCeAD05bW1XHq2q+qubn5ubWe9eSpJdgSPCfAPZO\nHO8Zn3vBVcCbgW8n+RFwM7DgC7eSdHkZEvzTwL4k1ya5AjgMLLxwZVU9U1W7quqaqroGOAUcrKrF\nDZlYkrQmU4NfVc8DdwIngceA+6vqTJJ7khzc6AElSbOxc8iiqjoBnFh17u5LrL1l/WNJkmbNd9pK\nUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAl\nqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS\n1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpoYFPwkB5I8nmQpyV0Xuf4DSc4meTTJN5O8\ncfajSpLWY2rwk+wAjgG3AvuBI0n2r1r2CDBfVX8AfBX4h1kPKklanyGP8G8ClqrqXFU9B9wHHJpc\nUFUPVNWz48NTwJ7ZjilJWq8hwd8NnJ84vjA+dyl3AN+42BVJjiZZTLK4vLw8fEpJ0rrN9EXbJLcD\n88C9F7u+qo5X1XxVzc/Nzc3yriVJU+wcsOYJYO/E8Z7xuf8nyduBDwNvrapfzmY8SdKsDHmEfxrY\nl+TaJFcAh4GFyQVJbgA+DRysqidnP6Ykab2mBr+qngfuBE4CjwH3V9WZJPckOThedi/wauArSf49\nycIlbk6StEWGPKVDVZ0ATqw6d/fE5bfPeC5J0oz5TltJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh\n8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow\n+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaGBT8JAeSPJ5kKcldF7n+N5J8eXz9Q0mumfWgkqT1mRr8JDuAY8CtwH7gSJL9\nq5bdATxdVb8L/BPw8VkPKklanyGP8G8ClqrqXFU9B9wHHFq15hDwb+PLXwXeliSzG1OStF47B6zZ\nDZyfOL4A/NGl1lTV80meAX4b+NnkoiRHgaPjw18m+f5aht6GdrFqrxpzL1a4FyvcixW/t9YvHBL8\nmamq48BxgCSLVTW/mfd/uXIvVrgXK9yLFe7FiiSLa/3aIU/pPAHsnTjeMz530TVJdgKvBZ5a61CS\npNkbEvzTwL4k1ya5AjgMLKxaswD85fjyXwDfqqqa3ZiSpPWa+pTO+Dn5O4GTwA7gs1V1Jsk9wGJV\nLQD/CnwhyRLwc0Y/FKY5vo65txv3YoV7scK9WOFerFjzXsQH4pLUg++0laQmDL4kNbHhwfdjGVYM\n2IsPJDmb5NEk30zyxq2YczNM24uJde9IUkm27Z/kDdmLJO8cf2+cSfLFzZ5xswz4P/KGJA8keWT8\n/+S2rZhzoyX5bJInL/VepYx8crxPjya5cdANV9WG/WP0Iu9/AL8DXAF8D9i/as1fAZ8aXz4MfHkj\nZ9qqfwP34k+B3xxffl/nvRivuwp4EDgFzG/13Fv4fbEPeAT4rfHx67Z67i3ci+PA+8aX9wM/2uq5\nN2gv/gS4Efj+Ja6/DfgGEOBm4KEht7vRj/D9WIYVU/eiqh6oqmfHh6cYvedhOxryfQHwMUafy/SL\nzRxukw3Zi/cCx6rqaYCqenKTZ9wsQ/aigNeML78W+MkmzrdpqupBRn/xeCmHgM/XyCng6iSvn3a7\nGx38i30sw+5Lramq54EXPpZhuxmyF5PuYPQTfDuauhfjX1H3VtXXN3OwLTDk++I64Lok30lyKsmB\nTZtucw3Zi48Ctye5AJwA3r85o112XmpPgE3+aAUNk+R2YB5461bPshWSvAL4BPDuLR7lcrGT0dM6\ntzD6re/BJL9fVf+1pVNtjSPA56rqH5P8MaP3/7y5qv5nqwd7OdjoR/h+LMOKIXtBkrcDHwYOVtUv\nN2m2zTZtL64C3gx8O8mPGD1HubBNX7gd8n1xAVioql9V1Q+BHzD6AbDdDNmLO4D7Aarqu8CrGH2w\nWjeDerLaRgffj2VYMXUvktwAfJpR7Lfr87QwZS+q6pmq2lVV11TVNYxezzhYVWv+0KjL2JD/I19j\n9OieJLsYPcVzbjOH3CRD9uLHwNsAkryJUfCXN3XKy8MC8K7xX+vcDDxTVT+d9kUb+pRObdzHMrzs\nDNyLe4FXA18Zv27946o6uGVDb5CBe9HCwL04Cfx5krPAfwMfqqpt91vwwL34IPCZJH/L6AXcd2/H\nB4hJvsToh/yu8esVHwFeCVBVn2L0+sVtwBLwLPCeQbe7DfdKknQRvtNWkpow+JLUhMGXpCYMviQ1\nYfAlqQmDL0lNGHxJauJ/Acz2XLpusNoKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"wB50N7UNhNIt","colab_type":"text"},"source":["## 4. Support Vector Machines"]},{"cell_type":"markdown","metadata":{"id":"XWu-lLc2huAr","colab_type":"text"},"source":["#### Support vectors\n","\n","Support vectors are defined as training examples that influence the decision boundary. \n","\n","In this exercise, you'll observe this behaviour by removing non-support vectors from the training set."]},{"cell_type":"code","metadata":{"id":"N53DsRAuhPjM","colab_type":"code","outputId":"5e3598f7-99ec-4e0a-9a8a-6992ea58f808","executionInfo":{"status":"ok","timestamp":1557815484005,"user_tz":-480,"elapsed":861,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["wine = datasets.load_wine()\n","\n","X = wine.data[:, :2]\n","print(X.shape)\n","y = wine.target\n","print(y.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["(178, 2)\n","(178,)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"7DOlD-AsClAj","colab_type":"text"},"source":["\n","\n","*   Train a linear SVM on the whole data set.\n","\n","*   Create a new data set containing only the support vectors.\n","\n","*   Train a new linear SVM on the smaller data set.\n","\n","\n"]},{"cell_type":"code","metadata":{"id":"pS4VkZUhCiK_","colab_type":"code","outputId":"d480e59f-4cc9-4c5a-a544-c33313ba91e2","executionInfo":{"status":"ok","timestamp":1557816085033,"user_tz":-480,"elapsed":800,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":269}},"source":["# Train a linear SVM\n","svm = SVC(kernel=\"linear\")\n","svm.fit(X, y)\n","plot_classifier(X, y, svm, lims=(11,15,0,6))"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADU9JREFUeJzt3GGI5Hd9x/H3xztTaYym9FaQu9Ok\n9NJ42ELSJU0Raoq2XPLg7oFF7iBYJXhgGylVhBRLlPjIhloQrtWTilXQGH0gC57cA40ExAu3ITV4\nFyLb03oXhawxzZOgMe23D2bSna53mX92Z3cv+32/4GD+//ntzJcfe++dndmZVBWSpO3vFVs9gCRp\ncxh8SWrC4EtSEwZfkpow+JLUhMGXpCamBj/JZ5M8meT7l7g+ST6ZZCnJo0lunP2YkqT1GvII/3PA\ngRe5/lZg3/jfUeBf1j+WJGnWpga/qh4Efv4iSw4Bn6+RU8DVSV4/qwElSbOxcwa3sRs4P3F8YXzu\np6sXJjnK6LcArrzyyj+8/vrrZ3D3ktTHww8//LOqmlvL184i+INV1XHgOMD8/HwtLi5u5t1L0ste\nkv9c69fO4q90ngD2ThzvGZ+TJF1GZhH8BeBd47/WuRl4pqp+7ekcSdLWmvqUTpIvAbcAu5JcAD4C\nvBKgqj4FnABuA5aAZ4H3bNSwkqS1mxr8qjoy5foC/npmE0mSNoTvtJWkJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYM\nviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMG\nX5KaMPiS1ITBl6QmDL4kNWHwJamJQcFPciDJ40mWktx1kevfkOSBJI8keTTJbbMfVZK0HlODn2QH\ncAy4FdgPHEmyf9Wyvwfur6obgMPAP896UEnS+gx5hH8TsFRV56rqOeA+4NCqNQW8Znz5tcBPZjei\nJGkWhgR/N3B+4vjC+NykjwK3J7kAnADef7EbSnI0yWKSxeXl5TWMK0laq1m9aHsE+FxV7QFuA76Q\n5Nduu6qOV9V8Vc3Pzc3N6K4lSUMMCf4TwN6J4z3jc5PuAO4HqKrvAq8Cds1iQEnSbAwJ/mlgX5Jr\nk1zB6EXZhVVrfgy8DSDJmxgF3+dsJOkyMjX4VfU8cCdwEniM0V/jnElyT5KD42UfBN6b5HvAl4B3\nV1Vt1NCSpJdu55BFVXWC0Yuxk+funrh8FnjLbEeTJM2S77SVpCYMviQ1YfAlqQmDL0lNGHxJasLg\nS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHw\nJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4\nktSEwZekJgy+JDUxKPhJDiR5PMlSkrsuseadSc4mOZPki7MdU5K0XjunLUiyAzgG/BlwATidZKGq\nzk6s2Qf8HfCWqno6yes2amBJ0toMeYR/E7BUVeeq6jngPuDQqjXvBY5V1dMAVfXkbMeUJK3XkODv\nBs5PHF8Yn5t0HXBdku8kOZXkwMVuKMnRJItJFpeXl9c2sSRpTWb1ou1OYB9wC3AE+EySq1cvqqrj\nVTVfVfNzc3MzumtJ0hBDgv8EsHfieM/43KQLwEJV/aqqfgj8gNEPAEnSZWJI8E8D+5Jcm+QK4DCw\nsGrN1xg9uifJLkZP8Zyb4ZySpHWaGvyqeh64EzgJPAbcX1VnktyT5OB42UngqSRngQeAD1XVUxs1\ntCTppUtVbckdz8/P1+Li4pbctyS9XCV5uKrm1/K1vtNWkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lN\nGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6Qm\nDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1IT\nBl+SmjD4ktSEwZekJgYFP8mBJI8nWUpy14use0eSSjI/uxElSbMwNfhJdgDHgFuB/cCRJPsvsu4q\n4G+Ah2Y9pCRp/YY8wr8JWKqqc1X1HHAfcOgi6z4GfBz4xQznkyTNyJDg7wbOTxxfGJ/7P0luBPZW\n1ddf7IaSHE2ymGRxeXn5JQ8rSVq7db9om+QVwCeAD05bW1XHq2q+qubn5ubWe9eSpJdgSPCfAPZO\nHO8Zn3vBVcCbgW8n+RFwM7DgC7eSdHkZEvzTwL4k1ya5AjgMLLxwZVU9U1W7quqaqroGOAUcrKrF\nDZlYkrQmU4NfVc8DdwIngceA+6vqTJJ7khzc6AElSbOxc8iiqjoBnFh17u5LrL1l/WNJkmbNd9pK\nUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAl\nqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS\n1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpoYFPwkB5I8nmQpyV0Xuf4DSc4meTTJN5O8\ncfajSpLWY2rwk+wAjgG3AvuBI0n2r1r2CDBfVX8AfBX4h1kPKklanyGP8G8ClqrqXFU9B9wHHJpc\nUFUPVNWz48NTwJ7ZjilJWq8hwd8NnJ84vjA+dyl3AN+42BVJjiZZTLK4vLw8fEpJ0rrN9EXbJLcD\n88C9F7u+qo5X1XxVzc/Nzc3yriVJU+wcsOYJYO/E8Z7xuf8nyduBDwNvrapfzmY8SdKsDHmEfxrY\nl+TaJFcAh4GFyQVJbgA+DRysqidnP6Ykab2mBr+qngfuBE4CjwH3V9WZJPckOThedi/wauArSf49\nycIlbk6StEWGPKVDVZ0ATqw6d/fE5bfPeC5J0oz5TltJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh\n8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow\n+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaGBT8JAeSPJ5kKcldF7n+N5J8eXz9Q0mumfWgkqT1mRr8JDuAY8CtwH7gSJL9\nq5bdATxdVb8L/BPw8VkPKklanyGP8G8ClqrqXFU9B9wHHFq15hDwb+PLXwXeliSzG1OStF47B6zZ\nDZyfOL4A/NGl1lTV80meAX4b+NnkoiRHgaPjw18m+f5aht6GdrFqrxpzL1a4FyvcixW/t9YvHBL8\nmamq48BxgCSLVTW/mfd/uXIvVrgXK9yLFe7FiiSLa/3aIU/pPAHsnTjeMz530TVJdgKvBZ5a61CS\npNkbEvzTwL4k1ya5AjgMLKxaswD85fjyXwDfqqqa3ZiSpPWa+pTO+Dn5O4GTwA7gs1V1Jsk9wGJV\nLQD/CnwhyRLwc0Y/FKY5vo65txv3YoV7scK9WOFerFjzXsQH4pLUg++0laQmDL4kNbHhwfdjGVYM\n2IsPJDmb5NEk30zyxq2YczNM24uJde9IUkm27Z/kDdmLJO8cf2+cSfLFzZ5xswz4P/KGJA8keWT8\n/+S2rZhzoyX5bJInL/VepYx8crxPjya5cdANV9WG/WP0Iu9/AL8DXAF8D9i/as1fAZ8aXz4MfHkj\nZ9qqfwP34k+B3xxffl/nvRivuwp4EDgFzG/13Fv4fbEPeAT4rfHx67Z67i3ci+PA+8aX9wM/2uq5\nN2gv/gS4Efj+Ja6/DfgGEOBm4KEht7vRj/D9WIYVU/eiqh6oqmfHh6cYvedhOxryfQHwMUafy/SL\nzRxukw3Zi/cCx6rqaYCqenKTZ9wsQ/aigNeML78W+MkmzrdpqupBRn/xeCmHgM/XyCng6iSvn3a7\nGx38i30sw+5Lramq54EXPpZhuxmyF5PuYPQTfDuauhfjX1H3VtXXN3OwLTDk++I64Lok30lyKsmB\nTZtucw3Zi48Ctye5AJwA3r85o112XmpPgE3+aAUNk+R2YB5461bPshWSvAL4BPDuLR7lcrGT0dM6\ntzD6re/BJL9fVf+1pVNtjSPA56rqH5P8MaP3/7y5qv5nqwd7OdjoR/h+LMOKIXtBkrcDHwYOVtUv\nN2m2zTZtL64C3gx8O8mPGD1HubBNX7gd8n1xAVioql9V1Q+BHzD6AbDdDNmLO4D7Aarqu8CrGH2w\nWjeDerLaRgffj2VYMXUvktwAfJpR7Lfr87QwZS+q6pmq2lVV11TVNYxezzhYVWv+0KjL2JD/I19j\n9OieJLsYPcVzbjOH3CRD9uLHwNsAkryJUfCXN3XKy8MC8K7xX+vcDDxTVT+d9kUb+pRObdzHMrzs\nDNyLe4FXA18Zv27946o6uGVDb5CBe9HCwL04Cfx5krPAfwMfqqpt91vwwL34IPCZJH/L6AXcd2/H\nB4hJvsToh/yu8esVHwFeCVBVn2L0+sVtwBLwLPCeQbe7DfdKknQRvtNWkpow+JLUhMGXpCYMviQ1\nYfAlqQmDL0lNGHxJauJ/Acz2XLpusNoKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"qDNH-jt1Eyak","colab_type":"text"},"source":["*svm.support_ returns the indices of the support vectors of the model  svm. You can use this to directly index into X and y and return subsets containly only those examples. For example, X[svm.support_].*"]},{"cell_type":"code","metadata":{"id":"VsXu59YQEwrb","colab_type":"code","outputId":"0fcc68bd-214b-431f-b512-ae482700d971","executionInfo":{"status":"ok","timestamp":1557816104402,"user_tz":-480,"elapsed":747,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["# Make a new data set keeping only the support vectors\n","print(\"Number of original examples\", len(X))\n","print(\"Number of support vectors\", len(svm.support_))\n","X_small = X[svm.support_]\n","y_small = y[svm.support_]"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Number of original examples 178\n","Number of support vectors 81\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"t4AJe7LzCrf6","colab_type":"code","outputId":"37eea9e1-5cc8-4291-c774-70c42ce975e2","executionInfo":{"status":"ok","timestamp":1557816109097,"user_tz":-480,"elapsed":731,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":269}},"source":["# Train a new SVM using only the support vectors\n","svm_small = SVC(kernel=\"linear\")\n","svm_small.fit(X_small, y_small)\n","plot_classifier(X_small, y_small, svm_small, lims=(11,15,0,6))"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADU9JREFUeJzt3GGI5Hd9x/H3xztTaYym9FaQu9Ok\n9NJ42ELSJU0Raoq2XPLg7oFF7iBYJXhgGylVhBRLlPjIhloQrtWTilXQGH0gC57cA40ExAu3ITV4\nFyLb03oXhawxzZOgMe23D2bSna53mX92Z3cv+32/4GD+//ntzJcfe++dndmZVBWSpO3vFVs9gCRp\ncxh8SWrC4EtSEwZfkpow+JLUhMGXpCamBj/JZ5M8meT7l7g+ST6ZZCnJo0lunP2YkqT1GvII/3PA\ngRe5/lZg3/jfUeBf1j+WJGnWpga/qh4Efv4iSw4Bn6+RU8DVSV4/qwElSbOxcwa3sRs4P3F8YXzu\np6sXJjnK6LcArrzyyj+8/vrrZ3D3ktTHww8//LOqmlvL184i+INV1XHgOMD8/HwtLi5u5t1L0ste\nkv9c69fO4q90ngD2ThzvGZ+TJF1GZhH8BeBd47/WuRl4pqp+7ekcSdLWmvqUTpIvAbcAu5JcAD4C\nvBKgqj4FnABuA5aAZ4H3bNSwkqS1mxr8qjoy5foC/npmE0mSNoTvtJWkJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYM\nviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMG\nX5KaMPiS1ITBl6QmDL4kNWHwJamJQcFPciDJ40mWktx1kevfkOSBJI8keTTJbbMfVZK0HlODn2QH\ncAy4FdgPHEmyf9Wyvwfur6obgMPAP896UEnS+gx5hH8TsFRV56rqOeA+4NCqNQW8Znz5tcBPZjei\nJGkWhgR/N3B+4vjC+NykjwK3J7kAnADef7EbSnI0yWKSxeXl5TWMK0laq1m9aHsE+FxV7QFuA76Q\n5Nduu6qOV9V8Vc3Pzc3N6K4lSUMMCf4TwN6J4z3jc5PuAO4HqKrvAq8Cds1iQEnSbAwJ/mlgX5Jr\nk1zB6EXZhVVrfgy8DSDJmxgF3+dsJOkyMjX4VfU8cCdwEniM0V/jnElyT5KD42UfBN6b5HvAl4B3\nV1Vt1NCSpJdu55BFVXWC0Yuxk+funrh8FnjLbEeTJM2S77SVpCYMviQ1YfAlqQmDL0lNGHxJasLg\nS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHw\nJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4\nktSEwZekJgy+JDUxKPhJDiR5PMlSkrsuseadSc4mOZPki7MdU5K0XjunLUiyAzgG/BlwATidZKGq\nzk6s2Qf8HfCWqno6yes2amBJ0toMeYR/E7BUVeeq6jngPuDQqjXvBY5V1dMAVfXkbMeUJK3XkODv\nBs5PHF8Yn5t0HXBdku8kOZXkwMVuKMnRJItJFpeXl9c2sSRpTWb1ou1OYB9wC3AE+EySq1cvqqrj\nVTVfVfNzc3MzumtJ0hBDgv8EsHfieM/43KQLwEJV/aqqfgj8gNEPAEnSZWJI8E8D+5Jcm+QK4DCw\nsGrN1xg9uifJLkZP8Zyb4ZySpHWaGvyqeh64EzgJPAbcX1VnktyT5OB42UngqSRngQeAD1XVUxs1\ntCTppUtVbckdz8/P1+Li4pbctyS9XCV5uKrm1/K1vtNWkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lN\nGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6Qm\nDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1IT\nBl+SmjD4ktSEwZekJgYFP8mBJI8nWUpy14use0eSSjI/uxElSbMwNfhJdgDHgFuB/cCRJPsvsu4q\n4G+Ah2Y9pCRp/YY8wr8JWKqqc1X1HHAfcOgi6z4GfBz4xQznkyTNyJDg7wbOTxxfGJ/7P0luBPZW\n1ddf7IaSHE2ymGRxeXn5JQ8rSVq7db9om+QVwCeAD05bW1XHq2q+qubn5ubWe9eSpJdgSPCfAPZO\nHO8Zn3vBVcCbgW8n+RFwM7DgC7eSdHkZEvzTwL4k1ya5AjgMLLxwZVU9U1W7quqaqroGOAUcrKrF\nDZlYkrQmU4NfVc8DdwIngceA+6vqTJJ7khzc6AElSbOxc8iiqjoBnFh17u5LrL1l/WNJkmbNd9pK\nUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAl\nqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS\n1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpoYFPwkB5I8nmQpyV0Xuf4DSc4meTTJN5O8\ncfajSpLWY2rwk+wAjgG3AvuBI0n2r1r2CDBfVX8AfBX4h1kPKklanyGP8G8ClqrqXFU9B9wHHJpc\nUFUPVNWz48NTwJ7ZjilJWq8hwd8NnJ84vjA+dyl3AN+42BVJjiZZTLK4vLw8fEpJ0rrN9EXbJLcD\n88C9F7u+qo5X1XxVzc/Nzc3yriVJU+wcsOYJYO/E8Z7xuf8nyduBDwNvrapfzmY8SdKsDHmEfxrY\nl+TaJFcAh4GFyQVJbgA+DRysqidnP6Ykab2mBr+qngfuBE4CjwH3V9WZJPckOThedi/wauArSf49\nycIlbk6StEWGPKVDVZ0ATqw6d/fE5bfPeC5J0oz5TltJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh\n8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow\n+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0Y\nfElqwuBLUhMGX5KaGBT8JAeSPJ5kKcldF7n+N5J8eXz9Q0mumfWgkqT1mRr8JDuAY8CtwH7gSJL9\nq5bdATxdVb8L/BPw8VkPKklanyGP8G8ClqrqXFU9B9wHHFq15hDwb+PLXwXeliSzG1OStF47B6zZ\nDZyfOL4A/NGl1lTV80meAX4b+NnkoiRHgaPjw18m+f5aht6GdrFqrxpzL1a4FyvcixW/t9YvHBL8\nmamq48BxgCSLVTW/mfd/uXIvVrgXK9yLFe7FiiSLa/3aIU/pPAHsnTjeMz530TVJdgKvBZ5a61CS\npNkbEvzTwL4k1ya5AjgMLKxaswD85fjyXwDfqqqa3ZiSpPWa+pTO+Dn5O4GTwA7gs1V1Jsk9wGJV\nLQD/CnwhyRLwc0Y/FKY5vo65txv3YoV7scK9WOFerFjzXsQH4pLUg++0laQmDL4kNbHhwfdjGVYM\n2IsPJDmb5NEk30zyxq2YczNM24uJde9IUkm27Z/kDdmLJO8cf2+cSfLFzZ5xswz4P/KGJA8keWT8\n/+S2rZhzoyX5bJInL/VepYx8crxPjya5cdANV9WG/WP0Iu9/AL8DXAF8D9i/as1fAZ8aXz4MfHkj\nZ9qqfwP34k+B3xxffl/nvRivuwp4EDgFzG/13Fv4fbEPeAT4rfHx67Z67i3ci+PA+8aX9wM/2uq5\nN2gv/gS4Efj+Ja6/DfgGEOBm4KEht7vRj/D9WIYVU/eiqh6oqmfHh6cYvedhOxryfQHwMUafy/SL\nzRxukw3Zi/cCx6rqaYCqenKTZ9wsQ/aigNeML78W+MkmzrdpqupBRn/xeCmHgM/XyCng6iSvn3a7\nGx38i30sw+5Lramq54EXPpZhuxmyF5PuYPQTfDuauhfjX1H3VtXXN3OwLTDk++I64Lok30lyKsmB\nTZtucw3Zi48Ctye5AJwA3r85o112XmpPgE3+aAUNk+R2YB5461bPshWSvAL4BPDuLR7lcrGT0dM6\ntzD6re/BJL9fVf+1pVNtjSPA56rqH5P8MaP3/7y5qv5nqwd7OdjoR/h+LMOKIXtBkrcDHwYOVtUv\nN2m2zTZtL64C3gx8O8mPGD1HubBNX7gd8n1xAVioql9V1Q+BHzD6AbDdDNmLO4D7Aarqu8CrGH2w\nWjeDerLaRgffj2VYMXUvktwAfJpR7Lfr87QwZS+q6pmq2lVV11TVNYxezzhYVWv+0KjL2JD/I19j\n9OieJLsYPcVzbjOH3CRD9uLHwNsAkryJUfCXN3XKy8MC8K7xX+vcDDxTVT+d9kUb+pRObdzHMrzs\nDNyLe4FXA18Zv27946o6uGVDb5CBe9HCwL04Cfx5krPAfwMfqqpt91vwwL34IPCZJH/L6AXcd2/H\nB4hJvsToh/yu8esVHwFeCVBVn2L0+sVtwBLwLPCeQbe7DfdKknQRvtNWkpow+JLUhMGXpCYMviQ1\nYfAlqQmDL0lNGHxJauJ/Acz2XLpusNoKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"zWIVNBb_hv2u","colab_type":"text"},"source":["#### Kernel SVMs"]},{"cell_type":"markdown","metadata":{"id":"MEU-g8nQFAWO","colab_type":"text"},"source":["**GridSearchCV warm-up**\n","\n","Larger values of gamma are better for training accuracy, but cross-validation helped us find something different (and better)."]},{"cell_type":"code","metadata":{"id":"hjbqk8P4FSUh","colab_type":"code","colab":{}},"source":["from sklearn.model_selection import GridSearchCV"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"UXsiKdO_FNjV","colab_type":"code","outputId":"64dd8d9f-d475-4dcc-9720-b4c517d1d2be","executionInfo":{"status":"ok","timestamp":1557816240907,"user_tz":-480,"elapsed":729,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"source":["# Instantiate an RBF SVM\n","svm = SVC()\n","\n","# Instantiate the GridSearchCV object and run the search\n","parameters = {'gamma':[0.00001, 0.0001, 0.001, 0.01, 0.1]}\n","searcher = GridSearchCV(svm, parameters)\n","searcher.fit(X, y)\n","\n","# Report the best parameters\n","print(\"Best CV params\", searcher.best_params_)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Best CV params {'gamma': 0.1}\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n","  warnings.warn(CV_WARNING, FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n","  DeprecationWarning)\n"],"name":"stderr"}]},{"cell_type":"markdown","metadata":{"id":"L68WDoVZFfI-","colab_type":"text"},"source":["**Jointly tuning gamma and C with GridSearchCV**\n","\n","In this exercise you'll search for the best combination of C and gamma using `GridSearchCV`.\n","\n","As in the previous exercise, the 2-vs-not-2 digits dataset is already loaded, but this time it's split into the variables `X_train, y_train, X_test`, and `y_test`. Even though cross-validation already splits the training set into parts, it's often a good idea to hold out a separate test set to make sure the cross-validation results are sensible.\n","\n","\n","\n","*   Run `GridSearchCV` to find the best hyperparameters using the training set.\n","\n","*   Print the best values of the parameters.\n","\n","*   Print out the accuracy on the test set, which was not used during the cross-validation procedure.\n","\n"]},{"cell_type":"code","metadata":{"id":"FLTfyx_2FwAS","colab_type":"code","outputId":"481ab6a3-0e1d-4608-e443-bb1169d20b9a","executionInfo":{"status":"ok","timestamp":1557816545752,"user_tz":-480,"elapsed":753,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["X = digits.data\n","y = digits.target\n","\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=899, random_state=42)\n","\n","print(\"digits dataset train set:\", X_train.shape)\n","print(\"digits dataset test set:\", X_test.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["digits dataset train set: (898, 64)\n","digits dataset test set: (899, 64)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Zar4rwGBFn6H","colab_type":"code","outputId":"ca248150-e035-4ded-becf-1e5060c674e7","executionInfo":{"status":"ok","timestamp":1557816570866,"user_tz":-480,"elapsed":6971,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":258}},"source":["# Instantiate an RBF SVM\n","svm = SVC()\n","\n","# Instantiate the GridSearchCV object and run the search\n","parameters = {'C':[0.1, 1, 10], 'gamma':[0.00001, 0.0001, 0.001, 0.01, 0.1]}\n","searcher = GridSearchCV(svm, parameters)\n","searcher.fit(X_train, y_train)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n","  warnings.warn(CV_WARNING, FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n","  DeprecationWarning)\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["GridSearchCV(cv='warn', error_score='raise-deprecating',\n","       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n","  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n","  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n","  shrinking=True, tol=0.001, verbose=False),\n","       fit_params=None, iid='warn', n_jobs=None,\n","       param_grid={'C': [0.1, 1, 10], 'gamma': [1e-05, 0.0001, 0.001, 0.01, 0.1]},\n","       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n","       scoring=None, verbose=0)"]},"metadata":{"tags":[]},"execution_count":44}]},{"cell_type":"code","metadata":{"id":"3mJViltCGlvH","colab_type":"code","outputId":"fbe85dad-033c-492f-e00b-baba6fad761f","executionInfo":{"status":"ok","timestamp":1557816570867,"user_tz":-480,"elapsed":1618,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["# Report the best parameters and the corresponding score\n","print(\"Best CV params\", searcher.best_params_)\n","print(\"Best CV accuracy\", searcher.best_score_)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Best CV params {'C': 10, 'gamma': 0.001}\n","Best CV accuracy 0.9810690423162584\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Qyg6_XYhGnD4","colab_type":"code","outputId":"db61d46d-8fec-470d-9c34-e6242a3a4c1f","executionInfo":{"status":"ok","timestamp":1557816574617,"user_tz":-480,"elapsed":741,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["# Report the test accuracy using these best parameters\n","print(\"Test accuracy of best grid search hypers:\", searcher.score(X_test, y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Test accuracy of best grid search hypers: 0.9877641824249166\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"sXNjeDg5hxZe","colab_type":"text"},"source":["#### Comparing logistic regression and SVM ( and beyond)"]},{"cell_type":"markdown","metadata":{"id":"9wRRY-kzGp6w","colab_type":"text"},"source":["**An advantage of SVMs over logistic regression**\n","\n","Having a limited number of support vectors makes kernel SVMs computationally efficient.\n","\n","**An advantage of logistic regression over SVMs**\n","\n","It natually outputs meaningful probabilities."]},{"cell_type":"markdown","metadata":{"id":"ZltP7FDPHJyO","colab_type":"text"},"source":["**Using SGD Classifier**\n","\n","In this final coding exercise, you'll do a hyperparameter search over the regularization type, regularization strength, and the loss (logistic regression vs. linear SVM) using SGDClassifier()."]},{"cell_type":"code","metadata":{"id":"JOaD1fEBIRYH","colab_type":"code","colab":{}},"source":["from sklearn.linear_model import SGDClassifier"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"hfNuMYilhxT2","colab_type":"code","colab":{}},"source":["# We set random_state=0 for reproducibility \n","linear_classifier = SGDClassifier(random_state=0)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"WAk4kbhIIK7R","colab_type":"code","outputId":"191e326b-c9ef-4245-a069-0314da521362","executionInfo":{"status":"ok","timestamp":1557817032569,"user_tz":-480,"elapsed":6123,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":8452}},"source":["# Instantiate the GridSearchCV object and run the search\n","parameters = {'alpha':[0.00001, 0.0001, 0.001, 0.01, 0.1, 1], \n","             'loss':['hinge', 'log'], 'penalty':['l1','l2']}\n","searcher = GridSearchCV(linear_classifier, parameters, cv=10)\n","searcher.fit(X_train, y_train)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_search.py:841: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n","  DeprecationWarning)\n","/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n","  FutureWarning)\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["GridSearchCV(cv=10, error_score='raise-deprecating',\n","       estimator=SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n","       early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n","       l1_ratio=0.15, learning_rate='optimal', loss='hinge', max_iter=None,\n","       n_iter=None, n_iter_no_change=5, n_jobs=None, penalty='l2',\n","       power_t=0.5, random_state=0, shuffle=True, tol=None,\n","       validation_fraction=0.1, verbose=0, warm_start=False),\n","       fit_params=None, iid='warn', n_jobs=None,\n","       param_grid={'alpha': [1e-05, 0.0001, 0.001, 0.01, 0.1, 1], 'loss': ['hinge', 'log'], 'penalty': ['l1', 'l2']},\n","       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n","       scoring=None, verbose=0)"]},"metadata":{"tags":[]},"execution_count":50}]},{"cell_type":"code","metadata":{"id":"Eps_2eu8h1Wm","colab_type":"code","outputId":"690938d9-4450-487b-bcf4-e3d0eb9a9aad","executionInfo":{"status":"ok","timestamp":1557817034759,"user_tz":-480,"elapsed":745,"user":{"displayName":"Zhenna LU","photoUrl":"","userId":"13596549313510787594"}},"colab":{"base_uri":"https://localhost:8080/","height":68}},"source":["# Report the best parameters and the corresponding score\n","print(\"Best CV params\", searcher.best_params_)\n","print(\"Best CV accuracy\", searcher.best_score_)\n","print(\"Test accuracy of best grid search hypers:\", searcher.score(X_test, y_test))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Best CV params {'alpha': 0.0001, 'loss': 'hinge', 'penalty': 'l1'}\n","Best CV accuracy 0.9432071269487751\n","Test accuracy of best grid search hypers: 0.9276974416017798\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"6Hli4tG2Ii8S","colab_type":"text"},"source":["**An advantage of SGDClassifier: **\n","\n","it is very fast - this would have taken a lot longer with LogisticRegression() or LinearSVC()"]}]}