chapter06_optimization/adagrad-scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adagrad from scratch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-22T11:47:15.145355Z",
     "start_time": "2017-10-22T11:47:14.385508Z"
    }
   },
   "outputs": [],
   "source": [
    "from mxnet import ndarray as nd\n",
    "\n",
    "# Adagrad.\n",
    "def adagrad(params, sqrs, lr, batch_size):\n",
    "    eps_stable = 1e-7\n",
    "    for param, sqr in zip(params, sqrs):\n",
    "        g = param.grad / batch_size\n",
    "        sqr[:] += nd.square(g)\n",
    "        div = lr * g / nd.sqrt(sqr + eps_stable)\n",
    "        param[:] -= div\n",
    "\n",
    "import mxnet as mx\n",
    "from mxnet import autograd\n",
    "from mxnet import gluon\n",
    "import random\n",
    "\n",
    "mx.random.seed(1)\n",
    "random.seed(1)\n",
    "\n",
    "# Generate data.\n",
    "num_inputs = 2\n",
    "num_examples = 1000\n",
    "true_w = [2, -3.4]\n",
    "true_b = 4.2\n",
    "X = nd.random_normal(scale=1, shape=(num_examples, num_inputs))\n",
    "y = true_w[0] * X[:, 0] + true_w[1] * X[:, 1] + true_b\n",
    "y += .01 * nd.random_normal(scale=1, shape=y.shape)\n",
    "dataset = gluon.data.ArrayDataset(X, y)\n",
    "\n",
    "\n",
    "# Construct data iterator.\n",
    "def data_iter(batch_size):\n",
    "    idx = list(range(num_examples))\n",
    "    random.shuffle(idx)\n",
    "    for batch_i, i in enumerate(range(0, num_examples, batch_size)):\n",
    "        j = nd.array(idx[i: min(i + batch_size, num_examples)])\n",
    "        yield batch_i, X.take(j), y.take(j)\n",
    "\n",
    "# Initialize model parameters.\n",
    "def init_params():\n",
    "    w = nd.random_normal(scale=1, shape=(num_inputs, 1))\n",
    "    b = nd.zeros(shape=(1,))\n",
    "    params = [w, b]\n",
    "    sqrs = []\n",
    "    for param in params:\n",
    "        param.attach_grad()\n",
    "        # \n",
    "        sqrs.append(param.zeros_like())\n",
    "    return params, sqrs\n",
    "\n",
    "# Linear regression.\n",
    "def net(X, w, b):\n",
    "    return nd.dot(X, w) + b\n",
    "\n",
    "# Loss function.\n",
    "def square_loss(yhat, y):\n",
    "    return (yhat - y.reshape(yhat.shape)) ** 2 / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-22T11:47:15.424501Z",
     "start_time": "2017-10-22T11:47:15.146979Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "mpl.rcParams['figure.dpi']= 120\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def train(batch_size, lr, epochs, period):\n",
    "    assert period >= batch_size and period % batch_size == 0\n",
    "    [w, b], sqrs = init_params()\n",
    "    total_loss = [np.mean(square_loss(net(X, w, b), y).asnumpy())]\n",
    "\n",
    "    # Epoch starts from 1.\n",
    "    for epoch in range(1, epochs + 1):\n",
    "        for batch_i, data, label in data_iter(batch_size):\n",
    "            with autograd.record():\n",
    "                output = net(data, w, b)\n",
    "                loss = square_loss(output, label)\n",
    "            loss.backward()\n",
    "            adagrad([w, b], sqrs, lr, batch_size)\n",
    "            if batch_i * batch_size % period == 0:\n",
    "                total_loss.append(np.mean(square_loss(net(X, w, b), y).asnumpy()))\n",
    "        print(\"Batch size %d, Learning rate %f, Epoch %d, loss %.4e\" % \n",
    "              (batch_size, lr, epoch, total_loss[-1]))\n",
    "    print('w:', np.reshape(w.asnumpy(), (1, -1)), \n",
    "          'b:', b.asnumpy()[0], '\\n')\n",
    "    x_axis = np.linspace(0, epochs, len(total_loss), endpoint=True)\n",
    "    plt.semilogy(x_axis, total_loss)\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('loss')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-22T11:47:16.971192Z",
     "start_time": "2017-10-22T11:47:15.426169Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch size 10, Learning rate 0.900000, Epoch 1, loss 5.3231e-05\n",
      "Batch size 10, Learning rate 0.900000, Epoch 2, loss 4.9388e-05\n",
      "Batch size 10, Learning rate 0.900000, Epoch 3, loss 4.9256e-05\n",
      "w: [[ 1.99946415 -3.39996123]] b: 4.19967 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApUAAAG8CAYAAACPGl7EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAASdAAAEnQB3mYfeAAAIABJREFUeJzs3Xl8ZHWd7//Xp1LZ97WTTtIJva/QTUMj0CAioCLuODrO\nFXEcfVwd/TnjldkuP211HEdn03EG7zDX0Zmro3fABRUEXEBAoBuabnqj9z2ddPY9laSqvvePqqSr\nQ9LdSapyKlXv5+NRjz45darOJwvhne9qzjlERERERGbD53UBIiIiIjL/KVSKiIiIyKwpVIqIiIjI\nrClUioiIiMisKVSKiIiIyKwpVIqIiIjIrClUioiIiMisKVSKiIiIyKwpVIqIiIjIrClUioiIiMis\nKVSKiIiIyKwpVIqIiIjIrClUioiIiMis+b0uIJWZWTHwWuAUMOJxOSIiIiIXkgXUA79xzvVM98UK\nlQlgZluAz3pdh4iIiMgMvA34yXRfZM65BNQiAGa2AXjpxz/+MUuXLvW6HBEREZEpHT58mLe//e0A\nVzrndkz39WqpTKwRgKVLl7JmzRqvaxERERG5FDMasqeJOiIiIiIyawqVIiIiIjJrCpUiIiIiMmsK\nlSIiIiIyawqVIiIiIjJrCpUiIiIiMmsKlSIiIiIyawqVUzCzj5rZS2Y2Gt0hR0RERESmoFA5tWZg\nC/ADj+sQERERSXraUWcKzrkfA5jZ7V7XIiIiIpLsUqKl0swKzOxzZvaomXWamTOzu6e4NtvMvmxm\nZ8xsyMy2mtmtc1yyiIiISEpJiVAJVACfAVYBL1/k2m8DnwK+C3wSCAGPmNnmRBYoIiIikspSpfu7\nGahxzrWY2VXAC5NdZGabgPcC9zjn/jZ67j+APcBXgOvmqN64+cH20xxu62dJZQF3bqzzuhwRERFJ\nUynRUumcG3bOtVzCpXcSaZm8P+a1AeCbwLVmVp+gEhPmX546wjeePMKje5q9LkVERETSWEqEymnY\nABx0zvVOOL8t+u/6sRNm5jezHCAD8JtZjpllzFGdl6y+NA+AU51DHlciIiIi6SxVur8vVQ2RrvKJ\nxs4tjDl3L/DZmI//J/BBImMyX8XMqoDKCaeXzKjKaagvi4bKrkGcc5hZom8pIiIi8irpFipzgeFJ\nzgdingfAObeFyDqVl+pjnB9C50RdaaTkwZEQnQMjlBdkz3UJIiIiImkXKoeAyVJXTszzM3Uf8MCE\nc0uAh2bxnhc11lIJcKprSKFSREREPJFuYyqbiXSBTzR27sxM39g51+qc2+uc2wu8m8iM8oQGSjg3\nphLgVOdgom8nIiIiMql0C5U7geVmVjTh/DUxz8+ac26Lc86AtfF4vwupLxvvsedUl0KliIiIeCPd\nQuWDRGZzf2TshJllE5mAs9U5d8qrwmaqMCeTkrxMAE53aQa4iIiIeCNlxlSa2ceBEs7N4H6LmY2t\nBv5151yPc26rmT0AfCk6W/sw8AGgEfjQXNccL3WluXQPjqr7W0RERDyTMqES+DTQEPPxO6MPgO8A\nPdHju4AvAO8HSoFdwB3OuafiVYiZbWEOZ4LXl+axp6lXLZUiIiLimZTp/nbONTrnbIrH8ZjrAs65\ne5xzNc65HOfcJufcY3GuZc7GVMK5GeBNXUOEw24ubikiIiJynpQJlemsPrpW5UgozNm+wEWuFhER\nEYk/hcoEMLMtZuaILCuUcHWxa1Vqu0YRERHxgEJlAsx593fMWpUnOgbm4pYiIiIi51GoTAEN5Xnk\nZEa+lTtPdXtcjYiIiKQjhcoUkJnhY0N9KQAvHO/0uBoRERFJRwqVCTDXYyoBNl1WBsDBs/10DozM\n1W1FREREAIXKhJjrMZVwLlSCWitFRERk7ilUpogNi0rw+wyAF44pVIqIiMjcUqhMEXlZftbWFgOw\nTS2VIiIiMscUKlPIWBf43jO9DAwHPa5GRERE0olCZQJ4MVEHYEN9CQChsOPg2b65vLWIiIikOYXK\nBPBiog7AiurC8eMDLQqVIiIiMncUKlNIQ3k+Wf7It/SAWipFRERkDilUppAMn7GsqgBA3d8iIiIy\npxQqU8yKBZEucHV/i4iIyFxSqEwxY+Mq2/tHaO8f9rgaERERSRcKlQng1exvgOUxk3XUBS4iIiJz\nRaEyAbya/Q3nur9BXeAiIiIydxQqU0xNcQ6FOX5ALZUiIiIydxQqU4yZjbdWvni8C+ecxxWJiIhI\nOlCoTEGvW1kFwKHWfrYd0z7gIiIikngKlSnovVfXk5UR+db++3PHPa1FRERE0oNCZQoqL8jmjitq\nAHhs71mae4Y8rkhERERSnUJlirr7ukYAQmHHf2496W0xIiIikvIUKhPAy3Uqx1xeV8L6+hIAvrft\nJMPBkFeliIiISBpQqEwAL9epjDXWWtneP8Iju5u9LEVERERSnEJlCnvTumoqCrIA+PdnT3hcjYiI\niKQyhcoUlu3P4H2bFgGw81Q3x9sHPK5IREREUpVCZYp77Yqq8eNDrf0eViIiIiKpTKEyxS2uyB8/\nPtqmUCkiIiKJoVCZ4krzsyjNywTgmLq/RUREJEEUKtPAZdHWyqMKlSIiIpIgCpVp4LKKAkAtlSIi\nIpI4CpVpYHFlpKWyrW+YvsCox9WIiIhIKlKoTAOXxUzWOd4+6GElIiIikqoUKhMgGbZpjBUbKo+2\nawa4iIiIxJ9CZQIkyzaNYxrLz4VKjasUERGRRFCoTAO5WRnUluQCCpUiIiKSGAqVaWKsC/yIFkAX\nERGRBFCoTBMrqwsB2N/cR/9w0ONqREREJNUoVKaJ65aWAxAMO7Yd6/C4GhEREUk1CpVpYtNl5fh9\nBsAzhxQqRUREJL4UKtNEQbaf9fUlADx7pN3jakRERCTVKFSmkeuWVgCwv6WPtr5hj6sRERGRVKJQ\nmUY2R0MlqLVSRERE4kuhchJmVmlmD5vZgJkdMLPXe11TPKyvLyEvKwOAJw+0eVyNiIiIpBKFysn9\nM9ACVAL3AP9lZmXeljR7WX4fNy6rBOBXr5xlNBT2uCIRERFJFQqVE5hZAfB24LPOuUHn3E+A3cDb\nvK0sPt6wdgEAvYEgW492elyNiIiIpIp5HyrNrMDMPmdmj5pZp5k5M7t7imuzzezLZnbGzIbMbKuZ\n3TrhsmVAv3PudMy53cCaBH0Kc+rmFQvGlxZ6bG+Lx9WIiIhIqpj3oRKoAD4DrAJevsi13wY+BXwX\n+CQQAh4xs80x1xQAvRNe1xs9P+8V52Vy7ZLIQuiP72shHHYeVyQiIiKpIBVCZTNQ45xrIDL+cVJm\ntgl4L/Dnzrl7nHP3AzcDJ4CvxFzaDxRNeHlR9HxKuG1NNQBne4fZe2ZifhYRERGZvnkfKp1zw865\nS+nHvZNIy+T9Ma8NAN8ErjWz+ujpQ0CBmdXGvHYtsDdOJXvuhpilhXae7vawEhEREUkVfq8LmEMb\ngIPOuYlNc9ui/64HTjnn+s3sIeBzZvYJ4PXA5cBDF3pzM6siMls81pLZlx1/DeV5FOb46QsE2XO6\nx+tyREREJAWkU6isIdJVPtHYuYUx5z4G/DvQAZwG3uOcu9hU6Y8Bn51tkXPBzFi7sJjnjnawu0mh\nUkRERGYvnUJlLjDZ3oSBmOcBcM61AbdP8/3vAx6YcG4JF2nh9Mq6ukioPHi2j8BoiJzMDK9LEhER\nkXksnULlEJA9yfmcmOdnzDnXCrTGnjOz2bxlQq2tLQYgGHbsb+ljfX2JxxWJiIjIfDbvJ+pMQzOR\nLvCJxs6dideNzGyLmTlgT7zeM97WRUMloC5wERERmbV0CpU7geVmNnG5oGtino8L59wW55wRmTWe\nlBrK8ijMjjRUa7KOiIiIzFY6hcoHgQzgI2MnzCwb+CCw1Tl3yqvCvODzGWtqI/laLZUiIiIyWykx\nptLMPg6UcG4G91vMrC56/HXnXI9zbquZPQB8Kbr8z2HgA0Aj8KE417OFeTAT/PK6Ep4/2qnJOiIi\nIjJrKREqgU8DDTEfvzP6APgOMNYUdxfwBeD9QCmwC7jDOfdUPItxzm0BtpjZGpJ4XGXsZJ0DLX1c\nock6IiIiMkMpESqdc42XeF2AyFaOU27nmE4mTtZRqBQREZGZSqcxlTLBeZN1NK5SREREZkGhMgHm\nw5JCcP5knV2aAS4iIiKzoFCZAPNhSaExY13gY5N1RERERGZCoTLNTZysIyIiIjITCpVpTjvriIiI\nSDwoVCbAfBlTCdBYnk+BJuuIiIjILClUJsB8GlPp8xlrFmpnHREREZkdhUrh8rpzk3WGg5qsIyIi\nItOnUCnjk3VGQ5qsIyIiIjOjUCmarCMiIiKzplCZAPNpog6cP1lntxZBFxERkRlQqEyA+TRRBzRZ\nR0RERGZPoVKA83fW0WQdERERmS6FSgFgXZ0m64iIiMjMKVQKAGsWnpuss79ZoVJERESmR6FSAGgs\nzyMzwwA41KpQKSIiItOjUJkA8232N4A/w8fiigIADrX2e1yNiIiIzDcKlQkw32Z/j1m6IBoqzypU\nioiIyPQoVMq4ZVWRUNnUPcTAcNDjakRERGQ+UaiUccuqCsePj7YNeFiJiIiIzDcKlTJuWbT7GzRZ\nR0RERKZHoVLGNZbnk+EbmwGucZUiIiJy6RQqZVyW30djeR6gyToiIiIyPQqVcp6xcZWH1f0tIiIi\n06BQmQDzcZ3KMWPjKk90DtIzOOpxNSIiIjJfKFQmwHxdpxLg6sYyAJyD5462e1yNiIiIzBcKlXKe\nTZeVkeWP/Fg8dUihUkRERC6NQqWcJyczg2sui7RWPnWwDeecxxWJiIjIfKBQKa9yw7IKAE53DXGi\nY9DjakRERGQ+UKiUV7lhWeX48dOH1QUuIiIiF6dQKa+ysrqQysJsAJ5VqBQREZFLoFApr2JmbKgv\nAeCV5l6PqxEREZH5QKFSJrWypgiIrFc5MBz0uBoRERFJdgqVMqlV1ZGddZyDg2e1u46IiIhcmEJl\nAsznHXXGrIiGSoD9LQqVIiIicmEKlQkwn3fUGdNQnk9OZuTHY7/GVYqIiMhFKFTKpDJ8xooFkdZK\ntVSKiIjIxShUypRWVkcm6+xv6dPOOiIiInJBCpUypbFxlT1Do7T0BjyuRkRERJKZQqVMaWXNuck6\nWq9SRERELkShUqa0pqYYn0WOf3u4w9tiREREJKkpVMqUivMyuaqxDIDH97VoXKWIiIhMSaFSLugN\na6oBONU5xCvNmgUuIiIik1OolAu6bfWC8ePH97V4WImIiIgkM4XKKZjZR83sJTMbNbMtXtfjlfqy\nPFZF9wF/fO9Zj6sRERGRZKVQObVmYAvwA4/r8Nwb1kRaK/c192ofcBEREZmUQuUUnHM/ds79BOj2\nuhavvWND7fjxd54/4WElIiIikqySOlSaWYGZfc7MHjWzTjNzZnb3FNdmm9mXzeyMmQ2Z2VYzu3WO\nS05JDeX5vHZ5JQA/fKmJ/uGgxxWJiIhIsknqUAlUAJ8BVgEvX+TabwOfAr4LfBIIAY+Y2eZEFpgu\n7rq2AYD+4SA/2tHkcTUiIiKSbJI9VDYDNc65BuCeqS4ys03Ae4E/d87d45y7H7gZOAF8ZcK1z0Rb\nPCd7/GUCP5d57aYVVdSW5ALwXy+c8rgaERERSTZJHSqdc8POuUtZx+ZOIi2T98e8NgB8E7jWzOpj\nzm92ztkUj3vj/kmkiAyf8a6NdQDsburhVOegxxWJiIhIMknqUDkNG4CDzrmJG1Rvi/67frpvaGZ+\nM8sBMgC/meWYWcYs65zX3rS2evz40T1as1JERETOSZVQWUOkq3yisXMLZ/Ce9wJDwB8A/zN6/P6p\nLjazKjNbE/sAlszgvklrZXUhl1XkA/DInsm+3CIiIpKuUiVU5gLDk5wPxDw/Lc65LZN0j3/7Ai/5\nGLBnwuOh6d43mZnZeGvljpPdnOke8rgiERERSRapEiqHgOxJzufEPJ9o9wFrJzzeNgf3nVNvWlsz\nfvzIbrVWioiISESqhMpmIl3gE42dO5PoApxzrc65vbEP4Eii7zvX1tYW0VieB8CD20/jnPO4IhER\nEUkGqRIqdwLLzaxowvlrYp6fM2a2xcwckS7wlGJm3BmdBb6/pY+9ZybOjRIREZF0lCqh8kEis7Q/\nMnbCzLKBDwJbnXNzurDi2HhMIl3gKecdV9ZhFjl+cPtpb4sRERGRpOD3uoCLMbOPAyWcm8H9FjOr\nix5/3TnX45zbamYPAF8ysyrgMPABoBH40FzXnOpqS3K5fkkFzxxu56GdTfzF7avI8qfK3yciIiIy\nE0kfKoFPAw0xH78z+gD4DtATPb4L+AKRZX9KgV3AHc65p+aoznFmtgX47Fzfdy69a2Mtzxxup2tw\nlG3HOtm8rMLrkkRERMRDSd+85JxrvMAOOMdjrgtEt2iscc7lOOc2Oece86jmlO7+Brh5xQIyfJE+\n8CcOtHpcjYiIiHgt6UOlJKfivEw2LioF4In9CpUiIiLpTqFSZuymlZUAHG0f4Hj7gMfViIiIiJcU\nKhMglZcUinXzyqrxY3WBi4iIpDeFygRIhzGVACsWFFJTHNm06IkDbR5XIyIiIl5SqJQZMzNuiM76\n3nGyS7vriIiIpDGFSpmVtbXFAPQFgpzumost1kVERCQZKVQmQLqMqQRYs/Dczph7z/Rc4EoRERFJ\nZQqVCZAuYyoBVlYXjW/ZqH3ARURE0pdCpcxKfrafy8rzAYVKERGRdKZQKbO2OtoFvk+hUkREJG0p\nVMqsjYXKlt4AHf3DHlcjIiIiXlCoTIB0mqgDsGZh8fixusBFRETSk0JlAqTTRB2A1TXnZoDva1ao\nFBERSUcKlTJrlYXZlOVnAWgPcBERkTSlUClxUV+aC8CprkGPKxEREREvKFRKXNSV5gFoVx0REZE0\npVCZAOk2UQegrizSUnmme4hQWHuAi4iIpBuFygRIt4k6cK6lcjTkONsb8LgaERERmWsKlRIXY2Mq\nAU51alyliIhIuklYqLSIm83sTWZWmKj7SHIYa6kEjasUERFJR3EJlWb2RTN7IuZjAx4HfgE8DOw2\nsyXxuJckp7rYlkrNABcREUk78WqpfBewLebjO4HXA/cCdwAZwJY43UuSUE5mBlWF2YBaKkVERNJR\nvEJlLXA45uN3Avucc19yzj0CfAO4KU73kiQ11lqpMZUiIiLpJ16hMghkw3jX9+uBR2OePwtUxOle\nkqTqy7RWpYiISLqKV6jcA/w3MysFPgiUExlLOaYBaI/TvZJeOq5TCedaKpt7hhgNhT2uRkREROZS\nvELl54H1RILjvwK/dc49EfP8m4EX4nSvpJeO61QC1EdngIcdtPRorUoREZF04o/HmzjnfmFmVwK3\nAt3A/x17Ltp6+RTwUDzuJcmroTx//Hh3U894d7iIiIikvriESgDn3D5g3yTnu4A/jtd9JHltWFRC\nTqaPwGiY3xxo4/Z1NV6XJCIiInMkXutUFppZ/YRzC83s82b2ZTO7Oh73keSWk5nBtYvLAfjNwTac\n0x7gIiIi6SJeYyrvBx4Y+8DMioDniaxT+T+Ap83spjjdS5LYTSuqAGjpDbC/pc/jakRERGSuxCtU\nbgZ+FvPxfwMWAtcBpcAuIgFTUtxrl1eOH//mYJuHlYiIiMhcileorACaYj5+K/CMc+5551wf8B/A\nFXG6lySxxop8GssjE3SePNDqcTUiIiIyV+IVKruBagAzywVuILL395ggoKnAaeLGaGvlSye7GQ6G\nPK5GRERE5kK8QuWzwMfM7B3AV4Eczl9CaDnnt2RKCtvYUArASDDMvjO9HlcjIiIicyFeofJPgVHg\nB8CHgb93zu0FMLMM4N3Ab+J0L0lyVy4qHT/ecbLbw0pERERkrsRr8fPDZrYCWA30OOeOxzydB3wc\neDke95oPzGwL8Fmv6/BKXWkuFQVZtPePsOOUQqWIiEg6iFdLJc65UefcyxMCJc65PufcQxPPp7J0\n3aZxjJmxIdpaueNkl8fViIiIyFyIW6g0swwz+4CZ/ZeZbY0+/svM7op2gUsa2bCoBIDTXUO09mkf\ncBERkVQXrx11ioHfAv8G3AZkRh+3At8CnokuiC5pYkO9xlWKiIikk3i1VH4R2Ah8Aqh0zl3pnLsS\nqCIynvKq6DWSJi6vK8ZnkePtJ9QFLiIikuriFSrfAdznnLvPOTc6djI6zvIbwDeAd8XpXjIP5Gf7\nWVcX6QL/xb6z2gdcREQkxcUrVJYDBy7w/H6gLE73knnizeuqATjWPsBerVcpIiKS0uIVKg8T2Zpx\nKm8FjsTpXjJP3L6uZvz44d3NHlYiIiIiiRavUHkfcJuZPWJmt5lZY/TxBjN7mMiEnX+K071knqgr\nzRufBf6zXWfUBS4iIpLC4rX4+X1mVgX8GfCGmKcMGAE+Hx1bKWnmjssXsuNkN6c6h9h7ppe1tcVe\nlyQiIiIJEM/Fz7cAdcDvAX8RfbwPqHPOfS5e95kLZpZtZv9mZifNrNfMnjeza72uaz66ZVXV+PFL\nWghdREQkZc2opdLMFl3g6WejjzF5Y9c7507O5H4e8APHgc3AaeB3gJ+aWaNzrt/LwuabRWV5FOX4\n6Q0E2dPU43U5IiIikiAz7f4+DsxkgNy82FnHOTcAfD7m1PfN7O+BFcB2b6qan8yMtbXFPHukgz1N\nmgEuIiKSqmYaKn+fmYXKaTGzAuAe4BpgE1AKfNA59+1Jrs0mEgTfH71uF3Cvc+4XcahjGZElkQ7P\n9r3S0VioPHi2j+FgiGz/vPjbQkRERKZhRqFyslCXIBXAZ4CTwMvATRe49tvAncBXgUPA3cAjZvY6\n59wzMy3AzHKB7wBfcs6p/3YG1iyM7NAZDDsOtvSzrk6TdURERFJN3CbqJEgzUOOcayDSYjkpM9sE\nvBf4c+fcPc65+4GbgRPAVyZc+4yZuSkefznh2kzgASItlLHd4TINsTO+95xRLhcREUlFcVlSKFGc\nc8NAyyVceicQAu6PeW3AzL4J/JWZ1TvnTkXPb76Ue5uZD/g/RLr5P+C0yOKMXVaeT35WBgMjIU3W\nERERSVHJ3lJ5qTYAB51zE2eCbIv+u34G7/kvQA3wbudccDbFpTufz1gd7QLfo+0aRUREUlJSt1RO\nQw2RrvKJxs4tnM6bmVkD8AdAAGg3s7Gn3uSce3qK11QBlRNOL5nOfVPZmoXFvHC8i1eae+kLjFKY\nk+l1SSIiIhJHqdJSmQsMT3I+EPP8JXPOnXDOmXMu1zlXEPOYNFBGfQzYM+Hx0HTum8peuzySt0eC\nYR7dcykjGkRERGQ+SZVQOQRkT3I+J+b5RLsPWDvh8bY5uO+8cMOyCioKsgD44UtNHlcjIiIi8ZYq\nobKZSBf4RGPnziS6AOdcq3Nur3NuL/Bu1FJ5Hn+Gj7deUQvA88c6aOqei5wvIiIicyVVQuVOYLmZ\nFU04f03M83PGObfFOWdEWisl6p1XRkKlc/DjHWqtFBERSSWpEiofJLIF5EfGTkR32PkgsHVsOSHx\n1pqFRSyrKgDg4V2TzasSERGR+SrpZ3+b2ceBEs7N4H6LmdVFj7/unOtxzm01sweAL0VnYR8GPgA0\nAh+a65plcmbG7etq+NqvDrGvuZeTHYMsKs/zuiwRERGJg/nQUvlp4AvAR6MfvzP68ReI7PE95i4i\nWzS+H/hHIBO4wzn31NyVGmFmW8zMERlXKTHeuLZ6/PixvZoFLiIikiqSPlQ65xqjy/tM9jgec10g\nukVjjXMuxzm3yTn3mEc1a0zlFFZWF9IQbZ18VKFSREQkZSR9qJTUYmbjrZUvneyitTdwkVeIiIjI\nfKBQmQDq/r6wN66JhErn4OHdmrAjIiKSChQqE0Dd3xd2RV0J9WWRTY5+pKWFREREUoJCpcw5n894\nx4bIBP5dp3s4dLbP44pERERkthQqxRPvii6EDvADbdsoIiIy7ylUJoDGVF5cQ3k+VzVEVoT60Y7T\nhMPO44pERERkNhQqE0BjKi/N29ZH1rM/2zvModZ+j6sRERGR2VCoFM9cs7h8/Hj7iS4PKxEREZHZ\nUqgUzyytLKAoJ7JT6IsnOj2uRkRERGZDoVI84/MZG6PjKtVSKSIiMr8pVCaAJupcurFQeaJjkLa+\nYY+rERERkZlSqEwATdS5dBsbysaP1VopIiIyfylUiqfW15eQ4TMAtmtcpYiIyLylUCmeys3KYM3C\nIgBeVEuliIjIvKVQKZ4bG1e5p6mHwGjI42pERERkJhQqxXNXRcdVjoYcu5t6PK5GREREZkKhMgE0\n+3t6xloqAV48ri5wERGR+UihMgE0+3t6qotzqC3JBTRZR0REZL5SqJSkcFXjuUXQnXMeVyMiIiLT\npVApSWGsC7xrcJSj7QMeVyMiIiLTpVApSeH8cZXqAhcREZlvFColKaysLqIoxw/AM4c7PK5GRERE\npkuhUpJChs/YvKwCgGcOtREOa1yliIjIfKJQKUnjhmWVQGRc5d4zvR5XIyIiItOhUJkAWqdyZm6I\ntlQCPHWozcNKREREZLoUKhNA61TOTF1pHosr8wF46qBCpYiIyHyiUClJ5cZoF/j2E130BUY9rkZE\nREQulUKlJJWbVkRCZTDs+PX+Vo+rERERkUulUClJ5bolFeNLCz28q9njakRERORSKVRKUsny+7h1\ndTUATx5so3846HFFIiIicikUKiXp3HF5DQAjwTC/euWsx9WIiIjIpVColKRz/dJzXeCP7FYXuIiI\nyHygUClJJ8vv46YVVQC8eLwL57S7joiISLJTqJSktK62GICOgRHa+oY9rkZEREQuRqFSktLqhUXj\nx3ubtWWjiIhIslOoTABt0zh7q2rOhcpXFCpFRESSnkJlAmibxtkry89iQVE2AK8093lcjYiIiFyM\nQqUkrdXR1sp9Z3o8rkREREQuRqFSktZYF/ix9gECoyGPqxEREZELUaiUpDUWKsMODrSoC1xERCSZ\nKVRK0oqdAb5Pk3VERESSmkKlJK3G8nzysjIA2HW62+NqRERE5EIUKiVpZfiM9fUlAGw/0eVxNSIi\nInIhCpWS1K5qKAXg4Nl+egZHPa5GREREpqJQKUntymioBHjppForRUREkpVC5RTM7H4zazazXjPb\nbWZv8bpYxdplAAAgAElEQVSmdHRlQylmkeMXT3R6W4yIiIhMSaFyan8PNDrnioDfB75jZuUe15R2\ninIyWbGgENC4ShERkWSmUDkF59x+59zw2IdAFlDrYUlpa2O0C3znqW5GQ2GPqxEREZHJJHWoNLMC\nM/ucmT1qZp1m5szs7imuzTazL5vZGTMbMrOtZnbrLO9/n5kNAS8AvwZ2z+b9ZGauaoyEysBomJdP\naWkhERGRZJTUoRKoAD4DrAJevsi13wY+BXwX+CQQAh4xs80zvblz7mNAAXAL8Lhzzs30vWTmblhW\niS86rvLRPS3eFiMiIiKTSvZQ2QzUOOcagHumusjMNgHvBf7cOXePc+5+4GbgBPCVCdc+E23xnOzx\nlxPf2zkXcs79CrjFzG6P5ycnl6aiIJurG8sA+PmeFpTtRUREkk9Sh0rn3LBz7lKapu4k0jJ5f8xr\nA8A3gWvNrD7m/GbnnE3xuPcC9/ADS2f4qcgs3b6uBoCm7iF2ne7xuBoRERGZKKlD5TRsAA465yZu\nEL0t+u/66byZmRWb2fuiYzr9ZvZu4HXAUxd4TZWZrYl9AEumc1+Z2hvXVo8fP7Kn2cNKREREZDKp\nEipriHSVTzR2buE0388BHwZOAx3AnwHvc87tvMBrPgbsmfB4aJr3lSksKMoZnwX+qLrARUREko7f\n6wLiJBcYnuR8IOb5SxZt8XzdNGu4D3hgwrklKFjGzW2rF7D9RBcnOgY51j7A4soCr0sSERGRqFRp\nqRwCsic5nxPzfEI551qdc3tjH8CRRN83ndy0omr8+MkDbR5WIiIiIhOlSqhsJtIFPtHYuTNzWAtm\ntsXMHJEucImT5QsKqCmO/J3w5EGFShERkWSSKqFyJ7DczIomnL8m5vk545zb4pwzYO1c3jfVmRk3\nragE4PmjHQyNhDyuSERERMakSqh8EMgAPjJ2wsyygQ8CW51zp7wqTOLrtcsjXeAjwTDPH+3wuBoR\nEREZk/QTdczs40AJ52Zwv8XM6qLHX3fO9TjntprZA8CXzKwKOAx8AGgEPuRBzVuAz871fdPB9UvL\n8fuMYNjx9KF2Xrey6uIvEhERkYRL+lAJfBpoiPn4ndEHwHeAsZWw7wK+ALwfKAV2AXc456ZcWzJR\nnHNbgC3RtSo1rjKOCnMyWVVTxO6mHl5pnrgsqYiIiHgl6UOlc67xEq8LENnKccrtHCU1rKwuZHdT\nD/tbenHOYWZelyQiIpL2UmVMpaSRlTWR+Vhdg6O09k22PKmIiIjMNYXKBNCSQom1srpw/Fhd4CIi\nIslBoTIBtKRQYsWGyv0tfR5WIiIiImMUKmXeKS/IprIwsoHSAYVKERGRpKBQKfPSWGulur9FRESS\ng0JlAmhMZeKNhcojbf2MBMMeVyMiIiIKlQmgMZWJt7I6MgN8NOQ41KoucBEREa8pVMq8dEV9yfjx\nN5855mElIiIiAgqVMk8trSrgDWsWAPCjHU3sPdNzkVeIiIhIIilUyrz1p29cid9nOAd//fP9Xpcj\nIiKS1hQqE0ATdebG4soC3nN1PQBPH2qnZ2jU44pERETSl0JlAmiiztx5/aqq8eNdp7s9rERERCS9\nKVTKvHZF3bkJOztPKlSKiIh4RaFS5rXygmwayvMA2HFKoVJERMQrCpUy722ILi+081Q3zjmPqxER\nEUlPCpUy762PhsrOgRFOdg56XI2IiEh6UqhMAM3+nlsbFpWOH+9UF7iIiIgnFCoTQLO/59aqmiKy\n/JEf5e0nujyuRkREJD0pVMq8l+X3ceWiSBf4z/e0EAyFPa5IREQk/ShUSkp4x4ZaANr6hnn6cLvH\n1YiIiKQfhUpJCbevqyEnM/Lj/IPtpz2uRkREJP0oVEpKKMzJ5I1rqgF4fN9Zega1ZaOIiMhcUqiU\nlPGujXUAjATDPLq32eNqRERE0otCpaSM65ZUUJ6fBcDje896XI2IiEh6UahMAK1T6Y0Mn3HLqgUA\nPH24nYHhoMcViYiIpA+FygTQOpXeuW1NJFSOBMM8dbDN42pERETSh0KlpJTrl1aQl5UBwGN7Wzyu\nRkREJH0oVEpKycnM4KYVlQD8an+rFkIXERGZIwqVknJet6IKgL5AkFea+zyuRkREJD0oVErKubqx\nbPz4xROdHlYiIiKSPhQqJeU0lOdRUZANwIsnujyuRkREJD0oVErKMTOuaigF4MXjnTjnPK5IREQk\n9SlUSkq6qjESKs/2DnO6a8jjakRERFKfQqWkpI3RlkqA7eoCFxERSTiFygTQjjreW7OwmJzMyI+3\nJuuIiIgknkJlAmhHHe9l+X1sqI+0Vj59qF3jKkVERBJMoVJS1s0rI+tVnugY5FBrv8fViIiIpDaF\nSklZt65eMH78i31nPaxEREQk9SlUSspqrMhnWVUBAI8rVIqIiCSUQqWktLHWypdPdXO2N+BxNSIi\nIqlLoVJSWmwX+C9fUWuliIhIoihUSkq7oq6EysLIlo0aVykiIpI4CpWS0nw+45ZVkVngzx7uoH84\n6HFFIiIiqUmhUlLeWBf4SCjMUwfbPK5GREQkNSlUXoSZXWtmYTO71+taZGauW1JBXlYGoC5wERGR\nRFGovAAz8wH/ALzgdS0yczmZGdy4rBKAX+9vZTQU9rgiERGR1KNQeWEfAbYCr3hdiMzOWBd4z9Ao\nLxzXXuAiIiLxltSh0swKzOxzZvaomXWamTOzu6e4NtvMvmxmZ8xsyMy2mtmts7h3OfBHwGdn+h6S\nPG5eWUWGzwB1gYuIiCRCUodKoAL4DLAKePki134b+BTwXeCTQAh4xMw2z/DeXwS+6pzrnuHrJYmU\n5mdxdWMpEAmVzjmPKxIREUktyR4qm4Ea51wDcM9UF5nZJuC9wJ875+5xzt0P3AycAL4y4dpnoi2e\nkz3+MnrNBuBq4F8T9HmJB25dXQ3A6a4h9rf0eVyNiIhIaknqUOmcG3bOtVzCpXcSaZm8P+a1AeCb\nwLVmVh9zfrNzzqZ4jM3wfi2wAmgysxbgPcCfmtm34vW5ydy7LWZ3HXWBi4iIxFdSh8pp2AAcdM71\nTji/Lfrv+mm+3/3A0ujr1gM/Af4Z+OPZFCneqi/LY2V1IQDf33aS3sCoxxWJiIikjlQJlTVEuson\nGju3cDpv5pwbdM61jD2AIaD/QuMrzazKzNbEPoAl07mvJN5d1zYCcKYnwJaf7PW2GBERkRSSKqEy\nFxie5Hwg5vkZc87d7Zz7y4tc9jFgz4THQ7O5r8Tf726q58blkTUrf/hSE79UN7iIiEhcpEqoHAKy\nJzmfE/N8ot0HrJ3weNsc3Femwcz4mzsvpyjHD8DXnzismeAiIiJxkCqhsplIF/hEY+fOJLoA51yr\nc25v7AM4kuj7yvQtKMrh7usaAXj5VDfbjmkxdBERkdlKlVC5E1huZkUTzl8T8/ycMbMtZuaIdIFL\nErrrukay/ZEf/3956qjH1YiIiMx/qRIqHwQyiGyrCER22AE+CGx1zp2ay2Kcc1ucc0akC1ySUEVB\nNu/aWAdE9gM/0tbvcUUiIiLzm9/rAi7GzD4OlHBuBvdbzKwuevx151yPc26rmT0AfMnMqoDDwAeA\nRuBDc12zzA8f2nwZ/7n1JADf23qSe+9Y7XFFIiIi89d8aKn8NPAF4KPRj98Z/fgLQGnMdXcBXwXe\nD/wjkAnc4Zx7au5KjVD39/ywpLKAaxeXA/DgS6cJjIY8rkhERGT+SvpQ6ZxrvMAOOMdjrgtEt2is\ncc7lOOc2Oece86hmdX/PE++7ZhEA3YOjPLrnUjZvEhERkckkffe3SCLdtmYBZflZdA6M8DePHaBr\ncISf7WomK8PH1353PVWFORd/ExEREUn+lkqRRMr2Z4wvL9TUPcTnfrqP7Se6eO5oB3d9cxs9g9rK\nUURE5FIoVCaAxlTOL3/4uqX8xe0rx5cYyor+u7+ljzd+7Sn+4RcHtU+4iIjIRZh2E0mc6P7fe/bs\n2cOaNWu8Lkcu4lTnILtO93Dj8gr+7Ae7eXj3ue3kl1YV8K27r6auNBcz87BKERGRxNi7dy9r164F\nWBvdxGVaNKZSJKq+LI/6sjwAvvbe9Vy/tILvbj3B3jO9HG7t54avPAHAxoZSPvn6ZdywrEIBU0RE\nJErd3yKT8Gf4eN81i/jpxzfzkRsXn/fc9hNd3PVv2/jYd1+ipSfA04faONkxCEBgNERT91xsNZ+8\negOjnOwY1HhUEZE0o5bKBDCzLcBnva5DZs/nM/7i9lVsXlrBiye66Bkc4cHtpxkYCfHzPS38PLoM\nUU6mj8+/dS1f+9UhmrqHuOPyGv7HbStYUJTNSye6ae8f5k3rqsn2Z3j8GSVO/3CQ///He/jxziac\ngwyf8eEbFvPHty5L6c/7Yjr6h/m7XxykrjSXD22+bM6+FoHREH6f4c9IvbaD7sERdpzq5rol5Wn9\nsyXz05nuIY63D3DN4nIyfKnV26UxlQmkMZWpqb1/mD95cBe/3t86rdfdtKKS+99/FVl+H32BUU52\nDrKqugifh79URoJheoZGqSzMntX7HG3r5w/+40WOtg286rmG8jw+fMNi7txYhxn83xdO8eSBNg60\n9LGqpoi3XFFDaV4Wl1Xkjw8/iLf+4SB7m3roCwS5bmk5eVmX/vf02d4AOZkZFOdm0jM0yvH2AdbV\nFl/w+/bonmYe33uWd15Zx9d/fYitxzoBWFldyD+9bwNLqwqnfK1zjheOd7GsqoDS/CwAdp7q5htP\nHuZU5xBh53jzuhruuraR4rzMV71+JBjmiw/v4z+3nWQ05KgtyeX3N1/G8gUFPHekA58ZDeV5vG19\n7fiktPnkdNcgd37jOVp6A6yrLeZ/vX8jtSW5XpeVEP3DQb72y4M8vKuZu69v5CM3LvG6pLho7x/G\n7zNK8rK8LmXOtfQEeOPXnqJ7cJTL64r5q3esY21tsddljZvtmEqFygRSqExd4bDj+y+c4khbP2X5\nWfzd4wcIR/9TuqK+hJdPdU/6uqsaSqkrzeXxfWcZHAlx7eJy/u53rqCmOIef7WrmV6+c5V0b67hh\nWeUl1REYDfHzPc2MBh03rajkWPsAJzoGedO6agpzXh04Yj1/tINPfG8HXQMjfOmd63j3VfXT+hpA\nJAA9f7STj313O13R7u7NSyt4yxU1fOf5k+xu6hm/tiw/i+LcTI61vzp4jnn9yirecWUtVzWUUV0c\nWSM0HHbc9+RhvrftFKtqilhSmU9r3zBVhdnctKKKDYtKyMmcurXqP547zhcffoXhYBiABUXZfPq2\nFbx1/cJJW7mcc/QMjTISDPPNZ47xr08fJSczsvTU9184RefACOtqi/nk65exeVnF+L1HQ2F2ne7h\nP7ee5AcvnZ6yntqSXH76ic2U5b/6f6jOOf7kwV08sP00RTl+/vtNSzjaNsAPXjrNxF/VhdmR58Nh\nx/6WPq5bWs6ahcV88eF9vHC8a8r7j7ll1QLuf//GKcNx/3CQ1t4ADeX5cWlNaesb5vvbTjIadqyu\nKeLG5RVsPdbJvT/aQ31ZLn/2plU8vOsMzx3toC8QpCDbz7KqAqqKcijPz2JFdSGB0RBfefQAR2N+\nhsrys/j+R17D8gVTB/V4GA6GcI7x73dgNDTpz93prkEOtfZTXZTDZRX5F/zZjNXaG+C/XjzF04fa\nyfAZI8Ewh1r76Rk6N4zkr96xbnzDhsmMhsJ87ZeH+M3BNj75+mXcsnrBlNee6Bhgd1MPq2uKWFxZ\ncEk1XkwwFMZnRsg5fvRSE8c7Brh+aQXXXFaGP8PH8fYB/umJw/zwpdPkZ/n59u9fzcaGsvHXn+oc\n5Ccvn2FldSE3r6ya1ph15xyvNPcxMBJkRXUhRTmZOOfYeqyT3qFR1tQW8+MdTfzylbO09g5TW5rL\n59+2hpXVRePv8eyRdr76i0P0BkapL8vjg9c3ct2SCgBePN7Jt549zg1LK7iqsYzP/XQvZsZn37Ka\nJZUFdA6M8JOdTXQMjLC0qoCa4lzK8jOpK80j2++jdyhIblYGH/3Odn4V0yCR5ffxrbuv5vqlFXH4\nDsyeQmUSU6hMH4/uaeabzxzjzo11vOfqRew81c2eph7a+oZZtqCAf3vmGC+dnDxoZmX4qC3NPS9s\nvfnyGq5dXM7CkhwGR0L8eEcTXYOjvPWKhRTnZvLskXb6h4NsO9ZFe//wq95zQVE2H7lxCVl+H3mZ\nGWT6fRxu7ed05yBt/cO0949w8GwfoWgS9vuMv3n35ayrLcbMCIyG6OgfIcvvo74sj/ysDLYe6+Sf\nnzhMW98wq2qK8PuMI239HIlpnfzEzUv5o1uWk+EzRkNhvvP8Cf7308deNc60uiiHtbVFPH+0k/7h\n4KRfl9qSXNYsLKIvEOS5ox1Tfu0zM4zVC4vZUF/C8gWFFOdmcqZ7iLO9AVp6A/xsV/OkryvNy2Tz\nskoWluTQ0T9C58AIA8NBjrT1094/MuX9YmX5fdy4rIKlVYU8uP3UlK9bVVPE5qXl/OvTxwC4fmk5\nf/KGleRnZ9AXCNIXCBIMh9nb1Mvf/eLg5PfK8HH90nKaewLsb+m7aG2X1xVz0/JKfrqrefxnaywb\njv0B9Lub6rl+aQX52X6y/T4GhkO8eLyTx/edHX9NZWE2q2uKONk5SGVhNretXsDldSWU5WfRNRj5\nfAtz/BTmZJKflYGZMTwaomdodPzR3BPgq788eN7XpyjHT/9wcLyW6bq8rphdpyN/tNQU5/BP77uS\nbL+PUNhhFgl/Of4MMv3GwHCIU52DvHy6m1OdkZ+N5p4hBkdCkVbyynxuXFZBTXEuZ3sD/Hp/K0Oj\nIVbVFI2/rqlrCAc0lOXRFwjSMTBCQ3keVzWUcXVjKYHREE8daueJA63jfwBk+X28ZnE5iyvy8Zlx\nrL0fnxkrqgtZVVNEfVkep7sGeWR3pGU7OMUXw2eR75nPIruA3bq6mqGREAda+jjY2sfpriH8PmNo\nJMS+5l4gMvzks29ZzabLygiHYXAkSP9wkJdP9fDzPc3n/QwtX1DAxoYy1tcXs662hJxMH72BIKc6\nBxkcCWIYZuAzw+eDopxMFlcW0Ds0ypG2fo62DfDC8U62n+giP9tPYY6f013n/psvzPFzWUX++Pdr\n/Hy2n4++bgmjQcfupm6eONA2/jtpVU0R6+tLqCuN/B6oLs4hw4yW3gDPH+3gNwfbKMzO5I1rq3HO\n8fDu5vP+mFpZXYjPbPzrMZn8rAx+d9MigmHHsfYBfnOw7VVf9z++ZTlhB1//9aHx748Z49/jnEwf\niysKONzaz0goPOn3LtufwdBoiAyfjX9+K6sLOdTaTyjsyM3MYGNDKUOjId60tprrl1bQPxykZ3CU\nzoERTnYO4vMZl9cWc0V9yax7li5EoTKJKVTKmJ6hUT7z0B5ePN7F0GiIK+qKKc3L4oc7mjyta6z7\ncyT46l+G0+H3GV98x1rec/WrW1GCoTCP7Gnhm08f5XjHIB+9aQkf2nwZmRk+hkZCHGrtY2A4xM92\nneEHL50mMDp5LQuLcwg5R3v/CFWF2bT0Bl7VejeVioJstrx1NQPDQf728YO09b06iE9lbW0R7X0j\ntPQGKMz2c8cVC/nxjiaGLrBX/I3LK/n465byL785QvfQKP/wO+upL8vlE9/bMWXIjVVZmI3P4Gzv\nMFl+H5uXVvAXt69iaVUBzjl+vb+Vv/75fg619gOR/zkOjETq8fuM926q5943ryYnM4PRUJgn9rcy\nEgpzw7JKzOAd//zb8/4Y8FKW33fez98VdcU0lOfTMTDM0bYBugZHzvuZyPAZH7i2kXvfvIpv/OYI\nf/PYAS/KTpilVQUU52aS4TPqS/O4YVkFi8rzeN+/Pj/lfxvzzWsWl/H80U5P7r2sqoDLKvL5xStn\nJ/39kZuZwWsWl/Hc0Y6Lfr1jw+V0VBRk8Ys/fi2/PdLO//e9HdP+w+q3f3ZzwoZ8KFQmoYkTdRQq\nZSrPHengsb0t7D3Tw5qFxbz7qjr+8VeR7qvYX2gF2X6KczPHW/wKs/0sKM6hLD+L921axMKSXJ49\n0k5tSS6jIcdXHttP94TZ12ZQU5RDZWE2FQXZVBfn8HvXNHC8Y4A/+v7OSf/KnqgkL5NrF5dzuLWf\nDJ9RmpfFjcsruePymksaD+mcu2CXVmA0xN4zPbx4vIvtJ7o42j5AR/8w1y+t4ItvX0dxXibhsMPn\nMzoHRnj2SDs7Tnaz42QXe5p6z/sccjJ9ZJixtraYv333FeP1BUZDPLa3hYd2nuFASx8tvQEqCrKo\nKMgmLyuD2pJcVtUUkZeVQWVhNresWsDASIgnD7Sy6bIyaopz6Rka5bkjHTyxv5VH9jTTFwiytraI\nu17TyA3LIy1ekxkYDnL3t7ZdsHs6y+/jex9+DWsWRlrJ6svyJu1CDYbCPHWojarCHJYtKOAH25s4\n0z3Ee66uv+j34lj7AHd/axsnoqsWTHRVQynXL62gNC+TX+1v5Wy0G/xASx8nOyd/zcUU5vj53FvX\n8IY11Tx9qI3/8/wJ+gJBvvj2dfQNj/LDl5q4ZdUC3rBmwat+RjoHRtjf3Is/w8e62mJysyJfD+cc\nW36yl39/7sQl17GgKJvq4lyqi7IpyM6kY2CYHSe7z+tmLsnLpCw/i6NtA5TnZ7GxoZQV1ZHu9YNn\n+yjIzqSmOIc9Z3rYfryLvmhre2leJm9cW8Ob19XQMTDM80c72Xq0g7b+YUaCYRrL8wmGwxxrHzgv\nSGT7fdxx+UJ+7zWL2FBfMul/I3uaevjbxw/w5IHzW9PqSnOpL80jFHac7QvwuhVV3Lyyij/8z5fo\nC0zeC7CyupA3ra3hmsVlbDvWybNH2tl1uofBkan/ULoQn8FlFflcv7SC3qFRmrqHuHnlAt61sZan\nDrbz3JEODrf2sb6+hN97TQPLFxTy/W0n+eIjr4zXWFeay6bGMn73mkU8ezjye7GlN0DnwOSt/+tq\ni2ntC3C2N/IHYmG2n/e9ZhFXLiplT1MPvz3cPt67c2VD5NyKBZFudZ/PeHxvC5//2T5ao3+41RTn\nsHphEZ+6dTkN5fnsOt3Nh//jxfH3ry/L5cvvvJwnD7ax82Q3f3jzUjJ9xv9+5hhh51hUlsft62pY\nX1/CkbZ+OvpHaO+P/GE0MBKkuiiHjoERmnsC3H1dIxsbSgF4aGcTf/3z/ZTmZTEwEpz0v0mfgSMS\nYMvzs3jx3lsStpydQmUSU0ulzFQ47GjqHqK9f5jhYJjL64rJyvCx9VgnZnB1YxmZF5jVOxIM09oX\nIMvvY3A4xNBoiIbyvCknqJztjXSpdke7M7MyfJQXZDM0GuJ01yDDo2HyszN449oainMvPFbTKyPB\nMO39w3QNjlBdFAncl/KL92JB92KGg5Gu3sqC7Et6n3DYcbR9gINn+xgNhce7jjN8Ru/QKI3l+TRW\n5M+4nksVDIVp7gkwOBJiYCQ4/j2uKc6dsnvNOcfJzkGOtg3QPTRCWX6kRbUvEKR3aJTBkRCOSDAu\nzs0871FTnHPJ4wunwznHzlPdtPQEyPAZ/gwjHIZAMERgNMxIMPJ5VRRks3Zh8aSTm0JhF20xD5KZ\n4WN1TRH+DB/DwRBZGb4Lfl9DYceRtn5yMzMueXOEoZEQB8/20dwToLYkl8WV+eRnX9rksePtA7T1\nD5OV4aOxIn/K/x57A6PsbeqlYyAyKSYvy09+tp8FRdnUlb76j45Q2HG4tZ/9LZHu4pzMDOpL8yjJ\ny8QR+bkNO0coHOktONbeT1FOJkuqClg0xR8+FzMaCjMcDOMzpvzd1DkwwivNvXQPjhIMh6kszGZJ\nZQELinIIRbuti3L8VBRkx33S43AwxKGz/RTnZrKwJDfhM7XDYccLxztp6x+mODeTopxMSvIyqSnO\nJRAMsed0D52DI9xx+cKE1aBQmcQUKkVERGS+mG2onH/rSYiIiIhI0lGoFBEREZFZU6gUERERkVlT\nqEwAM9tiZg7Y43UtIiIiInNBoTIBnHNbnHMGrPW6FhEREZG5oFAp8v/au/tYyer6juPvT1ceallA\nXZ6kIAURaC2FFUSLUaw2LVYllUVNTOwWWxVqqrRNsTGhmFYiVSutaBSqrGZr47pKhVI1YnkMlocC\nKhCQFRFSHrqAPJRlgdJv/zjn0uEy92nOzN65O+9XMpk7Z36/md/5znfvfu85v/MbSZLUmUWlJEmS\nOrOolCRJUmcWlZIkSerMolKSJEmdWVSOgEsKSZKkSWNROQIuKSRJkiaNRaUkSZI6e85iD2Arty3A\nhg0bFnsckiRJs+qpV7YdpH+qanij0TMkeTPwjcUehyRJ0gIcU1XnLbSTReUIJdkJeA1wJ/DEiN5m\nP5rC9RjgxyN6j0ljTIfPmA6X8Rw+YzpcxnP4tkRMtwX2Ai6pqocW2tnT3yPUfiALrvQXIsnUjz+u\nqhtH+V6TwpgOnzEdLuM5fMZ0uIzn8G3BmF43aEcv1JEkSVJnFpWSJEnqzKJSkiRJnVlULn0bgQ+3\n9xoOYzp8xnS4jOfwGdPhMp7DN/Yx9epvSZIkdeaRSkmSJHVmUSlJkqTOLColSZLUmUWlJEmSOrOo\nlCRJUmcWlWMqyXZJTk9yV5LHklyZ5Dfn2XfPJOuSPJjk4STfSLLvqMc87gaNaZJTk1Sf2+YtMe5x\nlWSHJB9O8q0kD7QxWb2A/jsnOSvJxiSPJrkoycoRDnnsdYlpktUz5Gkl2X3EQx9LSQ5PcmaSG9sc\nu6P93fiSefY3R3t0iaf52V+SX0ny1SS3JdmU5L4klyZ50zz7j1WO+t3f42sNsAo4A7gVWA38a5LX\nVtXlM3VKsgNwEbATcBrwJHAScEmSQ6rq/hGPe5ytYYCY9jgB+O+ex08Ne4BLzArgFOAO4PvAUfPt\nmOTngAuAXwM+BtwHnAhcnORlVXXr0Ee7NAwc0x6nAD+Ztu3BbsNask4GjgS+CvwA2B14H3BtkldU\n1Q0zdTRH+xo4nj3Mz2d6EbAc+CJwF/Bc4FjgvCTvqaqzZuo4ljlaVd7G7Aa8HCjgz3q2bQ9sAK6Y\nozBnMIMAAAkbSURBVO+ft30P79l2IPA/wGmLvW9LNKantn1XLPZ+jNMN2A7Yvf35sDZGq+fZ961t\n+1U923YBfgZ8ebH3bYnGdHXb/rDF3o9xuQG/Dmw7bdv+wGZg7Rx9zdHhxtP8nH+clwHXAzfP0W7s\nctTT3+NpFc1RsKf/QqmqzcDngVcm2WuOvldX1dU9fW8GvkuTgJOqS0ynJMmOSTKiMS4pVfV4Vd0z\nYPdVwL3A13tebyOwDjgmyXZDGOKS0zGmT0uyPMmyYYxpKauqK6rqiWnbbgVuBA6ao7s5Ok3HeD7N\n/JxdVT0F3AnsPEfTsctRi8rxdCjwo6p6eNr2q9r7Q/p1ag+FHwxc0+fpq4D9kiwf2iiXloFiOs1t\nwEPAI0nWJtltmAOcMIcC11bV/07bfhXN6Z95zXlTXxcBDwObkpyXZP/FHtA4af8o3I3mVOFszNF5\nWEA8p5iffST5hSQrkuyX5CTgaJqDQbMZuxy1qBxPewB399k+te2FM/R7Ps3ps0H6bu0GjSk0pxLO\nBN5D85fhPwBvAy5LsuMwBzlBunwe6m8TzbzhPwJ+F/gb4HXAFfM8Ej8p3gHsCXxljnbm6PzMN57m\n5+w+QfOd3huAjwPn0sxXnc3Y5agX6oynnwce77N9c8/zM/VjwL5bu0FjSlX93bRNX0tyFfCPNJOi\nPzqUEU6WgT8P9VdV62hOe0355yTfBi4FPgS8d1EGNkaSHAh8GvgezYURszFH57CQeJqfczoDWE9T\nCL6VZl7ltnP0Gbsc9UjleHqM5ojjdNv3PD9TPwbsu7UbNKZ9VdWXgXuA13cc16Qa6ueh/qpZ1eBK\nzFPaZWsuoJnCsqqdtzYbc3QWA8TzWczP/1dVN1fVhVX1pap6I7ADcP4cc/jHLkctKsfT3TSHtaeb\n2nbXDP0eoPmrZZC+W7tBYzqbO2mmHGjhRvF5qL+Jz9MkOwHfpLnw4beraj75ZY7OYMB4zmTi83MG\n64HDmX1e5NjlqEXleLoeeEmf+XpH9Dz/LO1k3R/SLEUy3RHAbVX1yNBGubQMFNOZtH897kMzB0YL\ndz2wsr24rNcRNHOvfrTlh7TV2pcJztMk2wPn0/zn/MaqummeXc3RPjrEcyYTnZ+zmDp1vdMsbcYu\nRy0qx9N6mvkU757a0C4N8PvAlVV1Z7tt73ZOy/S+hyc5rKfvAcBv0CxYO6kGjmmSXfq83gk064F9\na2Qj3kok2SPJgUm26dm8nuaK0bf0tFsBHAecX1X95gmp1S+m/fI0yRuAlzGhedouW/MV4JXAcVX1\nvRnamaPz0CWe5md/SXbts20b4J00p69varctiRxNu1imxkySdTRXyH2S5mqw36NZwPt1VXVp2+Zi\n4DVVlZ5+y4HraFbo/zjNN+r8CU1BdUi7htVE6hDTTTS/SH9IMwH6VcDbab7x5Miq2rQFd2OsJHkf\nzSmwF9IU2l+nyT+AT1XVQ0nW0MT6l6rq9rbfMuBy4KU885sg9qZZuP+WLbgbY6VDTG9t211DM89t\nJXA8zSmyw6vq3i24G2MhyRnA+2mOrK2b/nxVrW3brcEcnVPHeJqffSQ5F9iR5oKl/6T5lqJ30Hxp\nyZ9W1d+27dawFHJ0MVZc9zb3jWai7cdo/sFtpll36remtbm4+Qif1fcXaY5KPgQ8QvML4MWLvU+L\nfRs0psDZNIv7Pgw8QfMVjx8Fli/2Pi32Dbid5hsd+t32adus6X3c0/d5NMsz3Qc82sZ+4r9tY9CY\nAn9N85/2g22e/hT4DLDbYu/TIsby4lliWT3tzNERx9P8nDGmbwe+Q3Ph55M010Z8B3jztHZLIkc9\nUilJkqTOnFMpSZKkziwqJUmS1JlFpSRJkjqzqJQkSVJnFpWSJEnqzKJSkiRJnVlUSpIkqTOLSkmS\nJHVmUSlJkqTOLColaQIlOTVJJVmx2GORtHWwqJQkSVJnFpWSJEnqzKJSkiRJnVlUStIIJdkzyReS\n3Jvk8SQ3Jjm+5/mj2rmNb0tyWpJ7kjya5Lwke/V5veOS/EeSx5Lcl2Rtkj37tDswybokG9u2tyT5\nSJ8h7pxkTZIHkzyU5Jwkzx1yGCRNgOcs9gAkaWuVZDfg34ECzgQ2AkcDn0+yY1Wd0dP8Q22704Fd\ngQ8AFyY5pKoea19vNXAOcDXwF8BuwPuBI5McWlUPtu0OBi4DngTOAm4H9gPe1L5Pr3XAT9rXWwn8\nAfBfwMnDioOkyWBRKUmj8xFgGfCrVXV/u+2zSf4JODXJ53raPh84qKoeAUhyLU3B94fA3yfZhqbg\nvAF4dVVtbttdDvwLcBLwl+1rfQoIsLKq7ph6gyQf7DPG66rqXT1tXgC8C4tKSQvk6W9JGoEkAY4F\nzm8frpi6Ad8GdqI5MjjlS1MFZWs9cDfwhvbxYTRHMD8zVVACVNUFwM3A77TvuwvwauALvQVl27b6\nDPWz0x5fBrwgyY4L2V9J8kilJI3GLsDOwLvbWz+7Aj9rf76194mqqiQbgH3aTS9q72/p8zo3A69q\nf963vb9hnuO8Y9rjqfE8D3h4nq8hSRaVkjQiU2eC1gJfnKHND4Bf3jLDmdFTM2zPFh2FpCXPolKS\nRmMj8AiwrKounKlRkqmicv9p2wO8mKbwBPhpe38A8G/TXuaAnudva+9fOtiwJWkwzqmUpBGoqqeA\nrwHHJnlWgdfOfez1ziTLex6vAvYAvtk+vobmquz3Jtmu53WOBg4CLmjfdyNwKXB8kr2nvadHHyWN\njEcqJWl0Pgi8FrgyydnATTRXea8EXt/+POUB4PIk59AsFfQBYANwNkBVPZnkZJolhS5pryCfWlLo\nduCTPa/1x8DlwLVJzqJZMmgfmot5DhnFjkqSRaUkjUhV3Zvk5cApwFuAE4H7gRt59pI9pwEH06wX\nuRz4LnBiVW3qeb01STbRFKunA48C5wInT61R2bb7fpJXAH8FnABsT3N6fN0o9lOSANJ/hQlJ0paQ\n5CjgIuC4qlq/yMORpIE5p1KSJEmdWVRKkiSpM4tKSZIkdeacSkmSJHXmkUpJkiR1ZlEpSZKkziwq\nJUmS1JlFpSRJkjqzqJQkSVJnFpWSJEnqzKJSkiRJnVlUSpIkqTOLSkmSJHVmUSlJkqTOLColSZLU\n2f8BvB8jQIAG+SsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113fa6b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train(batch_size=10, lr=0.9, epochs=3, period=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Next\n",
    "[Adagrad with Gluon](../chapter06_optimization/adagrad-gluon.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For whinges or inquiries, [open an issue on  GitHub.](https://github.com/zackchase/mxnet-the-straight-dope)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}