mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import keras\n",
    "from keras.datasets import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 2s 0us/step\n"
     ]
    }
   ],
   "source": [
    "# the data, shuffled and split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# simple data preprocessing to help with training\n",
    "st_dev = np.std(x_train)\n",
    "mean = np.mean(x_train)\n",
    "x_train = (x_train - mean) / st_dev\n",
    "x_test = (x_test - mean) / st_dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Flatten, MaxPooling2D, Conv2D, GlobalAveragePooling2D\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_new_model():\n",
    "    model = Sequential([\n",
    "        Flatten(input_shape=(28, 28)),\n",
    "        Dense(20),\n",
    "        BatchNormalization(),\n",
    "        Activation('relu'),\n",
    "        Dense(10),\n",
    "        BatchNormalization(),\n",
    "        Activation('softmax')\n",
    "    ])\n",
    "    model.compile(Adam(lr=1e-2), 'sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 20)                15700     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 20)                80        \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 20)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 10)                40        \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 16,030\n",
      "Trainable params: 15,970\n",
      "Non-trainable params: 60\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "get_new_model().summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Deltas for step sizes in weight space - roughly 1 std dev for each set of weights based on a single data point\n",
    "dense1_delta = 0.25\n",
    "dense1_bias_delta = 0.004\n",
    "dense4_delta = 0.4\n",
    "dense4_bias_delta = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_results, test_results = [], []\n",
    "train_results_with_deltas = []\n",
    "\n",
    "step_size = 0.2\n",
    "models_to_evaluate = 100\n",
    "measurements_per_model = 20\n",
    "\n",
    "for i in range(models_to_evaluate):\n",
    "    model = get_new_model()\n",
    "    h = model.fit(x_train, y_train, batch_size=256, shuffle=True, epochs=4,\n",
    "                  validation_data=(x_test, y_test), verbose=0)\n",
    "\n",
    "    train_results.append(model.evaluate(x_train, y_train, batch_size=60000, verbose=0))\n",
    "    test_results.append(model.evaluate(x_test, y_test, batch_size=10000, verbose=0))\n",
    "    \n",
    "    # Storing weights so that we can alter them multiple times\n",
    "    dense1 = model.layers[1].get_weights()[0]\n",
    "    dense1_bias = model.layers[1].get_weights()[1]\n",
    "    dense4 = model.layers[4].get_weights()[0]\n",
    "    dense4_bias = model.layers[4].get_weights()[1]\n",
    "\n",
    "    results_with_deltas = []\n",
    "    \n",
    "    for _ in range(measurements_per_model):\n",
    "        # Applying random deltas to the weights\n",
    "        model.layers[1].set_weights([\n",
    "            dense1 + np.random.normal(size=(784, 20)) * dense1_delta * step_size,\n",
    "            dense1_bias + np.random.normal(size=(20,)) * dense1_bias_delta * step_size\n",
    "        ])\n",
    "        model.layers[4].set_weights([\n",
    "            dense4 + np.random.normal(size=(20, 10)) * dense4_delta * step_size,\n",
    "            dense4_bias + np.random.normal(size=(10,)) * dense4_bias_delta * step_size\n",
    "        ])\n",
    "\n",
    "        # Evaluating the model on the train set again and storing results\n",
    "        results_with_deltas.append(model.evaluate(x_train, y_train, batch_size=60000, verbose=0))\n",
    "        \n",
    "    train_results_with_deltas.append(results_with_deltas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average accuracy: 0.9535101228952407\n"
     ]
    }
   ],
   "source": [
    "train_loss = np.array([measurement[0] for measurement in train_results])\n",
    "test_loss = np.array([measurement[0] for measurement in test_results])\n",
    "train_loss_with_deltas = np.array([[m[0] for m in measurements] for measurements in train_results_with_deltas])\n",
    "print(f'Average accuracy: {np.mean(np.array([measurement[1] for measurement in test_results]))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "generalization_deltas = test_loss - train_loss\n",
    "perturbation_deltas = train_loss_with_deltas - train_loss.reshape(models_to_evaluate, -1)\n",
    "perturbation_deltas_sum = np.sum(np.abs(perturbation_deltas), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f827cc3b128>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEWCAYAAABWn/G6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucHFWd9/HPN5MBhosEJLoQAmEVQdAVMMvlARFwlYuL\nIOgqyqKsyqKygo+iuKsSXVeCrquyosgqiwhyUSAbvAU1IApGSEy4hNvm4R6iREkikABD8nv+qNOk\nptOX6pnp6p6e7/v1mtd0V1V3naquOr86lzqliMDMzKwsEzqdADMzG18ceMzMrFQOPGZmVioHHjMz\nK5UDj5mZlcqBx8zMStW1gUfShZI+12D+k5L+snpZSa+RdE9Z6ewVkq6X9N70+p2Sru10mkabpMWS\nDkqvJem/Ja2QdHMJ6y7tuJS0i6RFkp6Q9KEy1tkpkkLSS9Pr8yR9qtNpGgskDUi6RtIqSd8ve/0T\ny15hNUnXA68C/iIinin6uYjYvM70XwG75L7/AeC9EfHzYaRtGnA/0B8Rz7X6+bEqIi4BLul0OkZb\nROyee3sA8Hpg+4h4qoR1j9pxWcDHgOsiYo82fHfXioiTO52GMeQtwIuBF3Yib+toiSdl7K8BAnhT\nJ9PSyyR1/AKjC+0IPDCcoDMG9ueOwOJ2riCVGCc0mzbejNZ+aecxJqmP7Bi5t2MX1BHRsT/g08CN\nwH8AP6yadyFwHvAz4Angl8COufkBvDS37OfS64OAR9Lr7wLrgDXAk2RXgj8C/qlqXbcBb66RvofS\nep5Mf/uRBetPAg8CjwEXAVs22MaPAcuAR4H3VqV7Y+Df03r+kLZ3IL8dwEfSepYBJ+a+t8hnPw78\nPu2HrYAfAsuBFen19rnvu57sChzg3cCvc+l/Mvc3CFyY5m0JfDulbSnwOaCvxj7YLv0GW+em7Qn8\nEegHXpp+31Vp2uV19uXzv21u2gPA36TXM4Ar0m/yBFnmO716WeA9wNPA2rRNn0nz3wcsAR4HZgPb\nVR1vHwT+F7g/N+0DadoTwL8CLwFuAv6c0rLRaB+Xad6b0vatTL/dy9P0uWm7nk7f/bIan303cF9K\n8/3AO3P77+LcctPSNk7MHSP/RnbOrkm/W61p26X993jan++rOq8/V+83Tb/RR9O2rwIuBzbJzT+d\n9efTP9AkH6D++fNC4Jr0O91Cduz+usF5vG/6XVcCtwIHVZ07I90vM4AfABenNL23RhqOAO5Mv9tS\n4KPV52uD/PEbwI+Bp1KaniU7l58kOx9eQnbs/InsHLwEmJT7vqnAVWT5x5+Ar+Xm/QNwF1m+Modc\nPl13f7YSKEb7L+38DwCvTjvhxVUH6BPAgWSZ7FfzO7fZAVcrY0rv/w74be79q9KO3KhG+qaRO/Fy\nO3kJ8JfA5unH+G6d7TuMLOPfHdg0HVT5dH85HYhbA1uQnQhn5bbjOeCzZJnzEcBqYKsWPnt22ncD\nZCfasSkdWwDfB2ZVnTwbBJ6q7ZlKdsIfnt5fDXwT2Ax4EXAz8I919sVchp5oXwTOS68vBf6FLKhv\nAhxQ5zuG/LbVvy/Zyft02ld9wFnAvDrLDtlG4BCyE26vtM/+E7ih6nj7WdrfA7lp/wO8IP3GzwC/\nSMfGlmSZxLvacFy+jCwDeX06Nj5GdkxWgtzzv2WNz25GlrHtkt5vC+ye23/NAs9DaVsnpnXXmnYD\n8PX0W+5BllkdUn2uNtgvN5Nl0luTZWgn586nPwCvSNvxPRrnA43On8vS36bAbsDD1Ak8wJT0WxxB\ndoy+Pr2fPIr7ZQZZHnh0WsdAjXQsA16TXm8F7FXvfK2xX1YB+7P+HKv+rV+atmtjYHJK61fSvD6y\nYPvltN+fP0eBo8iOvZen7fwkcFPTvL/ZAu36I6tjHwS2Se/vBj6cm38hcFnu/eZkV3JT6+zYooFn\nE7LIvHN6/+/A1+ukcRobBp5fAB/Ivd8lbcfEGp+/gBQMcj9upP8iyzxekpu/H+uvpg8iu1LKr/sx\nsiuvIp99ltyVYo207QGsyL2/ngaBhyx4LQA+nt6/mCyjHcgtcxxZ20Kt9b0XmJtei+xEPzC9vwg4\nn1wJrM53DPltq39fspPp57l5uwFr6iw7ZBvJSm5fqDreBoFpuePtkBon9/6598/vn/T+S6w/eYek\nnZEdl58Crsi9n0B2BXxQ9W9Z47ObkV21H0tV5kaxwPPZqs8MmUZ2cbIW2CI37SzWl5IvpHngOT73\n/gusv0C5AJiZm/cyGucD9c6fvvTb7pKbV7fEQ1Zz8N2qaXNYf1ExGvtlBrkLnTrpeAj4R+AFVdPf\nXZ32Gvvloka/dY11HQ0sTK/3IwuStfK4nwDvqToWV9Ok1NPJ+th3AddGxB/T+++laXkPV15ExJNk\nRdTtRrLSiHiarPh+fKp3PY6s6qOo7ciq2SoeJIv0L66z7MO59/nXk8muthZIWilpJfDTNL3iTzG0\nDnY1WYZY5LPL07YCIGlTSd+U9KCkP5Nd0UxK9b1FfBu4JyLOTu93JLuKW5ZLwzfJSj61XAnsJ2lb\nslLsOuBXad7HyILRzann2T8UTFMtv8+9Xg1sUrC+fMjvmo63P5Fd7VY8XP0hsivwijU13tfsBFOt\nxeOyOq3rUtqm1Fk+v56ngLcBJ5P9dj+StGuRNCa19kF+2nbA4xHxRG7ag0XSllP9G1b2YfX5lD8P\na2l0/kyk/rlZbUfgrZXjPB3rB5CVFht9vtX90igNkF0sHAE8KOmXkvZrsny9tGxA0oslXSZpacof\nLga2SbOnAg9G7fagHYGv5vbL42TncsPfuyONpJIGyKoW+iRVDrKNyTLCV0XErWna1NxnNicrej/a\n4uqixrTvkJ3UvwZWR8RvWvjso2Q7u2IHsiL9H2osuwzYPvd+au71H8kypt0jYmmd9ddT5LPVaf8I\nWelsn4j4vaQ9gIVkB0lDks4gu7p8TW7yw2Qlnm3qHJBDExOxInXRfhtZsfyyqFyaRfyerH0FSQcA\nP5d0Q0Qsqfqap8gCbiVdfQwNtiMx5HeVtBlZ9WR+/9Y6HoZrJMflo8ArK28kiezYKnQcRcQcYE46\nDz8H/BfZbztk/wJ/UTDd+WmPAltL2iKXye6QS1uRddSzjKHn0A4tfDZvOdk5uz1wb5o2tf7iPExW\n4nlfg2VGul/qfcf6mRG3AEdJ6gdOIWtDnMqG50XR3y3v82mZV0bE45KOBr6W5j0M7CBpYo1z/WHg\n3yLrCVtYp0o8R5MVO3cjq/LZgywz+hVwQm65IyQdIGkjsobbeRHR7Kqg2h/I6tyfl07odWRVIY1K\nO8vTcvnPXwp8WNJOKRh+nqwxvFbmewVwoqSXS9qUrIqkkoZ1ZCf8lyW9CEDSFEmHNtugYX52C7Jg\ntVLS1sCZzdaTvvdw4ENkjdxrcmlYBlwLfEnSCyRNkPQSSa9t8HXfI/t935JeV9bxVkmVAL2C7ARY\nV+Pz95KVYN6YTr5Pkl2wjIZLyX6rPSRtTPa7/jYiHhil7682kuPyCuCNkl6X9sNHyC4Cbmq20nRl\ne1QKrM+QNS5X9vUi4EBJO0jaEvhEi9tEOj9vAs6StImkvyJrvL44t44jJG2dMsjTWvj6K4B3S9ot\nnU+FjuEaaVxL1jY7I9UE7MrQfKfaxcCRkg6V1Je266DcMVtknc32S0OSNlJ2f92WETFI1k5X+d1u\nBXZPx26l/aZVW5AdC6skTSHrxFFxM1nQnylps5T+/dO884BPSNo9pXNLSW9ttrJOBZ53Af8dEQ9F\nxO8rf2QR9p25qpHvkR1cj5N1QDh+GOs6C/hkKgp+NDf9IrKrxro/fESsJvVMSZ/fl6ye+btkVVX3\nkzVm/1Odz/8EOAe4jqwBbl6aVblf6eOV6al4+3Ny93o00epnv0LWTvPHlI6fFlzP28hKFXcpu2n3\nSUnnpXknABuRNaKvIOuVs23trwGyzhA7A7/PlWoB/hr4raQn0zKnRsR91R+OiFVknVG+RXal+BRZ\nz6URi+x+mk+RVQkuI+vl8/bR+O46RnJc3kN2Lvwn2e95JHBkRDxbYL0TgP9LdgX+OPBa4P3pe39G\nVt13G1l71Q9b3KaK48jahx4l64ByZqy/X+m7ZBnlA2QXLpcX/dJ0Pn2FrKPKkvR/uE4h6wBS6fV5\nKevPy+r1PkzWiP7PZBejD5NlzK3mn432SxF/DzyQzveTgXem9N1L1oni52Q9LH/dYroAPkPWsWYV\nWQ/LqyozUqA+kqxt+iGyc+5tad7VZJ2YLkvpugM4vNnKlGo7xh1JJwAnRcQBJa7z5WQ/zMZFqqds\n/OnEcWkg6Wyym9ir25mtDcblzV6pmP4Bsp5U7V7XmyVtLGkrsiuDaxx0rJYyj8vxTtKukv5Kmb3J\nqr2u7nS6xotxF3hSO8hysjr27zVZfDT8I1k3zv9H1q71/hLWaWNMB47L8W4Lsuqkp8iq+75Edk+W\nlWDcVrWZmVlnjLsSj5mZdVa3D3bYkm222SamTZvW6WSYmY0ZCxYs+GNEjNb9cIX0VOCZNm0a8+fP\n73QyzMzGDEnNRoAYda5qMzOzUjnwmJlZqRx4zMysVA48ZmZWKgceMzMrlQOPmZmVyoHHzMxK5cBj\nZmalcuAxM7NSOfCYmVmpHHjMzKxUDjxmZlYqBx4zMyuVA4+ZmZXKgcfMzErlwGNmZqVy4DEzs1I5\n8JiZWakceMzMrFQOPGZmVioHHjMzK5UDj5mZlcqBx8zMSuXAY2ZmpXLgMTOzUjnwmJlZqRx4zMys\nVA48ZmZWKgceMzMrlQOPmZmVyoHHzMxK5cBjZmalcuAxM7NStTXwSDpM0j2Slkg6o8Z8STonzb9N\n0l65eRdIekzSHe1Mo5mZlattgUdSH3AucDiwG3CcpN2qFjsc2Dn9nQR8IzfvQuCwdqXPzMw6o50l\nnr2BJRFxX0Q8C1wGHFW1zFHARZGZB0yStC1ARNwAPN7G9JmZWQe0M/BMAR7OvX8kTWt1mYYknSRp\nvqT5y5cvH1ZCzcysPGO+c0FEnB8R0yNi+uTJkzudHDMza6KdgWcpMDX3fvs0rdVlzMysh7Qz8NwC\n7CxpJ0kbAW8HZlctMxs4IfVu2xdYFRHL2pgmMzPrsLYFnoh4DjgFmAPcBVwREYslnSzp5LTYj4H7\ngCXAfwEfqHxe0qXAb4BdJD0i6T3tSquZmZVHEdHpNIya6dOnx/z58zudDDOzMUPSgoiYXuY6x3zn\nAjMzG1sceMzMrFQOPGZmVioHHjMzK5UDj5mZlcqBx8zMSuXAY2ZmpXLgMTOzUjnwmJlZqRx4zMys\nVA48ZmZWKgceMzMrlQOPmZmVyoHHzMxK5cBjZmalcuAxM7NSOfCYmVmpmgYeZY6X9On0fgdJe7c/\naWZm1ouKlHi+DuwHHJfePwGc27YUmZlZT5tYYJl9ImIvSQsBImKFpI3anC7rErMWLuWLc+7h0ZVr\n2G7SAKcfugtH7zml08kyszGsSOAZlNQHBICkycC6tqbKusKshUv5xFW3s2ZwLQBLV67hE1fdDuDg\nY2bDVqSq7RzgauBFkv4N+DXw+bamyrrCF+fc83zQqVgzuJYvzrmnQykys17QtMQTEZdIWgC8DhBw\ndETc1faUWcc9unJNS9O7masMzbpH08Aj6dvAf0bEublpMyJiRjsTZp233aQBltYIMttNGuhAaoav\nVpXh6T+4lRmzF7NqzaADkVnJilS1HQp8R9IJuWlvalN6rIucfuguDPT3DZk20N/H6Yfu0qEUDU+t\nKsPBtcHKNYME69uuZi1c2pkEmo0zRQLPY8CBwFslnStpIlmVm/W4o/ecwlnHvJIpkwYQMGXSAGcd\n88oxVzIoUjXotiuz8hTp1aaIWAUcKWkGcD2wZTsTZd3j6D2njLlAU61elWG1pSvXMGvh0jG/vWbd\nrkiJZ3blRWrXORt4oE3pMRt1taoM63GVm1n7NQ08EXFm1ftrIuKQ9iXJbHRVVxlutWk//RNq1xa7\nys2s/epWtUn6dUQcIOkJ0s2jlVlARMQL2p46s1FSXWU4a+FSTrt8Uc1lm7UJuWu22cjUDTwRcUD6\nv0V5yRlfysrAnFHW3gdThtFd3KM5mI1c3ao2SVs3+ivy5ZIOk3SPpCWSzqgxX5LOSfNvk7RXmj5V\n0nWS7pS0WNKpw9/E7lTJwJauXDPiLr2zFi5l/5lz2emMH7H/zLlDvmM019MujdI/Wt9fax8cvOvk\nlruLezQHs5Fr1MazAJif/lf/zW/2xWl8t3OBw4HdgOMk7Va12OHAzunvJOAbafpzwEciYjdgX+CD\nNT47po1WBtYssHR7RllGYKy3D667e3nL3cV7aTQHs05pVNW20wi/e29gSUTcByDpMuAo4M7cMkcB\nF0VEAPMkTZK0bUQsA5aldDwh6S5gStVnx7TRysAaBZaj95zS0YyySBVfs/SPhkb7oNXu4r0ymoNZ\nJxV6AqmkrSTtLenAyl+Bj00BHs69fyRNa2kZSdOAPYHfFknrWFEvo2o1A2sWWEZrPa0qWpIpIzCO\n5j7oldEczDqpyBNI3wvcAMwBPpP+z2hvsp5f9+bAlcBpEfHnOsucJGm+pPnLly8vI1mjYrQysGaZ\naqcyyqJVfGUExtHcB70ymoNZJxUZueBU4K+BeRFxsKRdKfZYhKXA1Nz77dO0QstI6icLOpdExFX1\nVhIR5wPnA0yfPj3qLddtKhnVSHubnX7oLkN6WcHQTHW01tOqoiWZZukfDaO9D3phNAezTioSeJ6O\niKclIWnjiLhbUpFc4RZgZ0k7kQWTtwPvqFpmNnBKav/ZB1gVEcskCfg2cFdE/EfxzRlbRiMDK5Kp\nlp1Rzlq4lAkSa2PD64DqkkxZgdHBwqx7FAk8j0iaBMwCfiZpBfBgsw9FxHOSTiGrmusDLoiIxZJO\nTvPPA34MHAEsAVYDJ6aP7w/8PXC7pMpdfv8cET8uvmnjRzdlqpW2nVpBp15JppvSb2btp6iRQdRd\nWHot2QChP42IZ9uWqmGaPn16zJ/ftKe3tdH+M+fW7PXVJ/Glv3uVA4xZl5G0ICKml7nOhiWedC/O\n4ojYFSAifllKqmzMqte2sy7CQcfMgCaBJyLWppEHdoiIh8pKlI1NrbTtdJqHETLrnCJtPFsBiyXd\nDDxVmRgRfgppG4zVDHE4bTud0svjrY3V48fGlyKB51NtT4UB3ZchtpKJ1bpvB7K2nW67z6WM0RI6\noduOH7N6ijyP55dkD37rT69vAX7X5nSNS900rlqrY6iNpbadXh1vrZuOH7NGioxc8D7gB8A306Qp\nZF2rbZR1U4bYaibW6ggE7R6RupFODSPUbt10/Jg1UmSstg+S3VfzZ4CI+F/gRe1M1HjVTRliq5lY\nK8PSdPpRDb063lq942TLgf6Wv6uTFwbW+4oEnmfy9+xImsjQJ5LaKOmmDLHVINjKGGaNSlNlZHi9\nOt7a6YfuUvOR3k89+1xL+7HTFwbW+5reQCrpC8BK4ATgn4APAHdGxL+0P3mt6YUbSLulV1J1QzVk\nQXA0MuidzvhR3SuXgf6+tqxzvNjzs9eyYvXgBtOnTBrgxjMOKfQd9W4CbuU7bOzouhtIkzOA9wC3\nA/9INszNt9qZqPGsW4aPaecYavWeadMn1SwJzZi9uCuCcTPdcNGwskbQgdbaedxWZO3WNPBExDrg\nv9KfjSPtCoL1RqSu1R0bYOWaQVauyTLUZl2EO5X5d0tX5tF4UJ0fdmftVqRX2/6SfibpXkn3Sbpf\n0n1lJM7K18k2likFM7Z6ves62TZRtBdgu/fvaLQTdlNbo/WmIlVt3wY+DCwAal+SWk9o11V7vVJI\nre+sLgnVk6/2qXx/rav0sm4MLVI9VUapaDSqSDv1DKeKbqiytPYqEnhWRcRP2p4S67h23NHfSmZb\nneE16vZSqfap1QmiWhltE0Wqp8oaMWG0nvPUDZ1aPPpCbyrSnfo6SV+UtJ+kvSp/bU+Zla7oVXsr\nVUWt3oh69J5TuPGMQ7h/5hvrVr0Jnq/2qTdUT14ZbRNFqqfcaN+cR18YH4qUePZJ//Pd7QJwv8oe\n0+yqfThXoyPJbGt1QhDwzn13eH59zb6nrLaJItVTbrRvzsF5fCjSq+3gMhJinVevt1mj0kWzqqKR\nZLYjycwh67Qw2u0DjdofalVP5ZeftGk//RPE4Lr1lYhutB/KwXl8aBp4JL0Y+DywXUQcLmk3YL+I\n+HbbU2elapbRt3o1OmvhUlY/+9wG01vJbJu1NZx+6C6c/oNbGVw7tEWof4LaEnRaKfFVL79i9SD9\nfWLSQD+r1gy64byGZhc/1huKVLVdCPw3UBmp4F7gcrLebtZjGmX0rVyN1mv0nzTQz4w37T6qvbhm\nzF78/H0+FYProm5JbLi9plot8dVafnBtsNnGE1l05huKbN640+kedVaOIoFnm4i4QtInACLiOUnu\nVj0OtXI1Wq/Rf7ONJ456JrKqKuhU1CqJjaTXVKslPrdXDE+3jN5h7VMk8Dwl6YWkgUEl7Qusamuq\nrCvNf/Bxnn5ufTDZtH8Cn68zjlq9dpdHV64Z9fs0WimJFSm11Etfq+0Pkzbtrzlu2qRNWx8t2qyX\nFAk8/xeYDbxE0o3AZOAtbU2VdZ1Pzrqdi+c9NGTa6sF1zH/w8Q0y7HpBB7Ih+kf7Po1WSmLNSiGN\nSkSttj/UG3+3ybi8pfGNmtYpRZ5A+jvgtcD/IRskdPeIuK3dCbPucslvH2o4PT9cTT0CJNpyn8Ym\n/esP5UkD/XVHtK5XOgmyUZlnzF7csETUyuMU6lUB1pteJj/6wDqpSK+2Y6omvUzSKuD2iHisPcky\n6K4r0mZX70Vu5AxGZ/TkvFqdGJ55LiuJ1dp3tUotFY2CZiV9rbQ/dHPX4LJGUTCrpUhV23uA/YDr\n0vuDyMZt20nSZyPiu21K27g2kkbwTgSsooFjy4H+DXqgwdCbVFtJe70MNF8tuHTlGk7/wa3A0F5T\njQJNvfS1opu7BndLx4duuriy8hQZMmci8PKIODYijgV2I7t43Qf4eDsTN54Nd+iQdlWhbNpf+1Cp\nTC+aMUvUHVpmOGkvGjwG1wafuWYxsH5Yng2f1Vnf6haf4llZT7c+6bQbHrPu6r7xq0jgmRoRf8i9\nfyxNexzofGV1jxruFWm7xrr6/DF/RfVTlScomw61xyqrZeXqwbqZ8XDS3qfi4aO6h1m9THarTfuZ\nNDC059mK1YPDyhQr1XvbTRrg0ZVrnn+8d6d1w6MPPC7b+FWkqu16ST8Evp/eH5umbUb2SGxrg+G2\nD7SrCqXZjX1Fq7C2mzRQt51kOGlfO4IuYvWqws48cne+OOeeDaoE1wyu5TPXtPY01G4dbbkbbtTs\nluo+K1+RwPNB4BjggPT+IuDKiAjA47i1yXDbB9rZoN2sYb0yf/+Zc2umIT+qdC3DSfuUBmO11Vp/\ndXqhdub74csX1fyOFasHny85FQki3dyI3+kbNbu584W1V5FBQgO4Mv31vG5p7BzuFWm7G7SL7J96\nV6yVssn+M+fW/Pxw0n76obtw+vdvHTLwZj21lqiX+TYafDSvWRDxVX193dz5wtqrSIln3Oi2apHh\nXJG2swql6P6pl2lvtWnjm0eHk/aj95zCZ65ZXHOEgGqV5/vkb3Ttk1gbscFI1gfvOplL5j3U8GF0\nFY2CiK/q6+uG6j7rDEW33EY9CqZPnx7z588f9ufrVRFNmTTAjWf48UNF988nZ92+QaY90N/HxhMn\n1OxK3Wj/zlq4dMggoFtt2s+ZRw4dZHSnM37UNEAM9Pdx1jGvBOo/XrvRMgIG+iewenBdzfSffugu\nNTPQWvcZVdbT7Rlst5T+rb0kLYiI6c2XHD2FSjySNgJelt7eExGFerNJOgz4KtAHfCsiZlbNV5p/\nBLAaeHdE/E7SJsANwMYpjT+IiDOLrHMkerVaZLQykKJPKL1ywdIhgUDAsa+ewiXzao9+0OixCtXV\naCtWD25wT06zarF8aWb/mXPr3uia71FVvUwAG03sI9AGQeTgXSc3LQmOtQy820r/1luadqeWdBDw\nv8C5wNeBeyUdWOBzfekzh5Pd+3NcepZP3uHAzunvJOAbafozwCER8SpgD+CwNDhpW3XDvQ2jbTTv\nlSiyf2o1pgdw3d3LW96/X5xzT822m8G1MaTLba2uwQKO33cHHpj5Rm4845DCTyx9dOWaususWlO7\nK/h1dy+v2YHgtMsXsf/MuQDPP847n5Zu5q7O1k5FSjxfAt4QEfcASHoZcCnw6iaf2xtYEhH3pc9d\nBhwF3Jlb5ijgotSBYZ6kSZK2jYhlwJNpmf701/Y6wV5s7BxJr6rqktLBu07mygVLG+6fepn20pVr\nOH7fHZp+Pq9RkMjPa6VU0ax0VAmC9dplarW71esBV/mesVhS6NXSv3WHIjeQ9leCDkBE3EsWCJqZ\nAjyce/9ImlZoGUl9khaR3bD6s4j4ba2VSDpJ0nxJ85cvX14gWfV1853mwzXcDKRWSenKBUs59tVT\nGu6fRqXDIp/Pa/Rd1fMqoxE0K1U0utG1EgRbvbmyWYl4LJYUerH0b92jSIlnvqRvARen9+8Eht+C\nX1BErAX2kDQJuFrSKyLijhrLnQ+cD1nngpGut9P3Noy24faqqldSuu7u5Q07WjQahLPI56u/q1ZX\n6f4+DbsUWn2ja71ebZVlirTLNNrmirFWUujF0r91jyKB5/1kN5F+KL3/FVnbTTNLgam599unaS0t\nExErJV0HHAZsEHh6zWj3JBpuBjLcklIlrafVqX5qJQOufFezXm2tKnJx0coFSJFRG8ZaSWGsdoqw\nsaFpd2pJp0bEV5tNq/G5icC9wOvIgsktwDsiYnFumTcCp5D1atsHOCci9pY0GRhMQWcAuBY4OyJ+\n2GidI+1O3Wm1ut72TxCbbzKRlasHh33yDyeYjbRr+Xjtmj6Wu0+PFnfDHlu6tTv1u8i6POe9u8a0\nISLiOUmnAHPIulNfEBGLJZ2c5p8H/Jgs6Cwh6059Yvr4tsB3Us+4CcAVzYJOL6hVvTW4LloaoqWW\n4VQfjrSqZbxW1YykpNALGba7YVsRdUs8ko4D3kE2RtuvcrO2ANZFxOvan7zWjPUST5EbIaG8UsMn\nZ93Opb/2vq9kAAART0lEQVR9mLUR9Ekct89UPnf0Kwt/vhcy0rL0SklpvJZ0x7JuK/HcBCwDtiHr\nUl3xBOBHX7dB0fHBymiortwIWhn9eW0EVy5YyvQdt26p7WMsZZqd1M2DibbC3bCtiLqBJyIeBB4k\ne/qolaBI7ygop6F6rGSEvVKq6pUM22PTtVevHO9F7uOxklTfRzRpoJ/+vqGD+ZfVTlI0I5y1cCn7\nz5zLTmf8iP1nzi31IWe17jU67fJF7PGZa7viYWut6JX7ZrrhAXO9qpee2OrRqbtMdfVUp65wily5\ndrohuVapDGDlmsEx16DdK50x3A27fcZKLUQRTQOPpCOBH0XEhsPyWtt1qp2kSEbY6ROhUTXUWDsh\neynDdttee/RKdSwUK/G8DfiKpCvJukTf3eY0WRcokhF2+kRo1hljrJ2QzrCtkV5qPyvyBNLjJb0A\nOA64UFIA/w1cGhFPtDuB1jm1MsJ81d+ENNxMtbJOhGadMcbiCWlWT69Ux0LBNp6I+LOkHwADwGnA\nm4HTJZ0TEf/ZzgTa6BpJm1F1m06toFPmiVBJd60nkI7VE9Ksnl6qji0yZM5RZCMVvBS4CPhORDwm\naVPgzoiY1u5EFjXWbyBtt5HepFjv5sA+iXURHT0ReqWbqVnZuu0G0opjgC9HxA35iRGxWtJ72pMs\na4eRdgao12ayLoL7Z77x+cz/w5cvKj3zd/uI2dhRJPD8vjroSDo7Ij4eEb9oU7qsDUbaGaBR4+Zo\ndK1uVGpxicasdxS5gfT1NaYdPtoJsZFrdjPnSG9SbHRz4Egfldzo5rheunHOzBoEHknvl3Q7sKuk\n23J/9+Ox2rpOkcx5pHeVN3pC60hLU40C10iDmpl1l0ZVbd8DfgKcBZyRm/5ERDze1lRZy4q034xG\nr5h6bSkjvcdgOIFrrN2n081clWllahR4IiIekPTB6hmStnbw6S5FM+52NcKP9B6DZoGrV26c60ad\nHvrIxp9GbTzfS/8XAPPT/wW599ZFOj3IZKNquCIaVQN64Mn2clWmla3RYxH+Nv3fqbzk2HB1w13N\nIylNFakGdFVQe4ykfc5VdDYcdQOPpL0afTAifjf6ybHh6oW7mhsFLt+n0z7DbZ9zFZ0NV6M2ni81\nmBdATz/HdixeyTlztuEYbmm506OT29jVqKrt4DIT0k18JWfjyXBLy50endzGrkZVbYdExFxJx9Sa\nHxFXtS9ZneUrud4xFkuunTCc0nIvDdNv5WpU1fZaYC5wZI15AfRs4PGVXG9wybW9uqFDi41Njara\nzkz/TywvOd3BV3K9wSXX9uqFDi3WGUUeff1C4EzgALKSzq+Bz0bEn9qcto7xlVxvcMm1/dyhxYaj\nyCChlwHLgWOBt6TXl7czUZ020pshrT2aDYJarV4JNaDQ582sPYo8CO6OiHhF1bTbI+KVbU3ZMPhB\ncL1rOA+xq/WZvFYegmfWqzrxILgiJZ5rJb1d0oT093fAnHYnzCxvOMO65EuutXhYGLPOaPRYhCck\n/Rl4H9m4bc+mv8uAk8pJnllmuO01R+85hRvPOAS1+L1m1j51A09EbBERL0j/J0TExPQ3ISJeUGYi\nzUY6CGqnB1E1s/WKVLUhaStJe0s6sPLX7oSZ5Y10hGqPcG3WPYp0p34vcCqwPbAI2Bf4DT0+Vpt1\nl5HeM+J7Tsy6R5FebbcDfw3Mi4g9JO0KfD4iag6l00nu1WZm1ppu7dX2dEQ8DSBp44i4GyhUPyHp\nMEn3SFoi6Ywa8yXpnDT/tupHMUjqk7RQ0g+LrM/MzLpf06o24BFJk4BZwM8krQAebPYhSX3AucDr\ngUeAWyTNjog7c4sdDuyc/vYBvpH+V5wK3AW4M4OZWY9oWuKJiDdHxMqImAF8Cvg2cHSB794bWBIR\n90VEpRv2UVXLHAVcFJl5wCRJ2wJI2h54I/CtwltjZmZdr0iJ53kR8csWFp8CPJx7/whDSzP1lpkC\nLAO+AnwM2KLRSiSdRLqvaIcddmgheWZm1gmFulOXTdLfAo9FxIJmy0bE+RExPSKmT548uYTUmZnZ\nSLQz8CwFpubeb5+mFVlmf+BNkh4gq6I7RNLF7UuqmZmVpZ2B5xZgZ0k7SdoIeDswu2qZ2cAJqXfb\nvsCqiFgWEZ+IiO0jYlr63NyIOL6NaTUzs5K01MbTioh4TtIpZAOK9gEXRMRiSSen+ecBPwaOAJYA\nq4Fx99A5M7PxpukNpGOJbyA1M2tNt95AamZmNmoceMzMrFQOPGZmVioHHjMzK5UDj5mZlcqBx8zM\nSuXAY2ZmpXLgMTOzUjnwmJlZqRx4zMysVA48ZmZWKgceMzMrlQOPmZmVyoHHzMxK5cBjZmalcuAx\nM7NSOfCYmVmpHHjMzKxUDjxmZlYqBx4zMyuVA4+ZmZXKgcfMzErlwGNmZqVy4DEzs1I58JiZWakc\neMzMrFQOPGZmVioHHjMzK5UDj5mZlcqBx8zMSuXAY2ZmpWpr4JF0mKR7JC2RdEaN+ZJ0Tpp/m6S9\ncvMekHS7pEWS5rcznWZmVp6J7fpiSX3AucDrgUeAWyTNjog7c4sdDuyc/vYBvpH+VxwcEX9sVxrN\nzKx87Szx7A0siYj7IuJZ4DLgqKpljgIuisw8YJKkbduYJjMz67B2Bp4pwMO594+kaUWXCeDnkhZI\nOqneSiSdJGm+pPnLly8fhWSbmVk7dXPnggMiYg+y6rgPSjqw1kIRcX5ETI+I6ZMnTy43hWZm1rJ2\nBp6lwNTc++3TtELLRETl/2PA1WRVd2ZmNsa1M/DcAuwsaSdJGwFvB2ZXLTMbOCH1btsXWBURyyRt\nJmkLAEmbAW8A7mhjWs3MrCRt69UWEc9JOgWYA/QBF0TEYkknp/nnAT8GjgCWAKuBE9PHXwxcLamS\nxu9FxE/blVYzMyuPIqLTaRg106dPj/nzfcuPmVlRkhZExPQy19nNnQvMzKwHOfCYmVmpHHjMzKxU\nDjxmZlYqBx4zMyuVA4+ZmZXKgcfMzErlwGNmZqVy4DEzs1I58JiZWakceMzMrFQOPGZmVqqeGiRU\n0nLgwQaLbAP8saTktJu3pTv1yrb0ynaAt6WZHSOi1Kdo9lTgaUbS/LJHYW0Xb0t36pVt6ZXtAG9L\nN3JVm5mZlcqBx8zMSjXeAs/5nU7AKPK2dKde2ZZe2Q7wtnSdcdXGY2ZmnTfeSjxmZtZhDjxmZlaq\nngs8kg6TdI+kJZLOqLPMQZIWSVos6Zdlp7GoZtsi6fS0HYsk3SFpraStO5HWZgpsy5aSrpF0a/pd\nTuxEOososC1bSbpa0m2Sbpb0ik6kswhJF0h6TNIddeZL0jlpW2+TtFfZaSyiwHbsKuk3kp6R9NGy\n09eKAtvyzvRb3C7pJkmvKjuNIxYRPfMH9AH/D/hLYCPgVmC3qmUmAXcCO6T3L+p0uoe7LVXLHwnM\n7XS6R/C7/DNwdno9GXgc2KjTaR/mtnwRODO93hX4RafT3WB7DgT2Au6oM/8I4CeAgH2B33Y6zcPc\njhcBfw38G/DRTqd3hNvyf4Ct0uvDu/U3afTXayWevYElEXFfRDwLXAYcVbXMO4CrIuIhgIh4rOQ0\nFlVkW/KOAy4tJWWtK7ItAWwhScDmZIHnuXKTWUiRbdkNmAsQEXcD0yS9uNxkFhMRN5Dt63qOAi6K\nzDxgkqRty0ldcc22IyIei4hbgMHyUjU8BbblpohYkd7OA7YvJWGjqNcCzxTg4dz7R9K0vJcBW0m6\nXtICSSeUlrrWFNkWACRtChwGXFlCuoajyLZ8DXg58ChwO3BqRKwrJ3ktKbIttwLHAEjaG9iRMZg5\nJIWPQ+uI95CVSMeUiZ1OQAdMBF4NvA4YAH4jaV5E3NvZZI3IkcCNEdHoyrXbHQosAg4BXgL8TNKv\nIuLPnU3WsMwEvippEVkQXQis7WySrNdIOpgs8BzQ6bS0qtcCz1Jgau799mla3iPAnyLiKeApSTcA\nrwK6LfAU2ZaKt9O91WxQbFtOBGZGVnG9RNL9ZO0jN5eTxMKabksKlidC1jgP3A/cV1YCR1krx6GV\nRNJfAd8CDo+IP3U6Pa3qtaq2W4CdJe0kaSOyDHl21TL/AxwgaWKqotoHuKvkdBZRZFuQtCXwWrLt\n6lZFtuUhslIoqT1kF7ozs266LZImpXkA7wVuGKMlN8i27YTUu21fYFVELOt0osYzSTsAVwF/P1Zr\nanqqxBMRz0k6BZhD1vvogohYLOnkNP+8iLhL0k+B24B1wLcioma3xU4qsi1p0TcD16YSXFcquC3/\nClwo6XayHlQfj4iuG8q+4La8HPiOpAAWk1WHdCVJlwIHAdtIegQ4E+iH57flx2Q925YAq0kluW7T\nbDsk/QUwH3gBsE7SaWS9EbvugqDAb/Jp4IXA17MCNc/FGBux2kPmmJlZqXqtqs3MzLqcA4+ZmZXK\ngcfMzErlwGNmZqVy4DEzs1I58Ji1KI10vEjSQkkv6YL0fFbS36TXD0japsYyN5WfMrPa3J3axgxJ\nfRGxtt77Bp+bGBGjMuCopD7gdGBiRHxuNL5zNEl6AJjejfdAmVW4xGNdQdLx6dk1iyR9M2XwSHpS\n0pck3Qrsl67oz5b0O+CtkvaQNC89n+RqSVulz10v6SuS5gOnVq3rtVr/HKOFkrZQ9oymH+aW+Zqk\nd6fX+XW+DTgNeL+k69L8WWnA2cWSTsp9x2GSfqfsGUO/SNM2S89buTmte4MRxyVtK+kGrX/O0mty\n++LLaT2/kDQ5Tb9Q0luqvmNA0k8kva/y2fT/oLRvfiDpbkmXpGF9kPRqSb9M2zJHaRRqSR+SdGfa\nx5fV24fD+uFtfOr0cxn85z+yO/2vAfrT+68DJ6TXAfxdbtkHgI/l3t8GvDa9/izwlfT6euDrddZ3\nDbB/er052QgeBwE/zC3zNeDdddY5g9wzXYCt0/8B4A6yu8onk43qvFPVMp8Hjk+vJ5GNEbhZVfo+\nAvxLet0HbJHbF+9Mrz8NfC29vhB4Sy6t04CfV/Zhmv5k+n8QsIpszLUJwG/IBpnsB24CJqfl3kY2\nKgNkI4ZvXElzvX3Y6ePIf2Pnr6eGzLEx63VkI4bfki6+B4DKc5LWsuHjHi6H58epmxQRlafIfgf4\nfvVyNdwI/IekS8iezfRIWm8j9b4L4EOS3pxeTwV2Jgs8N0TE/QCxfuTwNwBv0vqnYG4C7MDQ8QJv\nAS6Q1A/MiohFafq6XDouJhuvq5b/Ab4QEZfUmX9zRDwCoGwE7WnASuAVZKOCQxbwKmOy3QZcImkW\nMCtN22Af1lmX2QZc1WbdQMB3ImKP9LdLRMxI856ODdtxio5LV3O5iJhJNnjnAHCjpF3JHjqXPx82\nKfJdkg4C/gbYLyJeRfYIhOrPDvkIcGxuW3eIiCGD1Eb2ILADyUaBvlD1nxlVr4H2RuAw1Y+mz+Re\nryUr8QlYnEvXKyPiDWmZNwLnkj0V85bUZlZrH5oV4sBj3eAXwFskvQhA0taSdmz2oYhYBayotIEA\nfw/8ssFHSN//koi4PSLOJitd7Ao8COwmaWNJk0gjZRewJbAiIlanzHffNH0ecKCknSrblKbPAf4p\n166yZ4307Qj8ISL+i2zo+73SrAlApS3nHcCv66Tp08AKsmBR1D3AZEn7pTT0S9pd0gRgakRcB3w8\nbe/mdfahWSGuarOOi4g7JX0SuDZldIPAB8mCQTPvAs5T9oiL+yg2evJpyh6itY5s9OifRMQzkq4g\na6O5n6zkUsRPgZMl3UWWec9L27Q8dTS4Km3TY8DryUbh/gpwW5p+P/C3Vd95EHC6pEHgSaBS4nkK\n2Dvtq8fI2mHqOZWsuu4LEfGxZhsREc+mDgrnpCrMiSmd9wIXp2kCzomIlZL+tXofNluHWYW7U5uN\nEZKejIjNO50Os5FyVZuZmZXKJR4zMyuVSzxmZlYqBx4zMyuVA4+ZmZXKgcfMzErlwGNmZqX6//i+\n42FWDCMyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f833d005e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(perturbation_deltas_sum, generalization_deltas)\n",
    "plt.gca().invert_yaxis()\n",
    "plt.xlabel('error surface spikiness')\n",
    "plt.ylabel('ability to generalize')\n",
    "plt.title('Ability to generalize vs uniformity of surrounding error surface')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}