Skip to content

Commit

Permalink
new visualization linear approximation
Browse files Browse the repository at this point in the history
  • Loading branch information
reinhardh committed Oct 25, 2019
1 parent e49410b commit ee14e4d
Showing 1 changed file with 118 additions and 0 deletions.
118 changes: 118 additions & 0 deletions visualization_linear_approximation.ipynb
@@ -0,0 +1,118 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Approximation with a linear model\n",
"\n",
"Here, we visualy demonstrate that an overparameterized network can be well approximated around a random inital point with a linearized model."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"#%matplotlib notebook\n",
"#import matplotlib.pyplot as plt\n",
"from numpy import *\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4VFX+x/H3SQ9JSEgPhBBK6B1EelWpgiJ2BV1c7Gvv7Lr6c9fGWrFhxQYCKlUFqdIlID2EBEiAEEJIgRQSksz5/XFHH9alhGQmd+bO9/U8PGGGydzv9cZPzpx7itJaI4QQwrq8zC5ACCGEc0nQCyGExUnQCyGExUnQCyGExUnQCyGExUnQCyGExUnQCyGExUnQCyGExUnQCyGExfmYXQBAZGSkTkxMNLsMIYRwK5s3bz6utY660OtcIugTExNJTk42uwwhhHArSqnM6rxOum6EEMLiJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMIMWsPiZ+DoTqcfSoJeCCHMkL4U1k+FHAl6IYSwpnVvQUhDaH+N0w8lQS+EEHXtyFY48Av0vBu8fZ1+OAl6IYSoa+ungl8IdJtQJ4eToBdCiLpUeBB2fmeEfEBonRxSgl4IIerShvdBKaPbpo5I0AshRF05VQhbpkO7sRAaX2eHlaAXQoi6kvwJnC6G3vfX6WEvGPRKqU+UUseUUjvPeC5cKfWzUirN/rWB/XmllHpLKZWulNqulOrqzOKFEMJtVJTBxveh+WCI61inh65Oi/4zYNifnnsSWKa1TgKW2R8DDAeS7H8mAe85pkwhhHBz22dCcQ70eaDOD33BoNda/wLk/+npMcB0+9+nA1ed8fzn2rABCFNKxTmqWCGEcEu2Klj3NsR1hqYD6vzwNe2jj9FaZwPYv0bbn28EHDrjdYftzwkhhOfaswjy0o3WvFJ1fnhH34w92xnos75QqUlKqWSlVHJubq6DyxBCCBehNax9AxokQtsxppRQ06DP+b1Lxv71mP35w0DjM14XDxw52xtoradprbtrrbtHRUXVsAwhhHBxmWshazP0ug+8vE0poaZBPx/4fe7uBGDeGc+Pt4++6Qmc+L2LRwghPNKa1yEoCrrcYloJPhd6gVJqBjAQiFRKHQaeBV4CZimlJgIHgWvtL/8BGAGkA6XA7U6oWQgh3EP2dmM54iH/AN9A08q4YNBrrW88xz8NOctrNXBvbYsSQghLWPO6sXhZ94mmliEzY4UQwhny9sHuuXDJRAgMM7UUCXohhHCGdW+Dly/0vMfsSiTohRDC4YqOwtavoMvNEBJjdjUS9EII4XDrp4Ktss4XLzsXCXohhHCk0nzY9ImxF2x4M7OrASTohRDCsTZ+ABUl0Pdhsyv5gwS9EEI4SnmRsRRx61EQ09bsav4gQS+EEI6y6WMoK4R+rtOaBwl6IYRwjIpTsP4daDYIGnUzu5r/csGZsUIIIaphyxdQcgz6fWJ2Jf9DWvRCCFFbleXGUsSNe0JiX7Or+R8S9EIIUVvbZsDJLBjwmCkbi1yIBL0QQtRGVQWsfg0adoXm/7PWo0uQoBdCiNrYMRsKM2HA4y7ZmgcJeiGEqDlbFaz+D8R2gJbDzK7mnCTohRCipnZ9b2z63d81++Z/J0EvhBA1YauCVa9AVBtofaXZ1ZyXBL0QQtTE7rlwPNXom/dy7Sh17eqEEMIV2Wz21nxraHuV2dVckAS9EEJcrN1zIXePW7TmQYJeCCEuzu+t+chWbtGaBwl6IYS4OCnzIDfF3pr3NruaapGgF0KI6rJVwcqXILIltLva7GqqTVavFEKI6tr1vdE3P+4Tt2nNg7TohRCien5vzUe1gbbu05oHadELIUT17JgDeWlw3eduMdLmTO5VrRBCmKGqEla9DDHtXX4W7NlIi14IIS5kxyzI3wfXf+V2rXmQFr0QQpxf5Wmjbz62I7QeaXY1NSIteiGEOJ+tXxrrzd8026VXqDwfadELIcS5VJTBqleh8aWQdLnZ1dSYtOiFEOJckj+BoiMw9gO3bc2DtOiFEOLsyothzWvQdAA07W92NbVSq6BXSj2klNqllNqplJqhlApQSjVVSm1USqUppb5RSvk5qlghhKgzv34AJbkweLLZldRajYNeKdUI+BvQXWvdHvAGbgBeBl7XWicBBcBERxQqhBB15lQBrH0TkoZC4x5mV1Nrte268QEClVI+QD0gGxgMzLH/+3TAPdbxFEKI3619C8pOwpB/mF2JQ9Q46LXWWcAU4CBGwJ8ANgOFWutK+8sOA41qW6QQQtSZoqOw4T3oMA5i25tdjUPUpuumATAGaAo0BIKA4Wd5qT7H909SSiUrpZJzc3NrWoYQQjjWL6+CrQIGPW12JQ5Tm66by4ADWutcrXUF8B3QGwizd+UAxANHzvbNWutpWuvuWuvuUVFRtShDCCEcJP8AbP4Muo6H8GZmV+MwtQn6g0BPpVQ9pZQChgC7gRXAOPtrJgDzaleiEELUkZUvgpcv9H/c7EocqjZ99BsxbrpuAXbY32sa8ATwsFIqHYgAPnZAnUII4VxHd8D2WXDpJKgfZ3Y1DlWrmbFa62eBZ//09H7A/ccjCSE8y9LnICAU+j5kdiUOJzNjhRDiwC+Q/jP0ewQCG5hdjcNJ0AshPJvW8POzUD8eekwyuxqnkEXNhBCebfc8OLIFxrwLvgFmV+MU0qIXQniuqgpY9ryx4XenG8yuxmmkRS+E8FzJnxpbBN40C7y8za7GaaRFL4TwTGUnYdVLkNgPkq4wuxqnkqAXQnimtW9AaR5c8X9uvalIdUjQCyE8z4ksWP8OdLgOGnYxuxqnk6AXQnieFf8CbYMhfze7kjohQS+E8CzZ22Hr13DpnRCWYHY1dUKCXgjhObSGJc8Ys1/7PWp2NXVGgl4I4Tn2/mQsdzDoaQgMM7uaOiNBL4TwDFUVsGQyRLaEbreZXU2dkglTQgjPkPwp5KXDjd+At6/Z1dQpadELIazvVIGxqUjTAdByqNnV1DkJeiGE9a18GcoKYei/LD856mwk6IUQ1pa7FzZ9CF0nQGwHs6sxhQS9EMLaFj8NvkEweLLZlZhGgl4IYV17lxg7Rw14HIIiza7GNBL0QghrqjxttOYjWlh256jqkuGVQghr+nUa5KUZwyl9/MyuxlTSohdCWE9RDqx8yVhnvtUws6sxnQS9EMJ6lj0HlWUw9EWzK3EJEvRCCGs5nAxbv4Je90JkC7OrcQkS9EII67DZ4IfHIDgW+nvO6pQXIjdjhRDW8dsXcGQLXD0N/EPMrsZlSIteCGENpfmw9J+Q0Bs6Xmd2NS5Fgl4IYQ3LnoeyEzByikeuZ3M+EvRCCPeXtRk2f2ZsDxjTzuxqXI4EvRDCvdmqYNEjEBwNA580uxqXJDdjhRDubfNncOQ34wZsQKjZ1bgkadELIdxX8TFY+hwk9pMbsOchQS+EcF9LJkNFKYx8TW7Anketgl4pFaaUmqOU2qOUSlFK9VJKhSulflZKpdm/NnBUsUII8Yf9q2D7N9D3QYhqaXY1Lq22Lfo3gZ+01q2BTkAK8CSwTGudBCyzPxZCCMepLDduwDZIhH6PmF2Ny6tx0Cul6gP9gY8BtNantdaFwBhguv1l04GralukEEL8l7VvGksQj5gCvoFmV+PyatOibwbkAp8qpX5TSn2klAoCYrTW2QD2r9EOqPPstIac3U57eyGECzqeBr+8Cu3GQtLlZlfjFmoT9D5AV+A9rXUXoISL6KZRSk1SSiUrpZJzc3NrVsHKF+HDwZB/oGbfL4RwL1rDwoeMVvywl8yuxm3UJugPA4e11hvtj+dgBH+OUioOwP712Nm+WWs9TWvdXWvdPSoqqmYVdLsNvHxg4YPGD4AQwtq2fgUZq+Hy5yEkxuxq3EaNg15rfRQ4pJRqZX9qCLAbmA9MsD83AZhXqwrPp35DuOxZ2L8Sts102mGEEC6g5LgxnDKhF3QZb3Y1bqW2M2PvB75SSvkB+4HbMX55zFJKTQQOAtfW8hjn130i7JhtbAKcdLlH7/QuhKX9+ASUF8OVb4KXTAG6GLX6r6W13mrvfumotb5Ka12gtc7TWg/RWifZv+Y7qtiz8vKCK9+C8iL46SmnHkoIYZK9i2HnHOj/GES1uvDrxX+xxq/F6NbGWNods2DvErOrEUI4UtlJ4wZsdFvo+5DZ1bglawQ9QL+HIaq1cWO27KTZ1QghHGXpP6EoG0ZPBR8/s6txS9YJeh9/GPOO8QOx9FmzqxFCOELmOkj+GC69G+K7mV2N27JO0APEd4ee90DyJ3BgtdnVCCFq43QpzLsXwprA4GfMrsatWSvoAQY9A+HNYP59cLrE7GqEEDW1/AXI3298UvcLMrsat2a9oPerB6PfhoIMYw9JIYT7ObgRNrwLl9wBTfuZXY3bs17QAyT2hR53wsb3IWON2dUIIS5GxSmYdw+ENobLnjO7GkuwZtCDMWO2QVOjj0+6cIRwH8tfgLx0GPM2+AebXY0lWDfo/YKMvr2CDGN4lhDC9WWug/XvQLfbodlAs6uxDOsGPUBiH2NY1q/TjN1ohBCuq7wY5t4NYQlwxQtmV2Mp1g56gCH/gIgWMPceKDthdjVCiHP5+R9QkAlXvy9dNg5m/aD3qwdXfwBFR+BH2dVQCJe0b7kxMarXvdCkt9nVWI71gx6MiVT9HoFtX0PKArOrEUKcqTTf+MQd2RIGTza7GkvyjKAH6P84xHWCBQ9A8Vn3QhFC1DWtjU2+S3Jh7Iey/6uTeE7Q+/gZXTjlxTDvPtmRSghXsGMO7PoOBj4FDTubXY1leU7QA0S3MbYgS1ts9AcKIcxTeMhozTe+FPo8aHY1luZZQQ/QYxI0HwyLJ0PuXrOrEcIz2aqMoZS6yvik7V3bze7E+Xhe0Ht5wVXvGX2B390BlafNrkgIz7P2DWOT7+EvQ3hTs6uxPM8LeoCQWGPhs+xtsEImZghRpw5vhhX/hnZXQ+ebza7GI3hm0AO0GWVMs177pjGGVwjhfOVF8O1ECImDUa+DUmZX5BE8N+gBhv7b2H7w+7ugONfsaoSwvh8eg8JMGDsNAhuYXY3H8Oyg96sH4z6BU4XGjSGbzeyKhLCurTNg2wzo/5jMfq1jnh30ADHtYOi/IP1nY6MDIYTjHU8zhlI26WNMXhR1SoIejF1sWo8yljM+vNnsaoSwlooymH07+PjDNR/JUEoTSNCDcUNozFTjBtHs2+BUgdkVCWEdSyZDzg5jVcr6Dc2uxiNJ0P8usAFc+6mxyqUskSCEY+z8FjZ9CL3ug5ZDza7GY0nQnym+u7FH5Z6Fxn6zQoiaO54O8/8G8T3gsn+aXY1Hk6D/s173QqsRxsfNQ7+aXY0Q7qniFMwaD95+xidlb1+zK/JoEvR/ppSxREJoPMyaIOPrhaiJHx6FY7uMpYdD482uxiXZbJrP1h6gsNT5y7BI0J9NYBhc9zmU5hmz+GxVZlckhPvYPB1++xL6PQpJl5ldjUsqKDnNxOmb+OeC3cxOPuz040nQn0tcJxg5BQ6sMtblEEJcWNYWY/Zrs0Ew6Gmzq3FJWw4WMPKt1axNz+P/xrTjjn7OX9TN7Qe0VlTZ8PV20u+rruONfvrVU6BhF2N9HCHE2ZXkGf3ywdFwzcfg5W12RS5Fa83Haw7w0o97iAsL4Nu7e9MhPrROju3WLfoF244w6q01ZBWect5BRkyBhl2N9XBk/Xohzs5WZXRzFh8zuj2DIsyuyKWcKK1g0hebeWFRCoNbR7Pw/n51FvLggKBXSnkrpX5TSi20P26qlNqolEpTSn2jlPKrfZlnFxHsx5HCU4x9dy27jpxwzkF8A+D6L4xZfd/cDGUnnXMcIdzZ0n/C/hVGd2ejrmZX41K2Hipk5NurWbHnGH8f1ZYPbu1GaGDdjkJyRIv+ASDljMcvA69rrZOAAmCiA45xVr2bRzLn7t54KcX1H2xgdZqTRsiExsO1n0HePln8TIg/2zEH1r1lLCXSdbzZ1bgMm00z7Zd9jHtvHVrDrLt6MbFvU5QJSzPXKuiVUvHASOAj+2MFDAbm2F8yHbiqNse4kFaxIXx/Tx/iGwRy+6ebmJV8yDkHatrPWPxsz0JY9ZJzjiGEu8neZswkT+gNQ180uxqXkVdczl+mb+LfP+zhsjYx/PC3fnRNMG9Z5tq26N8AHgd+b+JGAIVa60r748NAo7N9o1JqklIqWSmVnJtbu5Z4bGgAs+/qRa/mETw+ZztTFqeinbGEwaV3QedbYNXLsGuu499fCHdSnAszb4Z64XDddPBxWi+tW1m1N5ehb6xm3T5jVM17t3QltJ65E8ZqHPRKqVHAMa31mcs9nu0zyVkTV2s9TWvdXWvdPSoqqqZl/CEkwJdPbruEG3s0ZuqKdB6YuZWyCgePf1cKRr1mTOmeezdkb3fs+wvhLirL4ZtboCQXrv/SGGnj4corq3hh4W4mfPIr4UG+zL+vD7f2SjSlq+bPajO8sg8wWik1AggA6mO08MOUUj72Vn08cKT2ZVaPr7cX/766AwnhQbz80x4OF5QybXx3IoP9HXcQH3/jB3vaQJh5E9yxDEJiHPf+Qrg6rWHBg3BoA4z7VG6+AruPnOShb7aSmlPE+F5NeHpEGwJ8XWd4aY1b9Frrp7TW8VrrROAGYLnW+mZgBTDO/rIJwLxaV3kRlFLcPbA5797cld3ZJxkzdS2pR4sce5CQGLhxhjFzduZNxroeQniKdW/Btq9hwJPQfqzZ1ZiqyqZ5d2U6Y95ZQ37paT65rTvPj2nvUiEPzhlH/wTwsFIqHaPP/mMnHOOCRnSIY9advaiosnHNe+tYlpLj2AM07Gzse5m1GebeI8saC8+QshB+fhbaXgUDnjC7GlOl5RQx9r11vPJTKle0jWXJg/0Z3No1P90rp9y0vEjdu3fXycnJTnnv7BOn+Ovnyew6cpLHh7bmrgHNHNtntuYNWPqs8UMvU76FlWVtgU9HQExbmLDQ2HPZA1VU2fhg1T7eWpZOkL83/xzdjtGdGprSF6+U2qy17n6h17n9EggXEhcayOw7e/PYnG28/NMeUo+e5KVrOjruo1WfByAvzRiJ0yAROt/kmPcVwpUUHoQZN0BQFNw402NDfuuhQp78djt7jhYxsmMcz41u59h7gE5i+aAHCPTz5u0bu9A6NoQpS/ay/3gJ79/SjYZhgbV/c6Vg5OtQeAjm329sldZsYO3fVwhXUXYCvr7e2Pt1/HyPHGFTXF7JlMWpTF+fQXSIP+/f0o1h7WPNLqva3Hqtm4uhlOK+wUl8OL47+3NLGD11DZsy8h3z5j5+xjIJkS3hm1shZ5dj3lcIs1WWG2Plj+81xspHtza7ojqltWbe1iwGT1nJ9PUZjO/ZhKUPD3CrkAcPCvrfXd42hrn39iYkwJcbp23giw2ZjplcFRAKN88GvyD46lo4kVX79xTCTDabMdAgYzWMeQeaDzK7ojqVkn2S6z/YwAMztxIbGsB3d/fmuTHtCQlwv92yPC7oAVpEhzD33j70bxnF3+fu5NHZ2x0zuSo03gj78iL4ciyUOugTgxBmWPoP2DkHhjwLnW4wu5o6U1BymslzdzDyrdWkHSvixbEdmHtPH7qYuIRBbXlk0AOEBvry0fjuPHhZEt9uOcw1763jUH5p7d84tgPc8BXk74cZN8oYe+Ge1r8D696GS/4KfR8yu5o6UVllY/q6DAZOWcmMXw8xvlciKx4dyI09EvDyMn92a21YfnhldSzfk8ODM7eilOK16zoxpI0DxsLu+h5m3w6thsN1X4C3R9z3FlawdQbMvQvajjFmvnrABiJr0o7z/MJd7M0ppk+LCJ69sh0tY0LMLuuCqju80mNb9Gca3DqGhff3I75BIBOnJ/PKT3uorKrlUsTtroYRr0LqDzD/PlnaWLiH1B9h3r3GyLGxH1o+5PflFnPH9GRu+XgjZRU2pt3ajS8nXuoWIX8xpJlplxBRj2/v7s1zC3bz7sp9JGcU8MYNnWs3BLPHX+FUAaz4F/jXh+EvG8MxhXBFGWtg9m3GfsnXf2ms62RRBSWneXNZGl9uyCTA15snhrXmL30T8fex5i82CfozBPh68+LYDvRo2oDJ3+9k+JureXVcR65oV4uhVP0fM8Yhr59qjMwZ/IzjChbCUQ4nG2Plw5rAzXPA31ot2t+VV1bx+bpM3l6eRnF5JTf2SOChy1u6xaSn2pCgP4uru8TTuXED7p+xhUlfbOaWngk8M6ItgX41+G2vFFzxghH2v7xizCj0kJtbwk1kbzdGiQVFwvh5ltzv1WbTLNyRzSs/7eFwwSkGtYriqRFtLNdFcy4S9OfQNDKIb+/uzas/pfLRmgOs35fHmzd0oX2jGmzoqxRc+SZUlBp7a3r7Q697HF6zEBctNxW+uBr8go1Zr/XjzK7IobTWrE47ziuL97Az6yRt4urz5cSO9E2KNLu0OiVBfx7+Pt5MHtWWga2ieWT2Vq5+dy0PX96KSf2b4X2xw628vOHqD6DqNCx+yphNe8kdzilciOrI3QufjQLlZYR8gyZmV+RQmzPzmbJ4L+v35xHfIJDXr+/E6E6NLv7/XQuQ4ZXVVFBymqe/38GPO4/SvUkD/nNdJ5pEBF38G1Wehlm3wt6fYNQb0P12xxcrxIUcT4fPRoKugtsWQVQrsytymO2HC/nPkr2s2ptLZLAf9w5qwU2XJljyRmt1h1dK0F8ErTVzt2bxj3m7qLJpnhrRhptrMpmistxYEydtMYx8DS6Z6JyChTibvH1GyFdVGCFvkfVrNmfm8/bydFam5tKgni93DmjO+F5NqOdn3Y4LCXonOlJ4iie+3c7qtOP0ahbBS9d0uPjWfWU5zBpvtOxHTDGGYgrhbLmpMP1KsFUaa8rHtDW7olrbsD+Pt5alsW5fHuFBfkzs25TxvZq45Zo0F0uC3sm01sxKPsQLC1OosNl4bGhrbuudeHH9f5Xlxrjl1B9g6Ityg1Y419Gd8PkY437R+Plu3ZLXWrN+Xx5vLEvj1wP5RIX4c2f/Ztx0aYKlW/B/JkFfR7JPnOKZ73eyfM8xOjUO4+VrOtA6tn7136DyNHw7EVLmw6DJ0P9RmVQlHO/IVvjiKvAJhAkLILKF2RXViNaaFanHmLo8nS0HC4kO8efugc25sUeCy+3TWhck6OuQ1pr5247w3ILdnDxVwV0DmnPf4BbV/8GrqjSWSdg2wxhjP+RZCXvhOBlrjclQgQ1gwnwIb2p2RRetssrGoh3ZvL9qPynZJ2kUFsik/s24/pLGHhnwv5OtBOuQUooxnRvRLymKFxbuZuqKdCP4x7RjUKtq7Mbj7QNj3gXfQFjzujG5asQUy68zIurA3sXGvaCwBLh1LoQ2Mruii1J6upJZmw7x0ZoDHC44RfOoIKZc24kxnRvi6y1LdVWXtOidYF36cSbP28n+3BKGt49l8qi2NKrOmjlaw7LnYc1rxsqBYz+09Hojwsm2zzZWoYxpD7d8a8x8dRO5ReVMX5fBFxsyOXGqgu5NGnDngOYMaR3t9ksGO5J03ZisvLKKD3/Zz9QV6SgU9w1uwR39mlZvLO/6d2Dx09C0P1z/FQRcRJ+/EGCsJb9kMjTpCzfOcJufoZTsk3y69gBztx6hosrGFW1jmNS/Gd2ahJtdmkuSoHcRh/JLeWHRbhbvyiExoh5/H9WWwa2jURfqg98201guNqqNsWuVxaamCyex2YyA3/AOtL3KmI3tG2B2Vedls2mW7znGJ2sPsG5fHgG+XlzTNZ6JfZvSLCrY7PJcmgS9i1m1N5fnFuxif24JA1pG8fdRbWkRfYEf4vSlMGsCBIQZYW+BMc/CiSpOwdy7jU1vLr3LGLLr5br92CXllczZfJhP1x4gI6+UuNAAJvRO5IZLGhNWz8/s8tyCBL0LqrBvVfbm0jROVVQxvlciD1yWRGjgeSZ2ZG83NhuvKIXrPve4DZpFNRXnwswbjeWGL38eet/vsiO3UrJP8vXGg3z/WxbF5ZV0SQhjYt+mDG0XKzdYL5IEvQs7XlzOlMWpfJN8iAb1/Hj0ilZcf0njc0+2KjwEX19nzGoc8aosmSD+W26q0RgoPgZjp0Hb0WZX9D8KSk6zcPsR5mzJYtuhQvx8vBjVMY5bejahqxtvum02CXo3sDPrBM8v2M2vGfm0iavPs1e2pWezc6wFXnbSmFiVtgR63AlD/y370ArYu8T4ufAJgJtmQqNuZlf0h+LySpal5LBoezYrUo9RUaVpHRvCuG7xjOsWL90zDiBB7ya01izakc2LP+whq/AUw9vH8tTwNiRE1PvfF9uqYMnfjRttzQYaGzfXk9EIHklrWPsGLH0OYjvADV9DWGOzq6KkvJLle479Ee7llTZi6vszqmNDxnZtRLuGNdjPQZyTBL2bKauoYtov+3lv5T6qbJrb+iRy76AWZ++/3/IFLHoYQuLghq+M/9GF5zhdAgsegB2zod1YGPOOsXOZScoqqliZmsv8bVks33OMsgob0SH+DG8fy6hODemW0EDGvjuJBL2bOnqijClLUvl2y2HCAn156PKW3NQjAZ8/36Q6nGwsdXyqAEa/DR2vNadgUbeOpxv7GRxLgcGTod8jptx0rayysX5/HvO3HuGnXUcpKqskMtiP4e3jGNUxju6J4R65wUddk6B3czuzTvDCot1s2J9PUnQwz4xsw8A/L6dQlGOsfnlwHXSfCMNelJm0Vpay0Bg+6eUD13wELYbU6eErq2z8mpHP4p1HWbQjm+PFpwn292Fou1jGdG5I7+YR/9sgEU4lQW8BWmuW7M7h3z+kkJlXSr+kSJ4e0YY2cWfMcqyqMJZNWPcWxHWGaz9zy0WrxHlUlsPPz8LG96BhF2OYbVhCnRz6ZFkFa9KOsyzlGMv25FBYWoG/jxeDW0czulNDBrWO9uhFxcwmQW8hpyttfL4+g7eXp3OyrIJxXeN5+IqWxIWesX7Onh+MdU20hlGvQ4dxptUrHChvH8y5HbK3GZOgLn/eqZ/aSsor2XqokE0Z+WzYn0dyRgGVNk39AB8Gt45maLtY+reMIshfRny5AqcHvVKqMfA5EAvYgGla6zeVUuHAN0AikAFcp7UuON97SdBXz4mdnkMOAAANbElEQVTSCqauSGP6ukxQcFvvRO4e0JwGQfZhagWZ8N1f4dBG6HQTjHgF/EPMLVrUjNbGstU/PG6sYjrmHWgzymFvX1ll42B+KWnHiknLKSLlaBGpR4vYn1uMTRvd/m1i6zOgVRSDWkXTNSFMumVcUF0EfRwQp7XeopQKATYDVwG3Afla65eUUk8CDbTWT5zvvSToL86h/FJeX7qX73/LItjPhzsHNOP2Pk2NVlZVJfzyCvzyqvHx/uoPIKGn2SWLi1Gab4yqSZlvLEp29fu1Hjp5vLicTQfySc4sYHNmAbuPnOR0le2Pf28cHkirmPq0iQuhW5MGdG3SgPoesBWfu6vzrhul1Dxgqv3PQK11tv2XwUqt9Xm3mJegr5nUo0W8ujiVpSk5RAb7cf/gJG7skYCfjxdkrofv74TCg9DnbzDoGblR6w5Sf4QFD0JpnjGqpvf9NdqX4MSpCjZn5rN+Xx5r0vNIyT4JgL+PF53iw+icEEbLmBCSooNpHh1MsHTFuKU6DXqlVCLwC9AeOKi1Djvj3wq01v8zx1kpNQmYBJCQkNAtMzOz1nV4qs2ZBby6eA8b9ucT3yCQhy5ryVVdGuFdUWysZLj5M4hqDaOnQuNLzC5XnE1pPvz0FGyfCdHt4Or3IK5Ttb5Va01GXim/HSxgy8ECkjMKSM0pQmvw8/aiW5MG9E2KpFfzCNo3DDUaAsIS6izolVLBwCrgX1rr75RShdUJ+jNJi772tNasTjvOq4tT2ZF1ghbRwTx8eUuGtYvFa99Soyvg5BHjht7gyeAvy7+6BK1h91z48QmjFd/vEej3KPice3mAiiobO7NOkJxRwKaMfDZnFpBXchqAYH8fuiSEcUliOJckhtO5cRiBfjIqxqrqJOiVUr7AQmCx1vo1+3OpSNeNabTWLN51lClL9pJ+rJik6GDuG9yCUa2C8V7+PGz6CEIbw/CXodUIl13h0CMUZMCiRyH9Z6P1Pvrts7biyyur+O1gIev35fHrgXx+O1RAWYXRv54YUY9uTcLt/ephJEWHyEQlD1IXN2MVMB3jxuuDZzz/KpB3xs3YcK314+d7Lwl6x6uyGWvoTF2ext6cYhIj6jGpf3PGRR3C78dHIDcFkoYagS/j7utWRZmxA9Tq/xj974MnwyV//WOROq01+3KLWZmay8rUXDZl5FNeaUMpaBtXn0sSw7m0aTjdEhsQHeLam4oI56qLoO8LrAZ2YAyvBHga2AjMAhKAg8C1Wuv8872XBL3z2GyaJbuP8s6KfezIOkF0iD939I7nVvUTgeteNSZc9b4P+j4kQzGdTWvYs8jYJrIwE9qMNmYzh8ajtSYlu4gF24+waHs2B/NLAWgZE0zfFlH0ah5Bj6bh59+7QHgcmTAl/ovWmnX78nh3ZTpr0/Oo5+fNXzr4cVfllwSnfgvBMUbLstNNsvyxM2RthiX/gMw1xvaQw1+GZgNIyyli4fZsFm4/wr7cEry9FH1bRHJFuxgGtoqu3qbywmNJ0Itz2nXkBJ+syWD+tiwqqjS3J+bxYMUnhOb9BpGtjMBvc6X03ztC3j5Y8S/Y+S3Ui0QPfJLdDceyePdxftp1lL05xSgFPRLDubJTQ0Z0iCM8SNZpF9UjQS8u6NjJMr7+9SBfbTxIblEZN9ffwSM+MwkvzYCGXWHgU5B0uQR+TRRkwKpXYdsMtI8/R9pO5CuvMSxILeJQ/im8FHRPDGdUxziGtYslur70tYuLJ0Evqu10pY3Fu47y1cZMNu3P5Rrv1TweMI/IqhxscZ3xGvA4tBzu0htNu4zcvbD2TfT2mWjlxcbwq3g2fyh7SwLx8/aiT4sIhrWP5bI2MUQEywQ2UTsS9KJGMo6XMHvzIb5PzqBP6TL+5juPxuRQFtYC/35/Q3W8Hnyl9flftIZDv6LXvQV7FlGp/FjgPYSXikdywieCIW2iGdY+jkGtogiRZQWEA0nQi1qpsmnWpB/nu+QMfFLm8he1gHZemZT6RaC7TiCo1x0Q2sjsMs1VWY5t5/ecWvMOQce3U0QQn1Zezpe2YbRq3ozRnRoyrH2shLtwGgl64TAnSitYsC2LvRsWMSB/NoO8tqKVIrfhECL634Fv0mUeNVLn1JHdHF35IVH7viW46gT7bHFMtw3jYPwYLuvcjOHtY6VbRtQJCXrhFOnHivh57Sbq7ZjOqKplRKgiTvqEU9xqHLF9bsUrroMlb96ezMtm34ovCEn7nhblu6nQ3qxQ3UmLH0d8t+EMbBVLaD1puYu6JUEvnKqyysba1CPsW/sdiYfn0Z8t+Cgbx/0TON16DLE9xuHVsJNbh/7xowfZv2YWgek/0uaUcX7pqgkHGo4irNd4Ordpia+s0S5MJEEv6kzp6UpW/ZZC3qY5ND/2Mz3UbryV5oRvFCVNhhDZeSR+zftDYNiF38xEFRUVpG1ZxYkdPxJ+dA1JFal4Kc1hFcvBmMtp0PNmWnfqiXLjX17CWiTohSmKyipYu3U3uVsWEpOzkt5sJ1iVYcOL3JA26ITeRLTug2+TnlA/ztRadXkRh3dvJGfXKvyyNtC0dAch6hQ2rUjzbUle3ABiLh1Hs7aXoGRoqXBBEvTCdGUVVWzYm03GtlX4ZK6m5aktdFL78FeVAJz0jaQ4rDXecR0JbdKBgJhWENEMAs+7qvXFqyyHwoOcOrKbY/u3U569m6D8XcRVHMQL4+c/Q8WTG94V3xYDadZjFPUjYhxbgxBOIEEvXE5uUTlb9udwNHUjtoO/En4yhZZk0kJl4auq/nhdqVcwxf4xlAdGYwuKwTsoHP+QcAKCwwmsVw8fv0Dwso/ysVWBrRIqSuF0MbayIsoKczh9MgddfAzf4iyCTx//rzqydTgZPs0pimhPYEI3mnUZSKNGtduqTwgzSNALl2ezaQ4VlLIn6zh5B/dy+thevAoOEFR6mPoVx4kmjyhVSCglBKnyar1npfaigBCO6/rk6focIZKSwIYQ2ph6DdvSMKkj7Zo2lvVkhCVUN+g9Z/CzcDleXoomEUE0iQiCjk2Ay//4N601ReWV5BaVc6ionLyTxZwszKektJTS0hLKyk5RYVNUauOPd0AwPoH1CQyoR0xYIHGhAcTWD6RbZJDssCQ8ngS9cElKKeoH+FI/wJfmUcFABNDE7LKEcEsylEAIISxOgl4IISxOgl4IISxOgl4IISxOgl4IISxOgl4IISxOgl4IISxOgl4IISzOJZZAUErlApk1/PZI4PgFX2U9nnjennjO4Jnn7YnnDBd/3k201lEXepFLBH1tKKWSq7PWg9V44nl74jmDZ563J54zOO+8petGCCEsToJeCCEszgpBP83sAkziieftiecMnnnennjO4KTzdvs+eiGEEOdnhRa9EEKI83DroFdKDVNKpSql0pVST5pdjzMopRorpVYopVKUUruUUg/Ynw9XSv2slEqzf3XwRqvmU0p5K6V+U0ottD9uqpTaaD/nb5RSltsmSikVppSao5TaY7/mvTzkWj9k//neqZSaoZQKsNr1Vkp9opQ6ppTaecZzZ722yvCWPdu2K6W61ubYbhv0Silv4B1gONAWuFEp1dbcqpyiEnhEa90G6Ancaz/PJ4FlWuskYJn9sdU8AKSc8fhl4HX7ORcAE02pyrneBH7SWrcGOmGcv6WvtVKqEfA3oLvWuj3gDdyA9a73Z8CwPz13rms7HEiy/5kEvFebA7tt0AM9gHSt9X6t9WlgJjDG5JocTmudrbXeYv97Ecb/+I0wznW6/WXTgavMqdA5lFLxwEjgI/tjBQwG5thfYsVzrg/0Bz4G0Fqf1loXYvFrbecDBCqlfIB6QDYWu95a61+A/D89fa5rOwb4XBs2AGFKqbiaHtudg74RcOiMx4ftz1mWUioR6AJsBGK01tlg/DIAos2rzCneAB4HbPbHEUCh1rrS/tiK17sZkAt8au+y+kgpFYTFr7XWOguYAhzECPgTwGasf73h3NfWofnmzkGvzvKcZYcQKaWCgW+BB7XWJ82ux5mUUqOAY1rrzWc+fZaXWu16+wBdgfe01l2AEizWTXM29n7pMUBToCEQhNF18WdWu97n49Cfd3cO+sNA4zMexwNHTKrFqZRSvhgh/5XW+jv70zm/f5Szfz1mVn1O0AcYrZTKwOiSG4zRwg+zf7QHa17vw8BhrfVG++M5GMFv5WsNcBlwQGudq7WuAL4DemP96w3nvrYOzTd3DvpNQJL9zrwfxs2b+SbX5HD2vumPgRSt9Wtn/NN8YIL97xOAeXVdm7NorZ/SWsdrrRMxrutyrfXNwApgnP1lljpnAK31UeCQUqqV/akhwG4sfK3tDgI9lVL17D/vv5+3pa+33bmu7XxgvH30TU/gxO9dPDWitXbbP8AIYC+wD3jG7HqcdI59MT6ybQe22v+MwOizXgak2b+Gm12rk85/ILDQ/vdmwK9AOjAb8De7Piecb2cg2X695wINPOFaA88Be4CdwBeAv9WuNzAD4x5EBUaLfeK5ri1G18079mzbgTEiqcbHlpmxQghhce7cdSOEEKIaJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMLiJOiFEMLi/h9i6/brFsxBxAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x109c00f60>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# generator network\n",
"\n",
"n = 10\n",
"k = 100 \n",
"v = np.ones(k)\n",
"v[:int(k/2)] = -np.ones( int(k/2) )\n",
"v = v/np.sqrt(k)\n",
"U = np.eye(n)\n",
"\n",
"def G(C): \n",
" return np.maximum( U @ C , 0 ) @ v\n",
"\n",
"# Jaccobian\n",
"def J(C):\n",
" return np.vstack( [ve * (U.T @ np.diag(c > 0)) for ve,c in zip(v,C.T)] ).T\n",
" \n",
"# original loss\n",
"def loss(y,C):\n",
" return np.linalg.norm( y - G(C) )**2\n",
"\n",
"# associated linearized loss\n",
"def losslin(y,C,C0):\n",
" return np.linalg.norm( G(C0) + J(C0) @ np.hstack((C-C0).T) - y )**2\n",
"\n",
"\n",
"\n",
"y = np.random.randn(n)\n",
"\n",
"# initial vector\n",
"C0 = np.random.randn(n,k)\n",
"\n",
"# random direction\n",
"Crand = np.random.randn(n,k)\n",
"\n",
"R = 3\n",
"epsilons = np.linspace(-R,R,100)\n",
"\n",
"\n",
"losses = [loss(y, C0+ep*Crand) for ep in epsilons]\n",
"linlosses = [losslin(y, C0+ep*Crand,C0) for ep in epsilons]\n",
"\n",
"\n",
"plt.plot(losses)\n",
"plt.plot(linlosses)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

0 comments on commit ee14e4d

Please sign in to comment.