L10/code/L2-log-reg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "STAT 453: Deep Learning (Spring 2021)  \n",
    "Instructor: Sebastian Raschka (sraschka@wisc.edu)  \n",
    "\n",
    "Course website: http://pages.stat.wisc.edu/~sraschka/teaching/stat453-ss2021/  \n",
    "GitHub repository: https://github.com/rasbt/stat453-deep-learning-ss21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sebastian Raschka\n",
      "\n",
      "Python implementation: CPython\n",
      "Python version       : 3.9.2\n",
      "IPython version      : 7.20.0\n",
      "\n",
      "torch: 1.9.0a0+d819a21\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Runs on CPU or GPU (if available)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression with L2 Regularization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of *classic* logistic regression for binary class labels."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "LAMBDA = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing a toy dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaoAAACnCAYAAABAZhicAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZYklEQVR4nO3df7AdZX3H8feXcNFLpVwcYpFLYujUSatCid6htHFaG9QgBoxU8WerxWmGP5xqxEgiHYdRZwyTKVSqrZMq41ipJo4QKOIAGh2nTHFMID/AgEWLkgstsZL4g6vcJN/+cc4m5567u2d3z56zz+75vGYyufecc/c892Q3332+z/d5HnN3REREQnVC1Q0QERFJo0AlIiJBU6ASEZGgKVCJiEjQFKhERCRoClQiIhK0E6t409NPP92XLFlSxVuLlGLnzp0/dfeFVbcjomtKmiDpuqokUC1ZsoQdO3ZU8dYipTCzH1fdhk66pqQJkq4rpf5ERCRolfSopF62PTDNprse4YmDM5w5Mc66lUtZvWyy79dKOjNbAOwApt19VdXtEamKApWk2vbANBtu2cvM7BEApg/OsOGWvQDzAlCe10om7wP2Ab9ddUNEqqTUn6TadNcjxwJPZGb2CJvueqSv10o6MzsLeD3w2arbIlI1BSpJ9cTBmcyP53mt9PQPwIeAo0kvMLM1ZrbDzHYcOHBgaA0TGTYFKkl15sR45sfzvFaSmdkq4Cl335n2Onff7O5T7j61cGEwlfIipVOgklTrVi5lfGzBnMfGxxawbuXSvl4rqZYDl5rZY8CXgRVm9sVqmyRSHRVTSKqoCCKpkq+7yu8vXjHJtx4+oKq/Prj7BmADgJm9Cvigu7+zyjaJVEmBSnpavWwyNtjEVfl9dec0n7jsnKCDk0roRepFqT8prI5VflFwnT44g3O8hH7bA9NVNy2Wu39bc6hk1KlHJYWVWeU3rF5OUnC9autu1m7ZpR6WSIDUo5LCyqryG2YvJymIHnGvRQ9LZBQpUElhZVX5DTOFmCWIhp6+FBk1ClSS27YHplm+cTtrt+ziuWMnMDE+hgGTE+OFCimGOVE4LrgO671FpBiNUUku3ZV+Tz8zy/jYAm54y3mFx3XOnBhnOiYwDGKicHe5/QlmHHEfynuLSDEKVJIorsAhLU1XNFCtW7l0TvCDwU4U7iy37w68g35vEclPqT+JlVTgENfzgf5SZauXTfKJy85hcmJ8TgoRYPnG7Zy9/mss37i9lAKHKG0ZHROY896nnTzGc048gbVbdpX2niLSH/WoJFZSz2nBgFJl3ZOKB7FlSNIxP3HZOdy7foW2KREJlHpUEiutjDtPpV93DyZrDyVtvlPRXk6v6sI6TmAWGQUKVBIrqYcUpeW603RpSywVmR+VFiiLznPqVV2obUpEwqRAJbHiyrjHFhi/+s1h1m7ZBcANbzmPe9evYPWyydieU68eSlpvKy2VWLSX02uCsrYpEQlTKYHKzG4ys6fM7MEyjifV6y5wOO3kMXA4ODM7r3dUpPCiV2+r13ynIr2cXhOUtU2JSJjKKqb4PPAp4AslHU8C0FngsHzjdp5+ZnbO8509m7yFF73K3KP3vWrr7tKKN3ptWdLreRGpRimByt2/Y2ZLyjiWhKnI+E1UeBE3RylKH6YdLwoQZc5zStqyJOvzIjJ8QxujMrM1ZrbDzHYcOHBgWG8rJUkbvylSeJFnPOi5Y8dP04nxseD3uxKRcg1tHpW7bwY2A0xNTc3P5UjQeq0ekfRcUg8ly2oUcatG/OrZw1x7+0NztuQApetEmkwTfiVR3m3m8wSLLONBceNYs0ecgzOtsbLpgzOs+8pusNbj0WOapCvSLApUEivvNvNFxnZ6/UyWyr7Zo/M75/2uPSgiYSmrPP1LwH8CS81sv5m9p4zjSnVCWKWhn/lLmqQr0hxlVf29rYzjSDiS/qOfPjjD8o3b56TpulOEf/77C1NThFnFjWNlVedJuma2iNZUjzOAo8Bmd/9kta0SqY5SfxIraY8omDsOBMxLEX7xvp/EvrZIahCOj2NNnDzGL399eE66b+wEmzNGBY2YpHsYuMrd7zezU4CdZnaPu3+/6oaJVEGBSmL16s2kTfZNem2RXlXcqurdBRjQrKo/d38SeLL99S/MbB8wCShQyUhSoJJYnb2ZMvagyvPauGCUtHrEprseYd3Kpdy7fkXm49dJeyL9MuC7FTdFpDJalFYSrV42yb3rVzCZMjl34uSxTMfKOmYUtwbg2i27+LttexOfL7qaeujM7HnAV4H3u/vPY57XJHoZCQpU0lPaYq0xy/DNk2fMKK7a0IGb7/tJphXZm8LMxmgFqZvd/Za417j7ZnefcvephQsXDreBIkOkQCU9JW0Vv3rZJIdmZhN/rtd+VXGSUoTO8XRfnp+rIzMz4HPAPne/vur2iFRNY1SSKOrBTB+cObYS+mTXmFFSdeDkxHihcaO0asNozCru+TqXo8dYDvwlsNfMdrUf+7C731ldkySvzrHWiZPHcIdDM7ONKPgZNvWoJFbnWBBwbKuNLPtG9VMevm7lUizhuegCb/qeUe7+H+5u7n6uu5/X/qMgVSPdY6lPPzM7Zy+3tVt2sSRmw1CJp0DVVHu2wg0vg2snWn/v2Zrrx+PGgiKdY0JpacEiVi+b5B0XLJ4XrDoXuS3z/UQGIe36gVYqG5pdDFQmpf6aaM9W+Pe/hdl2iuzQ463vAc69PNMheo35dO8bVWag+Pjqc5h60fNj50st37j92GM3vOU8BSgJUp4xU61N2ZsCVRN986PHg1Rkdqb1eMZAlTZWFD0/SHETfbtXwNAq6RKqXtdPtyYVAw2CUn9NdGh/vsdjxI0FRbKMCW17YJrlG7dzdkl5+FEpS5dmSLt+4jSsGKh06lE10alntdJ9cY9n1L0yRVLVX5xB9H5GoSxdmiNunUp3ODgzi3F8jAqaVww0CApUTXThR+aOUQGMjbcez6Ho2FNa76dooBqRsnSpubTlv/K8RuZqdqDas7U1LnNof6s3ceFHMo/R1Fr0O1b0uw+i95Nl63qRKmXNJJRdfDQKmhuoSqh8q7VzL6/s9xxE7ydp63qYWwmou1OpwrYHprlq6+5j8w0jM7NHuGrrbtZu2aXzsw/NDVQlVL5JMUV7P71SIqoElBBF52F3kIp0T5YHnZ95Nbfqr4TKt4Hrc1JuqNIm5SZVAxZZFV2VgBKCXpN7O+n8LKa5PaoSKt8GquGpybg8fFoPqEgBhioBJQR5z7e016vQIl5ze1QXfqRV6dapQOXbwKSlJhsqLRgVCTpJY16qBJRhynu+Jb1+lPZay6u5gercy+GSG+HURYC1/r7kxnB6K3VITfapO83Xa1X0OGn/CYzCArUSvqTz8J0XLM51fiqVnay5qT+otPKtp9BTk32KS/N1T3SMRCmOvAUYSZWASpXIMKWdh3FrViqVnV+zA1XISpqUO2xZc+hJO/UmzcovGnQ0J0VCkHQeJo3Vxp3nmtSeTIGqKhVPyi0iTzl42k69kxPjscEorvxcc6SkruICEpB4DWlSe7JSApWZXQR8ElgAfNbdN5Zx3MYLLTXZYyWPPJV5/e78qzlSUmdx5+/aLbsYHzuBmdmjc14bXUPRdaFU9nx9ByozWwB8GngNsB/4npnd7u7f7/fYMkQZyuXz5ND7vTscxHqBIgMRc4O36a7TY1Pfz3QFqcj0wRmWb9zOupVLM93IjZoyqv7OBx519x+5+7PAl4E3lHBcGaYM5fJ5KvP63YlXA8tSC9EN3qHHAT92gzf183tyH0rl6MnKSP1NAp3la/uBP+p+kZmtAdYALF68uIS3lVJlKJfP20vqp9BBA8tSCwk3eBtO+gq3/fqVuQ+XljUY5cnAZfSoLOaxeVXI7r7Z3afcfWrhwoUlvK2UKqksvuPxfntJeYz6HCkzu8jMHjGzR81sfdXtkQQJN3i/w09j/2MEmBgfYzLlhisuazDqk4HLCFT7gUUd358FPFHCcesr5DX8ktqWcSWP1csmuXf9Cv574+u5d/2Kgd3RDTMohqZj3Pd1wEuAt5nZS6pt1Qgoct0m3ODZqWfxjgsWzwtW42MLuPbSl3Lv+hWJwSouazDqk4HLSP19D3ixmZ0NTANvBd5ewnHrKeQ1/LK0LaBy+RGeI3Vs3BfAzKJxXxUoDUrR6zZlPuTHzz0ndcJvnlT6qI/Z9h2o3P2wmb0XuItWefpN7v5Q3y2rq5C3F+nVttDK5UdXpnFfKVHR67bHDV7azVaeSe6jPmZbyjwqd78TuLOMYwWj6O7AIa/hN8C2jfJA7wBkGvdVgVKJ+rk2+rjBy5o1GPXJwM1dlLYfCSWn/eSsg1jDb0BtG/WB3gHINO6rAqUShXzdMtpjtqBAFa+fLTj63V5kkIUYA9r6ZNQHegfg2LivmZ1Ea9z39orb1GyhbwvE8AqZQqRA1SkKEnGrmkP2NEDR7UX66cllMaCtT0Z9oLds7n4YiMZ99wFbR3rcN68iN3uhbws04rQobaS76idO1jRA0Zz1MAoxBlAwMeoDvYPQyHHfYein6lbFRMFSjyoSFyQ6DSMNEHIhRopRn5wrARnBnbNHgQJVJC0YDCsNEPiAbpJRH+iVgBS52StzXDjkyf41ptRfJHHH3UWw9sHhtCFu8iDWatcNL6t8Am6aEZ6cKyHJu3N2XKrwlr+Br18Nr7su3/UW8mT/mgurR1Xl3UgIVT9zBnRhzn64ZRdWiDRR3us4KeU/87P815vSjgMTTqAadMVbL6FU/Zx7easHd+oi5s3x1Ekvki7vdZyWEoyut6w30DUdY66DcFJ/ISw9FFLVj056kWLyXMdJqcJIdMOcJZ2XN+0omYXTo9J/zHPVtLBCpFbiUoWdbEH2dF4IwwcNFU6gGuR/zHWsxNFJL5JNP9d3lCocf/7858bGwY/Mfxzm3kBH73/LGjhxvH0sTRouUzipv5Tl8vsSaiVOr0VvA9x2QyQ4ea7vzmtu/LTWYzNPt66t113X+r77evvmR9PTed3vP/Oz1v9bl23WtVoic5+3KPPATU1N+Y4dO+Y/UXTF8jRJSyLlLTsvs21xq2DYAsDBj7a+fsW7YdX1xY4vA2dmO919qup2RBKvqabLen33WnlmbDy+9xP3c52vLev/FwGSr6twelQwmGKGMsa+yu6VxRWOdKYY/Ajs+Fzr61XXDyaAizRBlut7z1a49crkNB4kF271ymxobH0owgpUg1BGJU7ZFYlZT+Kdn4fFF4SZuhQJQa/rO7rJTAtSkaTrMu0GWpV+QxFOMcWglFGUUPZdU9aT2I9oEqFIml7Xd681PDsVCS4qehqK5geqMibyll2R2KskNmILlFoQSdPr+s56nRQNLqEsFNBwzU/9Qf9jX0UrEpPGlrrz3mMnw+yv5v/8K94N/3V379SCxrBklBVJzdkJ8NyJ41V//VwzIS0U0FCjEaj6VaRUvFcBRvfJfccHWmNSfmRu1V9S1VEUJEMtvxcJQdJNpno9taJAlUWRHkveAoxV18eXo/cKkiEsPSUSKs1HbAQFql6K9lgSt7NPWVcsSVpqQWNYIumUmqu95hdT9Kto1Z0tyPd4UVoTUEQaToGql6I9lqR5G1nmc+Sh8lgRabi+ApWZvdnMHjKzo2YWzHIypSraYzm2+WHGx4uKK4/9w7e3enx1WoRXADCzTWb2sJntMbNbzWyi6jaJVK3fHtWDwGXAd0poS5iK9liy/lwZK7tHmy1ee7B1/N3/Nn8Dyjs+UL8V5EfTPcDL3P1c4AfAhorbU3913D1B5uirmMLd9wGYWTmtCVHRqqEsPzeI0vKkMbUdNzFvW/t+3kcGwt3v7vj2PuBNVbWlETR9oxGGVvVnZmuANQCLFy8e1tuWo2jVUK+fG0RpeeLYWcK29rpYQ3YFsKXqRtSapm80Qs9AZWbfAM6Ieeoad78t6xu5+2ZgM7S2JMjcwpD1uyLEIErLe22tXdb7SGFZrikzuwY4DNyccpz63vwNi6ZvNELPQOXurx5GQ2qnjJRCYlDxVi69yMTEuJn4GPN6VNH7y9D1uqbM7F3AKuBCT9kwrpE3f2XT6uaNoPL0opJSCrdemX3QNm1x2ijw5R34jasCnLpCJew1YWYXAVcDl7r7M1W3p/Y0faMR+hqjMrM3Av8ILAS+Zma73H1lKS0LXVLqIJonlaWHNafgIuaur2guPW5sbPEFWkamHj4FPAe4p12kdJ+7X1ltk2pMSyg1Qlhb0ddJ0hbU3bJuSX3tBLHpOaxVdi5B0Vb0IuVLuq6U+isq655SWQdttRSSiEgsBaqiuseCktbwyxpolEsXEYml1dP70TkW1GvfqCzHgvm5dGinGZVfF5HRpEBVljIGbbuLIPKWwGunXxFpIAWqMpW9702eWfVaKkZEGkpjVCHLM6u+6L5ZIiKBU6AK1Z6tYAn/PHEFGloqRkQaSoEqRFEaL26TxaQCDZW3i0hDKVCFKC6NB60S+EtujB9zGnR5u/b0EZGKqJgiRInLMx3NuBxTyVV/KtQQkQopUIWo6IrPZVcdRrSnj4hUSKm/EIW2SoUKNUSkQgpUZSprHCduq46ksalhUKGGiFRIqb+ylD2OU2Yar98VK+I2Y9Q6hCIyJOpRlSXUCbdRAD30OODFNmQMrYcnIiNFPaqyhDqOU1YhxKAKNUREelCPqiyhjuOEGkBFRDJSoEqStzAitEq9SKgBVEQkIwWqOEXGdUIdxxl2ANUKFiJSMo1RxSk6rhPiOM4gV6zophUsRGQAFKjihDiu00+J+bACqFawKI2ZfRDYBCx0959W3R6RKin1Fye0cZ0ySsyHIcQAX0Nmtgh4DfCTqtsiEgIFqjihFUaEOkerW2gBvr5uAD4EeNUNEQmBAlWc0Aoj6tJTCS3A15CZXQpMu/vuqtsiEoq+xqjMbBNwCfAs8EPgr939YAntql5IhRFFV1MftmEWbtSYmX0DOCPmqWuADwOvzXicNcAagMWLF5fWPpHQ9FtMcQ+wwd0Pm9l1wAbg6v6bJXPUaa29kAJ8oNz91XGPm9k5wNnAbjMDOAu438zOd/f/iTnOZmAzwNTUlNKE0lh9BSp3v7vj2/uAN/XXHImlnspIcPe9wAui783sMWBKVX8y6sosT78C2JL0pNIUfVJPRURGVM9AlZZPd/fb2q+5BjgM3Jx0HKUpRPJx9yVVt0EkBD0DVVI+PWJm7wJWARe6e3MDUL97OomISCH9Vv1dRKt44s/c/ZlymhQgLQ0kIlKZfudRfQo4BbjHzHaZ2WdKaFN46jLhVkSkgfqt+vu9shoStLpMuBURaSCtTJGFlgYSEamMAlUWWhpIRKQyClRZhLb2n4jICNF+VFlpwq2ISCXUoxIRkaBZFXN0zewA8OMBv83pQGhrpIXYJgizXSG2CY6360XuvrDqxkQKXlOhfsbd6tDOOrQRwm9n7HVVSaAaBjPb4e5TVbejU4htgjDbFWKbINx2FVGX36UO7axDG6E+7eym1J+IiARNgUpERILW5EC1ueoGxAixTRBmu0JsE4TbriLq8rvUoZ11aCPUp51zNHaMSkREmqHJPSoREWmARgcqM9tkZg+b2R4zu9XMJgJo05vN7CEzO2pmlVbfmNlFZvaImT1qZuurbEvEzG4ys6fM7MGq2xIxs0Vm9i0z29f+t3tf1W0qwsyuNbPp9k4Hu8zs4oTXVXZeZL1mzewxM9vb/j12DLF9qZ+NtdzYfn6Pmb18WG3raEPP89XMXmVmhzrOhbDXg3P3xv4BXguc2P76OuC6ANr0B8BS4NvAVIXtWAD8EPhd4CRgN/CSAD6fPwVeDjxYdVs62vRC4OXtr08BfhDCZ1Xg97gW+GDI50XWaxZ4DDh9yJ9fz88GuBj4OmDABcB3K/h37nm+Aq8C7hh224r+aXSPyt3vdvfD7W/vAypf7tzd97n7I1W3AzgfeNTdf+TuzwJfBt5QcZtw9+8AP6u6HZ3c/Ul3v7/99S+AfcBkta0amErPixCv2Q5ZPps3AF/wlvuACTN74TAb2cTztdGBqssVtO50pGUSeLzj+/3U/GQeBjNbAiwDvltxU4p6bzsldZOZnRbzfEjnRdo168DdZrbTzNYMqT1ZPpuQPr9e5+sfm9luM/u6mb10uC3Lp/aL0prZN4AzYp66xt1va7/mGuAwcHMobQqAxTymEtAUZvY84KvA+93951W3J07auQf8M/AxWv/OHwP+nlYwmHOImJ8t9bwo6Zpd7u5PmNkLaO0w/nC7Nz5IWT6bYK6rHufr/bSWK/ple6xyG/DiITcxs9oHKnd/ddrzZvYuYBVwobeTs1W3KRD7gUUd358FPFFRW4JnZmO0Lvqb3f2WqtuTJOu5Z2b/AtwR89TAz4syrll3f6L991NmdiuttNygA1WWzyaI66rX+doZuNz9TjP7JzM73d2DXAew0ak/M7sIuBq41N2fqbo9gfke8GIzO9vMTgLeCtxecZuCZGYGfA7Y5+7XV92eorrGSt4IxFVWVnpeZLlmzey3zOyU6GtaBRjDqBLN8tncDvxVu/rvAuCQuz85hLYdk+V8NbMz2q/DzM6nFQv+b3itzKnqao5B/gEepZUv3tX+85kA2vRGWnddvwH+F7irwrZcTKsi6Ie00i4h/Jt9CXgSmG1/Tu8JoE2vpJW+2dNxLl1cdbsK/B7/Cuxt/x63Ay9sP34mcGcI50XSNdvZRlpVd7vbfx4aZhvjPhvgSuDK9tcGfLr9/F4qqOxNOl+72vne9me3m1bRyp9UfX6m/dHKFCIiErRGp/5ERKT+FKhERCRoClQiIhI0BSoREQmaApWIiARNgUpERIKmQCUiIkFToBIRkaD9P7hnPOP3NBpOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 504x180 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "##########################\n",
    "### DATASET\n",
    "##########################\n",
    "\n",
    "data = np.genfromtxt('data/toydata.txt', delimiter='\\t')\n",
    "x = data[:, :2].astype(np.float32)\n",
    "y = data[:, 2].astype(np.int64)\n",
    "\n",
    "np.random.seed(123)\n",
    "idx = np.arange(y.shape[0])\n",
    "np.random.shuffle(idx)\n",
    "X_test, y_test = x[idx[:25]], y[idx[:25]]\n",
    "X_train, y_train = x[idx[25:]], y[idx[25:]]\n",
    "mu, std = np.mean(X_train, axis=0), np.std(X_train, axis=0)\n",
    "X_train, X_test = (X_train - mu) / std, (X_test - mu) / std\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, figsize=(7, 2.5))\n",
    "ax[0].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1])\n",
    "ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1])\n",
    "ax[1].scatter(X_test[y_test == 1, 0], X_test[y_test == 1, 1])\n",
    "ax[1].scatter(X_test[y_test == 0, 0], X_test[y_test == 0, 1])\n",
    "plt.xlim([x[:, 0].min()-0.5, x[:, 0].max()+0.5])\n",
    "plt.ylim([x[:, 1].min()-0.5, x[:, 1].max()+0.5])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L2-Regularized Logistic Regression via `weight_decay`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001 | Train ACC: 0.973 | Cost: 0.055\n",
      "Epoch: 002 | Train ACC: 0.973 | Cost: 0.065\n",
      "Epoch: 003 | Train ACC: 0.973 | Cost: 0.080\n",
      "Epoch: 004 | Train ACC: 0.973 | Cost: 0.094\n",
      "Epoch: 005 | Train ACC: 0.973 | Cost: 0.104\n",
      "Epoch: 006 | Train ACC: 0.973 | Cost: 0.108\n",
      "Epoch: 007 | Train ACC: 0.973 | Cost: 0.110\n",
      "Epoch: 008 | Train ACC: 0.973 | Cost: 0.111\n",
      "Epoch: 009 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 010 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 011 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 012 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 013 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 014 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 015 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 016 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 017 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 018 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 019 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 020 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 021 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 022 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 023 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 024 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 025 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 026 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 027 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 028 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 029 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 030 | Train ACC: 0.973 | Cost: 0.112\n",
      "\n",
      "Model parameters:\n",
      "  Weights: Parameter containing:\n",
      "tensor([[1.7546, 1.5997]], requires_grad=True)\n",
      "  Bias: Parameter containing:\n",
      "tensor([-0.0401], requires_grad=True)\n",
      "\n",
      "\n",
      "Test set accuracy: 96.00%\n"
     ]
    }
   ],
   "source": [
    "def custom_where(cond, x_1, x_2):\n",
    "    return (cond * x_1) + ((1-cond) * x_2)\n",
    "\n",
    "\n",
    "class LogisticRegression(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, num_features):\n",
    "        super(LogisticRegression, self).__init__()\n",
    "        self.linear = torch.nn.Linear(num_features, 1)\n",
    "        # initialize weights to zeros here,\n",
    "        # since we used zero weights in the\n",
    "        # manual approach\n",
    "        \n",
    "        self.linear.weight.detach().zero_()\n",
    "        self.linear.bias.detach().zero_()\n",
    "        # Note: the trailing underscore\n",
    "        # means \"in-place operation\" in the context\n",
    "        # of PyTorch\n",
    "        \n",
    "    def forward(self, x):\n",
    "        logits = self.linear(x)\n",
    "        probas = torch.sigmoid(logits)\n",
    "        return probas\n",
    "\n",
    "model = LogisticRegression(num_features=2).to(device)\n",
    "\n",
    "#########################################################\n",
    "## Apply L2 regularization\n",
    "optimizer = torch.optim.SGD(model.parameters(), \n",
    "                            lr=0.1, \n",
    "                            weight_decay=LAMBDA)\n",
    "#-------------------------------------------------------\n",
    "\n",
    "\n",
    "def comp_accuracy(label_var, pred_probas):\n",
    "    pred_labels = custom_where((pred_probas > 0.5).float(), 1, 0).view(-1)\n",
    "    acc = torch.sum(pred_labels == label_var.view(-1)).float() / label_var.size(0)\n",
    "    return acc\n",
    "\n",
    "\n",
    "num_epochs = 30\n",
    "\n",
    "X_train_tensor = torch.tensor(X_train, dtype=torch.float32, device=device)\n",
    "y_train_tensor = torch.tensor(y_train, dtype=torch.float32, device=device).view(-1, 1)\n",
    "\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    #### Compute outputs ####\n",
    "    out = model(X_train_tensor)\n",
    "    \n",
    "    #### Compute gradients ####\n",
    "    cost = F.binary_cross_entropy(out, y_train_tensor, reduction='sum')\n",
    "    optimizer.zero_grad()\n",
    "    cost.backward()\n",
    "    \n",
    "    #### Update weights ####  \n",
    "    optimizer.step()\n",
    "    \n",
    "    #### Logging ####      \n",
    "    pred_probas = model(X_train_tensor)\n",
    "    acc = comp_accuracy(y_train_tensor, pred_probas)\n",
    "    print('Epoch: %03d' % (epoch + 1), end=\"\")\n",
    "    print(' | Train ACC: %.3f' % acc, end=\"\")\n",
    "    print(' | Cost: %.3f' % F.binary_cross_entropy(pred_probas, y_train_tensor))\n",
    "\n",
    "\n",
    "    \n",
    "print('\\nModel parameters:')\n",
    "print('  Weights: %s' % model.linear.weight)\n",
    "print('  Bias: %s' % model.linear.bias)\n",
    "\n",
    "\n",
    "X_test_tensor = torch.tensor(X_test, dtype=torch.float32, device=device)\n",
    "y_test_tensor = torch.tensor(y_test, dtype=torch.float32, device=device)\n",
    "\n",
    "pred_probas = model(X_test_tensor)\n",
    "test_acc = comp_accuracy(y_test_tensor, pred_probas)\n",
    "\n",
    "print('\\n\\nTest set accuracy: %.2f%%' % (test_acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L2-Regularized Logistic Regression via Manual Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001 | Train ACC: 0.973 | Cost: 0.055\n",
      "Epoch: 002 | Train ACC: 0.973 | Cost: 0.065\n",
      "Epoch: 003 | Train ACC: 0.973 | Cost: 0.080\n",
      "Epoch: 004 | Train ACC: 0.973 | Cost: 0.094\n",
      "Epoch: 005 | Train ACC: 0.973 | Cost: 0.104\n",
      "Epoch: 006 | Train ACC: 0.973 | Cost: 0.108\n",
      "Epoch: 007 | Train ACC: 0.973 | Cost: 0.110\n",
      "Epoch: 008 | Train ACC: 0.973 | Cost: 0.111\n",
      "Epoch: 009 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 010 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 011 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 012 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 013 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 014 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 015 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 016 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 017 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 018 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 019 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 020 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 021 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 022 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 023 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 024 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 025 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 026 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 027 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 028 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 029 | Train ACC: 0.973 | Cost: 0.112\n",
      "Epoch: 030 | Train ACC: 0.973 | Cost: 0.112\n",
      "\n",
      "Model parameters:\n",
      "  Weights: Parameter containing:\n",
      "tensor([[1.7546, 1.5997]], requires_grad=True)\n",
      "  Bias: Parameter containing:\n",
      "tensor([-0.0401], requires_grad=True)\n",
      "\n",
      "\n",
      "Test set accuracy: 96.00%\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegression(num_features=2).to(device)\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    #### Compute outputs ####\n",
    "    out = model(X_train_tensor)\n",
    "    \n",
    "    #### Compute gradients ####\n",
    "    \n",
    "    #########################################################\n",
    "    ## Apply L2 regularization (weight decay)\n",
    "    cost = F.binary_cross_entropy(out, y_train_tensor, reduction='sum')\n",
    "    cost = cost + 0.5 * LAMBDA * torch.mm(model.linear.weight,\n",
    "                                          model.linear.weight.t())\n",
    "    \n",
    "    # note that PyTorch also regularizes the bias, hence, if we want\n",
    "    # to reproduce the behavior of SGD's \"weight_decay\" param, we have to add\n",
    "    # the bias term as well: \n",
    "    cost = cost + 0.5 * LAMBDA * model.linear.bias**2\n",
    "    #-------------------------------------------------------\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    cost.backward()\n",
    "    \n",
    "    #### Update weights ####  \n",
    "    optimizer.step()\n",
    "    \n",
    "    #### Logging ####      \n",
    "    pred_probas = model(X_train_tensor)\n",
    "    acc = comp_accuracy(y_train_tensor, pred_probas)\n",
    "    print('Epoch: %03d' % (epoch + 1), end=\"\")\n",
    "    print(' | Train ACC: %.3f' % acc, end=\"\")\n",
    "    print(' | Cost: %.3f' % F.binary_cross_entropy(pred_probas, y_train_tensor))\n",
    "\n",
    "\n",
    "    \n",
    "print('\\nModel parameters:')\n",
    "print('  Weights: %s' % model.linear.weight)\n",
    "print('  Bias: %s' % model.linear.bias)\n",
    "\n",
    "\n",
    "X_test_tensor = torch.tensor(X_test, dtype=torch.float32, device=device)\n",
    "y_test_tensor = torch.tensor(y_test, dtype=torch.float32, device=device)\n",
    "\n",
    "pred_probas = model(X_test_tensor)\n",
    "test_acc = comp_accuracy(y_test_tensor, pred_probas)\n",
    "\n",
    "print('\\n\\nTest set accuracy: %.2f%%' % (test_acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note: for easier comparison we plotted the regular cost, not the regularized cost (strictly, plotting the regularized cost is more useful as the regular ost may not always go down while making \"progress\")**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}