Autograd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf98e458",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\amirr\\AppData\\Roaming\\Python\\Python38\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dbf50bf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9086, 0.2819, 0.3977],\n",
       "        [0.2706, 0.2237, 0.5631],\n",
       "        [0.8484, 0.2819, 0.5571]], requires_grad=True)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.rand(3, 3, requires_grad=True)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5ed3b1a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = x + 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "807671f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2.9086, 2.2819, 2.3977],\n",
       "        [2.2706, 2.2237, 2.5631],\n",
       "        [2.8484, 2.2819, 2.5571]], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "012f5d58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[16.9203, 10.4141, 11.4979],\n",
       "        [10.3114,  9.8895, 13.1394],\n",
       "        [16.2270, 10.4139, 13.0779]], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z = y * y * 2\n",
    "z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dea1eb9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(12.4324, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = z.mean()\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f678bf03",
   "metadata": {},
   "outputs": [],
   "source": [
    "m.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7ac68679",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.2927, 1.0142, 1.0656],\n",
       "        [1.0092, 0.9883, 1.1392],\n",
       "        [1.2660, 1.0142, 1.1365]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "958d3b09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0227, 0.8076],\n",
       "        [0.1830, 0.6825]], requires_grad=True)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1 = torch.rand(2, 2, requires_grad=True)\n",
    "x1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "04fbae13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[5.1653e-04, 6.5223e-01],\n",
       "        [3.3472e-02, 4.6574e-01]], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1 = x1 * x1\n",
    "y1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a2a5e7b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9896],\n",
       "        [0.6690]], requires_grad=True)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x2 = torch.rand(2, 1, requires_grad=True)\n",
    "x2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b44f0d8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.4268]], grad_fn=<MmBackward0>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y2 = torch.matmul(x2.T, x2)\n",
    "y2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "16d283e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "y2.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "69ee4f0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.9791],\n",
       "        [1.3380]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x2.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "1607a22b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.9791],\n",
       "        [1.3380]], grad_fn=<MulBackward0>)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2 * x2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "046c1212",
   "metadata": {},
   "outputs": [],
   "source": [
    "w = torch.tensor(0, requires_grad=True, dtype=torch.float32)\n",
    "b = torch.tensor(0, requires_grad=True, dtype=torch.float32)\n",
    "X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)\n",
    "y = torch.tensor([3, 5, 7, 9], dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "d0b4c0fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(x):\n",
    "    return w * x + b\n",
    "\n",
    "def loss(y, yp):\n",
    "    return ((y - yp) ** 2).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "c5e94ef4",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(100000):\n",
    "    yp = forward(X)\n",
    "    l = loss(y, yp)\n",
    "    l.backward()\n",
    "    with torch.no_grad():\n",
    "        w -=  0.01 * w.grad\n",
    "        b -= 0.01 * b.grad\n",
    "    w.grad.zero_()\n",
    "    b.grad.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "9271199c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.000007390975952, 0.9999802112579346)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w.item(), b.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "ce35e224",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6.7757e-11, grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}