1-2_Basic_Autograd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scalar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.])\n"
     ]
    }
   ],
   "source": [
    "scalar1 = torch.tensor([1.])\n",
    "print(scalar1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([3.])\n"
     ]
    }
   ],
   "source": [
    "scalar2 = torch.tensor([3.])    \n",
    "print(scalar2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([4.])\n"
     ]
    }
   ],
   "source": [
    "add_scalar = scalar1 + scalar2\n",
    "print(add_scalar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-2.])\n"
     ]
    }
   ],
   "source": [
    "sub_scalar = scalar1 - scalar2\n",
    "print(sub_scalar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([3.])\n"
     ]
    }
   ],
   "source": [
    "mul_scalar = scalar1 * scalar2\n",
    "print(mul_scalar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.3333])\n"
     ]
    }
   ],
   "source": [
    "div_scalar = scalar1 / scalar2\n",
    "print(div_scalar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.add(scalar1, scalar2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-2.])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sub(scalar1, scalar2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3.])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.mul(scalar1, scalar2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.3333])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.div(scalar1, scalar2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1., 2., 3.])\n"
     ]
    }
   ],
   "source": [
    "vector1 = torch.tensor([1., 2., 3.])\n",
    "print(vector1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([4., 5., 6.])\n"
     ]
    }
   ],
   "source": [
    "vector2 = torch.tensor([4., 5., 6.])\n",
    "print(vector2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([5., 7., 9.])\n"
     ]
    }
   ],
   "source": [
    "add_vector = vector1 + vector2\n",
    "print(add_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-3., -3., -3.])\n"
     ]
    }
   ],
   "source": [
    "sub_vector = vector1 - vector2\n",
    "print(sub_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 4., 10., 18.])\n"
     ]
    }
   ],
   "source": [
    "mul_vector = vector1 * vector2\n",
    "print(mul_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.2500, 0.4000, 0.5000])\n"
     ]
    }
   ],
   "source": [
    "div_vector = vector1 / vector2\n",
    "print(div_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5., 7., 9.])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.add(vector1, vector2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-3., -3., -3.])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sub(vector1, vector2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 4., 10., 18.])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.mul(vector1, vector2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.2500, 0.4000, 0.5000])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.div(vector1, vector2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(32.)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.dot(vector1, vector2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 2.],\n",
      "        [3., 4.]])\n"
     ]
    }
   ],
   "source": [
    "matrix1 = torch.tensor([[1., 2.], [3., 4.]])\n",
    "print(matrix1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[5., 6.],\n",
      "        [7., 8.]])\n"
     ]
    }
   ],
   "source": [
    "matrix2 = torch.tensor([[5., 6.], [7., 8.]])\n",
    "print(matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 6.,  8.],\n",
      "        [10., 12.]])\n"
     ]
    }
   ],
   "source": [
    "sum_matrix = matrix1 + matrix2\n",
    "print(sum_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-4., -4.],\n",
      "        [-4., -4.]])\n"
     ]
    }
   ],
   "source": [
    "sub_matrix = matrix1 - matrix2\n",
    "print(sub_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 5., 12.],\n",
      "        [21., 32.]])\n"
     ]
    }
   ],
   "source": [
    "mul_matrix = matrix1 * matrix2\n",
    "print(mul_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.2000, 0.3333],\n",
      "        [0.4286, 0.5000]])\n"
     ]
    }
   ],
   "source": [
    "div_matrix = matrix1 / matrix2\n",
    "print(div_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 6.,  8.],\n",
       "        [10., 12.]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.add(matrix1, matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-4., -4.],\n",
       "        [-4., -4.]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sub(matrix1, matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 5., 12.],\n",
       "        [21., 32.]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.mul(matrix1, matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2000, 0.3333],\n",
       "        [0.4286, 0.5000]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.div(matrix1, matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[19., 22.],\n",
       "        [43., 50.]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(matrix1, matrix2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[1., 2.],\n",
      "         [3., 4.]],\n",
      "\n",
      "        [[5., 6.],\n",
      "         [7., 8.]]])\n"
     ]
    }
   ],
   "source": [
    "tensor1 = torch.tensor([[[1., 2.], [3., 4.]], [[5., 6.], [7., 8.]]])\n",
    "print(tensor1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[ 9., 10.],\n",
      "         [11., 12.]],\n",
      "\n",
      "        [[13., 14.],\n",
      "         [15., 16.]]])\n"
     ]
    }
   ],
   "source": [
    "tensor2 = torch.tensor([[[9., 10.], [11., 12.]], [[13., 14.], [15., 16.]]])\n",
    "print(tensor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[10., 12.],\n",
      "         [14., 16.]],\n",
      "\n",
      "        [[18., 20.],\n",
      "         [22., 24.]]])\n"
     ]
    }
   ],
   "source": [
    "sum_tensor = tensor1 + tensor2\n",
    "print(sum_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[-8., -8.],\n",
      "         [-8., -8.]],\n",
      "\n",
      "        [[-8., -8.],\n",
      "         [-8., -8.]]])\n"
     ]
    }
   ],
   "source": [
    "sub_tensor = tensor1 - tensor2\n",
    "print(sub_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[  9.,  20.],\n",
      "         [ 33.,  48.]],\n",
      "\n",
      "        [[ 65.,  84.],\n",
      "         [105., 128.]]])\n"
     ]
    }
   ],
   "source": [
    "mul_tensor = tensor1 * tensor2\n",
    "print(mul_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[0.1111, 0.2000],\n",
      "         [0.2727, 0.3333]],\n",
      "\n",
      "        [[0.3846, 0.4286],\n",
      "         [0.4667, 0.5000]]])\n"
     ]
    }
   ],
   "source": [
    "div_tensor = tensor1 / tensor2\n",
    "print(div_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 31.,  34.],\n",
       "         [ 71.,  78.]],\n",
       "\n",
       "        [[155., 166.],\n",
       "         [211., 226.]]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(tensor1, tensor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[10., 12.],\n",
       "         [14., 16.]],\n",
       "\n",
       "        [[18., 20.],\n",
       "         [22., 24.]]])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.add(tensor1, tensor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-8., -8.],\n",
       "         [-8., -8.]],\n",
       "\n",
       "        [[-8., -8.],\n",
       "         [-8., -8.]]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.sub(tensor1, tensor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[  9.,  20.],\n",
       "         [ 33.,  48.]],\n",
       "\n",
       "        [[ 65.,  84.],\n",
       "         [105., 128.]]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.mul(tensor1, tensor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.1111, 0.2000],\n",
       "         [0.2727, 0.3333]],\n",
       "\n",
       "        [[0.3846, 0.4286],\n",
       "         [0.4667, 0.5000]]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.div(tensor1, tensor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1., 2.],\n",
       "         [3., 4.]],\n",
       "\n",
       "        [[5., 6.],\n",
       "         [7., 8.]]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 9., 10.],\n",
       "         [11., 12.]],\n",
       "\n",
       "        [[13., 14.],\n",
       "         [15., 16.]]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 31.,  34.],\n",
       "         [ 71.,  78.]],\n",
       "\n",
       "        [[155., 166.],\n",
       "         [211., 226.]]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(tensor1, tensor2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration:  100 \t Loss:  527.76318359375\n",
      "Iteration:  200 \t Loss:  3.209841728210449\n",
      "Iteration:  300 \t Loss:  0.03574322536587715\n",
      "Iteration:  400 \t Loss:  0.0007254641968756914\n",
      "Iteration:  500 \t Loss:  8.230483217630535e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    DEVICE = torch.device('cuda')\n",
    "else:\n",
    "    DEVICE = torch.device('cpu')\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "INPUT_SIZE = 1000\n",
    "HIDDEN_SIZE = 100\n",
    "OUTPUT_SIZE = 10\n",
    "\n",
    "x = torch.randn(BATCH_SIZE, \n",
    "                INPUT_SIZE, \n",
    "                device = DEVICE, \n",
    "                dtype = torch.float, \n",
    "                requires_grad = False)  \n",
    "\n",
    "y = torch.randn(BATCH_SIZE, \n",
    "                OUTPUT_SIZE, \n",
    "                device = DEVICE,\n",
    "                dtype = torch.float, \n",
    "                requires_grad = False)  \n",
    "\n",
    "w1 = torch.randn(INPUT_SIZE, \n",
    "                 HIDDEN_SIZE, \n",
    "                 device = DEVICE, \n",
    "                 dtype = torch.float,\n",
    "                 requires_grad = True)  \n",
    "\n",
    "w2 = torch.randn(HIDDEN_SIZE,\n",
    "                 OUTPUT_SIZE, \n",
    "                 device = DEVICE,\n",
    "                 dtype = torch.float,\n",
    "                 requires_grad = True)  \n",
    "\n",
    "learning_rate = 1e-6                                           \n",
    "for t in range(1, 501):                                        \n",
    "    y_pred = x.mm(w1).clamp(min = 0).mm(w2)                    \n",
    "\n",
    "    loss = (y_pred - y).pow(2).sum()                          \n",
    "    if t % 100 == 0:\n",
    "        print(\"Iteration: \", t, \"\\t\", \"Loss: \", loss.item()) \n",
    "    loss.backward()                                           \n",
    "\n",
    "    with torch.no_grad():                                      \n",
    "        w1 -= learning_rate * w1.grad                          \n",
    "        w2 -= learning_rate * w2.grad                          \n",
    "\n",
    "        w1.grad.zero_()                                      \n",
    "        w2.grad.zero_()                                      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Iteration:  100 \t Loss:  976.0719604492188\n",
    "# Iteration:  200 \t Loss:  5.061180114746094\n",
    "# Iteration:  300 \t Loss:  0.04102771729230881\n",
    "# Iteration:  400 \t Loss:  0.0006718397489748895\n",
    "# Iteration:  500 \t Loss:  6.94335249136202e-05"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}