PytorchIntro.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PytorchIntro.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tathatj/Pytorchcodes/blob/master/PytorchIntro.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KPqFTlOiiiKv",
        "colab_type": "text"
      },
      "source": [
        "## Outline\n",
        "* Initialising, slicing, reshaping tensors\n",
        "* Numpy and PyTorch interfacing\n",
        "* GPU support for PyTorch + Enabling GPUs on Google Colab\n",
        "* Speed comparisons, Numpy -- PyTorch -- PyTorch on GPU\n",
        "* Autodiff concepts and application\n",
        "* Writing a basic learning loop using autograd\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j4wIFDnRakTz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WJyLCI8PcPZq",
        "colab_type": "text"
      },
      "source": [
        "## Initialise tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NV2jveDIayX-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.ones(3, 2)\n",
        "print(x)\n",
        "x = torch.zeros(3, 2)\n",
        "print(x)\n",
        "x = torch.rand(3, 2)\n",
        "print(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BuAQVNaPFN1P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.empty(3, 2)\n",
        "print(x)\n",
        "y = torch.zeros_like(x)\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWrOMr-hFbwo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.linspace(0, 1, steps=5)\n",
        "print(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l_QspfvYEtuB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.tensor([[1, 2], \n",
        "                 [3, 4], \n",
        "                 [5, 6]])\n",
        "print(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wKub-KJLcSDJ",
        "colab_type": "text"
      },
      "source": [
        "## Slicing tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UxSlfSVrbH8h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(x.size())\n",
        "print(x[:, 1]) \n",
        "print(x[0, :]) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGWkj2utcrz9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = x[1, 1]\n",
        "print(y)\n",
        "print(y.item())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6YvWGrX0cUpf",
        "colab_type": "text"
      },
      "source": [
        "## Reshaping tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mn1q-Hm7b6hP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(x)\n",
        "y = x.view(2, 3)\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1EbIwPvBF4Lg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = x.view(6,-1) \n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2XxOq0ObdXEC",
        "colab_type": "text"
      },
      "source": [
        "## Simple Tensor Operations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rv4jjqBVdIB2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.ones([3, 2])\n",
        "y = torch.ones([3, 2])\n",
        "z = x + y\n",
        "print(z)\n",
        "z = x - y\n",
        "print(z)\n",
        "z = x * y\n",
        "print(z)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dVHnXB78dl8s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "z = y.add(x)\n",
        "print(z)\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LewBBuz_eL1m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "z = y.add_(x)\n",
        "print(z)\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PDuBSdzTc2Bq",
        "colab_type": "text"
      },
      "source": [
        "## Numpy <> PyTorch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NlvqO8_1ccML",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_np = x.numpy()\n",
        "print(type(x), type(x_np))\n",
        "print(x_np)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tLhS3Hrmc-M2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = np.random.randn(5)\n",
        "print(a)\n",
        "a_pt = torch.from_numpy(a)\n",
        "print(type(a), type(a_pt))\n",
        "print(a_pt)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kwZhRYVtdp-X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.add(a, 1, out=a)\n",
        "print(a)\n",
        "print(a_pt) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6z-Mhf2hewcU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(100):\n",
        "  a = np.random.randn(100,100)\n",
        "  b = np.random.randn(100,100)\n",
        "  c = np.matmul(a, b)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFzIX2qge3x9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(100):\n",
        "  a = torch.randn([100, 100])\n",
        "  b = torch.randn([100, 100])\n",
        "  c = torch.matmul(a, b)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pdat0Hnm6hGA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a = np.random.randn(10000,10000)\n",
        "  b = np.random.randn(10000,10000)\n",
        "  c = a + b"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XlRx5OKl6kEq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a = torch.randn([10000, 10000])\n",
        "  b = torch.randn([10000, 10000])\n",
        "  c = a + b"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "de5YwtfUgMWO",
        "colab_type": "text"
      },
      "source": [
        "## CUDA support"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-nI4nYcWgY1B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(torch.cuda.device_count())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_3E-PMC1gfKU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(torch.cuda.device(0))\n",
        "print(torch.cuda.get_device_name(0))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_eZYxVpMgor4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cuda0 = torch.device('cuda:0')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j1r7y57x9JZU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = torch.ones(3, 2, device=cuda0)\n",
        "b = torch.ones(3, 2, device=cuda0)\n",
        "c = a + b\n",
        "print(c)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wSt3W1s-_Gc3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(a)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sVfOAU2EfTXB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a = np.random.randn(10000,10000)\n",
        "  b = np.random.randn(10000,10000)\n",
        "  np.add(b, a)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9GLUek5hCfn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a_cpu = torch.randn([10000, 10000])\n",
        "  b_cpu = torch.randn([10000, 10000])\n",
        "  b_cpu.add_(a_cpu)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FqSYioGrgyMI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a = torch.randn([10000, 10000], device=cuda0)\n",
        "  b = torch.randn([10000, 10000], device=cuda0)\n",
        "  b.add_(a)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kjsl8xRFjPtT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a = np.random.randn(10000,10000)\n",
        "  b = np.random.randn(10000,10000)\n",
        "  np.matmul(b, a)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avFqbCgXjT3F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a_cpu = torch.randn([10000, 10000])\n",
        "  b_cpu = torch.randn([10000, 10000])\n",
        "  torch.matmul(a_cpu, b_cpu)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hFfMhN2gjlZJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  a = torch.randn([10000, 10000], device=cuda0)\n",
        "  b = torch.randn([10000, 10000], device=cuda0)\n",
        "  torch.matmul(a, b)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P_6TU64Gi7jv",
        "colab_type": "text"
      },
      "source": [
        "## Autodiff"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PjySsLMThEX7",
        "colab_type": "code",
        "outputId": "b1cef456-8281-42b0-8205-d7b988c3c383",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "x = torch.ones([3, 2], requires_grad=True)\n",
        "print(x)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[1., 1.],\n",
            "        [1., 1.],\n",
            "        [1., 1.]], requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "neb3oFWBjAtJ",
        "colab_type": "code",
        "outputId": "a9e1faf0-841d-41d1-f9c4-e429e1fdd0bc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "y = x + 5\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[6., 6.],\n",
            "        [6., 6.],\n",
            "        [6., 6.]], grad_fn=<AddBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5M0pnstAjLa-",
        "colab_type": "code",
        "outputId": "b9861325-9ec6-41bb-995f-ae54210628c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "z = y*y + 1\n",
        "print(z)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[37., 37.],\n",
            "        [37., 37.],\n",
            "        [37., 37.]], grad_fn=<AddBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wHHDSmiUkMOw",
        "colab_type": "code",
        "outputId": "93374258-d506-4eba-c359-13fc160d0123",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "t = torch.sum(z)\n",
        "print(t)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(222., grad_fn=<SumBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AXj896azkM_S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t.backward()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wSYAcNN1lAWS",
        "colab_type": "code",
        "outputId": "c82aa737-eb40-45f5-ad44-5e28484d5e72",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "print(x.grad)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[12., 12.],\n",
            "        [12., 12.],\n",
            "        [12., 12.]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6nrD44oJiEIY",
        "colab_type": "text"
      },
      "source": [
        "$t = \\sum_i z_i, z_i = y_i^2 + 1, y_i = x_i + 5$\n",
        "\n",
        "$\\frac{\\partial t}{\\partial x_i} = \\frac{\\partial z_i}{\\partial x_i} = \\frac{\\partial z_i}{\\partial y_i} \\frac{\\partial y_i}{\\partial x_i} = 2y_i \\times 1$\n",
        "\n",
        "\n",
        "At x = 1, y = 6, $\\frac{\\partial t}{\\partial x_i} = 12$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZFCWPPAP6ipv",
        "colab_type": "code",
        "outputId": "d1d2243a-af30-4317-e303-3eee8058400e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 132
        }
      },
      "source": [
        "x = torch.ones([3, 2], requires_grad=True)\n",
        "y = x + 5\n",
        "r = 1/(1 + torch.exp(-y))\n",
        "print(r)\n",
        "s = torch.sum(r)\n",
        "s.backward()\n",
        "print(x.grad)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.9975, 0.9975],\n",
            "        [0.9975, 0.9975],\n",
            "        [0.9975, 0.9975]], grad_fn=<MulBackward0>)\n",
            "tensor([[0.0025, 0.0025],\n",
            "        [0.0025, 0.0025],\n",
            "        [0.0025, 0.0025]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ts1wsONqlE5h",
        "colab_type": "code",
        "outputId": "f42bd6a2-0c97-4b44-96d0-3e9eae2b90a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "x = torch.ones([3, 2], requires_grad=True)\n",
        "y = x + 5\n",
        "r = 1/(1 + torch.exp(-y))\n",
        "a = torch.ones([3, 2])\n",
        "r.backward(a)\n",
        "print(x.grad)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.0025, 0.0025],\n",
            "        [0.0025, 0.0025],\n",
            "        [0.0025, 0.0025]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "56AqY5hY77dx",
        "colab_type": "text"
      },
      "source": [
        "$\\frac{\\partial{s}}{\\partial{x}} = \\frac{\\partial{s}}{\\partial{r}} \\cdot \\frac{\\partial{r}}{\\partial{x}}$\n",
        "\n",
        "For the above code $a$ represents $\\frac{\\partial{s}}{\\partial{r}}$ and then $x.grad$ gives directly $\\frac{\\partial{s}}{\\partial{x}}$\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AKhxwdYUpUfj",
        "colab_type": "text"
      },
      "source": [
        "## Autodiff example that looks like what we have been doing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "THNkQLR6mmpO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.randn([20, 1], requires_grad=True)\n",
        "y = 3*x - 2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-t4_8qgdnjDk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "w = torch.tensor([1.], requires_grad=True)\n",
        "b = torch.tensor([1.], requires_grad=True)\n",
        "\n",
        "y_hat = w*x + b\n",
        "\n",
        "loss = torch.sum((y_hat - y)**2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gvpc37u-o6ob",
        "colab_type": "code",
        "outputId": "6846154f-587d-4aa3-cad5-4e3be6d08fc2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "print(loss)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(318.2823, grad_fn=<SumBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-tnKq6DXo-RB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss.backward()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I38qmZLhpM2F",
        "colab_type": "code",
        "outputId": "9ab231ff-c34e-450f-9d3a-8ef2b8068096",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "print(w.grad, b.grad)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([-106.4956]) tensor([141.1912])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WfDV6saTq8XA",
        "colab_type": "text"
      },
      "source": [
        "## Do it in a loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ivmJgJQTpN79",
        "colab_type": "code",
        "outputId": "831ec258-0f1e-4dbb-843f-d413a54c0784",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227
        }
      },
      "source": [
        "learning_rate = 0.01\n",
        "\n",
        "w = torch.tensor([1.], requires_grad=True)\n",
        "b = torch.tensor([1.], requires_grad=True)\n",
        "\n",
        "print(w.item(), b.item())\n",
        "\n",
        "for i in range(10):\n",
        "  \n",
        "  x = torch.randn([20, 1])\n",
        "  y = 3*x - 2\n",
        "  \n",
        "  y_hat = w*x + b\n",
        "  loss = torch.sum((y_hat - y)**2)\n",
        "  \n",
        "  loss.backward()\n",
        "  \n",
        "  with torch.no_grad():\n",
        "    w -= learning_rate * w.grad\n",
        "    b -= learning_rate * b.grad\n",
        "    \n",
        "    w.grad.zero_()\n",
        "    b.grad.zero_()\n",
        "\n",
        "  print(w.item(), b.item())\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.0 1.0\n",
            "1.694516658782959 -0.32816600799560547\n",
            "2.5244972705841064 -0.9011859893798828\n",
            "2.6990771293640137 -1.3381099700927734\n",
            "2.7810328006744385 -1.5905817747116089\n",
            "2.821857213973999 -1.7378290891647339\n",
            "2.943121910095215 -1.868725061416626\n",
            "2.9525837898254395 -1.9191371202468872\n",
            "2.9741718769073486 -1.9551563262939453\n",
            "2.9911296367645264 -1.972025752067566\n",
            "2.994936943054199 -1.9838125705718994\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vyOqrZZiuLkl",
        "colab_type": "text"
      },
      "source": [
        "## Do it for a large problem"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qq3Iykk1rMfh",
        "colab_type": "code",
        "outputId": "cd799469-ac56-4328-aa9e-33e0c17e57c6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "%%time\n",
        "learning_rate = 0.001\n",
        "N = 10000000\n",
        "epochs = 200\n",
        "\n",
        "w = torch.rand([N], requires_grad=True)\n",
        "b = torch.ones([1], requires_grad=True)\n",
        "\n",
        "# print(torch.mean(w).item(), b.item())\n",
        "\n",
        "for i in range(epochs):\n",
        "  \n",
        "  x = torch.randn([N])\n",
        "  y = torch.dot(3*torch.ones([N]), x) - 2\n",
        "  \n",
        "  y_hat = torch.dot(w, x) + b\n",
        "  loss = torch.sum((y_hat - y)**2)\n",
        "  \n",
        "  loss.backward()\n",
        "  \n",
        "  with torch.no_grad():\n",
        "    w -= learning_rate * w.grad\n",
        "    b -= learning_rate * b.grad\n",
        "    \n",
        "    w.grad.zero_()\n",
        "    b.grad.zero_()\n",
        "\n",
        "#   print(torch.mean(w).item(), b.item())\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 36.7 s, sys: 443 ms, total: 37.2 s\n",
            "Wall time: 37.2 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "owaeEn4A01zF",
        "colab_type": "code",
        "outputId": "1dba52f0-f0de-41b8-8325-ab1e1a6e21b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "%%time\n",
        "learning_rate = 0.001\n",
        "N = 10000000\n",
        "epochs = 200\n",
        "\n",
        "w = torch.rand([N], requires_grad=True, device=cuda0)\n",
        "b = torch.ones([1], requires_grad=True, device=cuda0)\n",
        "\n",
        "# print(torch.mean(w).item(), b.item())\n",
        "\n",
        "for i in range(epochs):\n",
        "  \n",
        "  x = torch.randn([N], device=cuda0)\n",
        "  y = torch.dot(3*torch.ones([N], device=cuda0), x) - 2\n",
        "  \n",
        "  y_hat = torch.dot(w, x) + b\n",
        "  loss = torch.sum((y_hat - y)**2)\n",
        "  \n",
        "  loss.backward()\n",
        "  \n",
        "  with torch.no_grad():\n",
        "    w -= learning_rate * w.grad\n",
        "    b -= learning_rate * b.grad\n",
        "    \n",
        "    w.grad.zero_()\n",
        "    b.grad.zero_()\n",
        "\n",
        "  #print(torch.mean(w).item(), b.item())\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 467 ms, sys: 305 ms, total: 772 ms\n",
            "Wall time: 784 ms\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}