02_Tensors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Y2fkP_np3Zpl"
   },
   "source": [
    "<!--BOOK_INFORMATION-->\n",
    "<img align=\"left\" style=\"width:80px;height:98px;padding-right:20px;\" src=\"https://raw.githubusercontent.com/joe-papa/pytorch-book/main/files/pytorch-book-cover.jpg\">\n",
    "\n",
    "This notebook contains an excerpt from the [PyTorch Pocket Reference](http://pytorchbook.com) book by [Joe Papa](http://joepapa.ai); content is available [on GitHub](https://github.com/joe-papa/pytorch-book)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SMa9uOzPFbl-"
   },
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/joe-papa/pytorch-book/blob/main/02_Tensors.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aXkszegKMMMn"
   },
   "source": [
    "# Chapter 2 - Tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QbzMwCbOLF4T"
   },
   "source": [
    "## Simple CPU Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 358,
     "status": "ok",
     "timestamp": 1612991134258,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "HWv6NA5YKpLU",
    "outputId": "6f60ffef-148b-4ee5-e499-4b54fb8ee652"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 8, 10, 12],\n",
      "        [14, 16, 18]])\n",
      "torch.Size([2, 3])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "x = torch.tensor([[1,2,3],[4,5,6]])\n",
    "y = torch.tensor([[7,8,9],[10,11,12]])\n",
    "z = x + y\n",
    "print(z) \n",
    "# out: \n",
    "#  tensor([[ 8, 10, 12], [14, 16, 18]])\n",
    "\n",
    "print(z.size()) \n",
    "# out: torch.Size([2, 3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7cvJwOfpRjyp"
   },
   "source": [
    "## Simple GPU Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 248,
     "status": "ok",
     "timestamp": 1612991309666,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "lB62TcNkQpjn",
    "outputId": "d913f0b3-7f2d-428a-8bee-f88582e5dc7d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 8, 10, 12],\n",
      "        [14, 16, 18]], device='cuda:0')\n",
      "torch.Size([2, 3])\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() \\\n",
    "  else \"cpu\"\n",
    "x = torch.tensor([[1,2,3],[4,5,6]], device=device)\n",
    "y = torch.tensor([[7,8,9],[10,11,12]],\n",
    "device=device)\n",
    "z = x + y\n",
    "print(z)\n",
    "# out: \n",
    "#   tensor([[ 8, 10, 12],\n",
    "#          [14, 16, 18]], device='cuda:0')\n",
    "\n",
    "print(z.size())\n",
    "# out: torch.Size([2, 3])\n",
    "\n",
    "print(z.device)\n",
    "# out: cuda:0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "siupc0FJR0ma"
   },
   "source": [
    "## Moving Tensors between CPU & GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 248,
     "status": "ok",
     "timestamp": 1612991465747,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "U5yCYvynR1md",
    "outputId": "b10eefc8-cc7e-4297-aabf-7902b3bef246"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 8, 10, 12],\n",
       "        [14, 16, 18]])"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() \\\n",
    "  else \"cpu\"\n",
    "x.to(device)\n",
    "y.to(device)\n",
    "z = x + y\n",
    "z.to(\"cpu\")\n",
    "# out:\n",
    "# tensor([[ 8, 10, 12],\n",
    "#         [14, 16, 18]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XIAXms9fS1LA"
   },
   "source": [
    "# Creating Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vHDeTeERR525"
   },
   "outputs": [],
   "source": [
    "import numpy \n",
    "\n",
    "# Created from pre-existing arrays\n",
    "w = torch.tensor([1,2,3]) # <1>\n",
    "w = torch.tensor((1,2,3)) # <2>\n",
    "w = torch.tensor(numpy.array([1,2,3])) # <3>\n",
    "\n",
    "# Initialized by size\n",
    "w = torch.empty(100,200) # <4>\n",
    "w = torch.zeros(100,200) # <5>\n",
    "w = torch.ones(100,200)  # <6>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZUDxzPJk030g"
   },
   "source": [
    "1. from a list\n",
    "2. from a tuple\n",
    "3. from a numpy array\n",
    "4. uninitialized, elements values are not predictable\n",
    "5. all elements initialized with 0.0\n",
    "6. all elements initialized with 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "L4VLExcC0EFk"
   },
   "outputs": [],
   "source": [
    "# Initialized by size with random values\n",
    "w = torch.rand(100,200)     # <1>\n",
    "w = torch.randn(100,200)    # <2>\n",
    "w = torch.randint(5,10,(100,200))  # <3> \n",
    "\n",
    "# Initialized with specified data type or device\n",
    "w = torch.empty((100,200), dtype=torch.float64, \n",
    "                device=\"cuda\")\n",
    "\n",
    "# Initialized to have same size, data type, \n",
    "#   and device as another tensor\n",
    "x = torch.empty_like(w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aC3pI1jr1XK4"
   },
   "source": [
    "1. Creates a 100 x 200 tensor with elements from a uniform distribution on the interval [0, 1)\n",
    "2. elements are random numbers from a normal distribution with mean 0 and variance 1\n",
    "3. elements are random integers between 5 and 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uvwi0dyuTZFA"
   },
   "source": [
    "## Data Types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 350,
     "status": "ok",
     "timestamp": 1612988520904,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "Zy_IzoICSt_E",
    "outputId": "b0019ac2-4072-4f42-a92c-965c40d6e838"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.float32\n"
     ]
    }
   ],
   "source": [
    "# Specify data type at creation using dtype\n",
    "w = torch.tensor([1,2,3], dtype=torch.float32)\n",
    "\n",
    "# Use casting method to cast to a new data type\n",
    "w.int()       # w remains a float32 after cast\n",
    "w = w.int()   # w changes to int32 after cast\n",
    "\n",
    "# Use to() method to cast to a new type\n",
    "w = w.to(torch.float64) # <1>\n",
    "w = w.to(dtype=torch.float64) # <2>\n",
    "\n",
    "# Python automatically converts data types during operations\n",
    "x = torch.tensor([1,2,3], dtype=torch.int32)\n",
    "y = torch.tensor([1,2,3], dtype=torch.float32)\n",
    "z = x + y # <3>\n",
    "print(z.dtype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BLvRdCYET-cG"
   },
   "source": [
    "## Tensor Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 281,
     "status": "ok",
     "timestamp": 1612993669230,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "YnLlmRdwTesO",
    "outputId": "adba42a9-f5da-454b-dc39-493d1518008b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2],\n",
      "        [3, 4],\n",
      "        [5, 6],\n",
      "        [7, 8]])\n",
      "tensor(4)\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1,2],[3,4],[5,6],[7,8]])\n",
    "print(x)\n",
    "# out:  \n",
    "# tensor([[1, 2],\n",
    "#         [3, 4],\n",
    "#         [5, 6],\n",
    "#         [7, 8]])\n",
    "\n",
    "# Indexing, returns a tensor\n",
    "print(x[1,1])\n",
    "# out: tensor(4)\n",
    "\n",
    "# Indexing, returns value as Python number\n",
    "print(x[1,1].item())\n",
    "# out: 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 237,
     "status": "ok",
     "timestamp": 1612993669909,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "BouHJUe84a-g",
    "outputId": "085155a5-a11d-4478-fbdb-474fabf0243f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([2, 4])\n",
      "tensor([1, 2, 3, 4])\n"
     ]
    }
   ],
   "source": [
    "# Slicing\n",
    "print(x[:2,1])\n",
    "# out: tensor([2, 4])\n",
    "\n",
    "# Boolean indexing\n",
    "# Only keep elements less than 5\n",
    "print(x[x<5])\n",
    "# out: tensor([1, 2, 3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 245,
     "status": "ok",
     "timestamp": 1612993670711,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "4a2FDWWD4myP",
    "outputId": "898253dc-f7a0-4ed0-9278-c5c5dcf08bed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 3, 5, 7],\n",
      "        [2, 4, 6, 8]])\n",
      "tensor([[1, 2, 3, 4],\n",
      "        [5, 6, 7, 8]])\n"
     ]
    }
   ],
   "source": [
    "# Transpose array, x.t() or x.T can be used\n",
    "print(x.t())\n",
    "# tensor([[1, 3, 5, 7],\n",
    "#         [2, 4, 6, 8]])\n",
    "\n",
    "# Changing shape, Usually view() is preferred over reshape()\n",
    "print(x.view((2,4)))\n",
    "# tensor([[1, 3, 5, 7],\n",
    "#         [2, 4, 6, 8]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 243,
     "status": "ok",
     "timestamp": 1612993671431,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "jqXM3ECQ53_h",
    "outputId": "5c2af4fe-f1ca-4f29-e9e8-2d56d9eaf0c6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[1, 2],\n",
      "         [3, 4],\n",
      "         [5, 6],\n",
      "         [7, 8]],\n",
      "\n",
      "        [[1, 2],\n",
      "         [3, 4],\n",
      "         [5, 6],\n",
      "         [7, 8]]])\n",
      "tensor([1, 3, 5, 7]) tensor([2, 4, 6, 8])\n"
     ]
    }
   ],
   "source": [
    "# Combining tensors\n",
    "y = torch.stack((x, x))\n",
    "print(y)\n",
    "# out: \n",
    "# tensor([[[1, 2],\n",
    "#          [3, 4],\n",
    "#          [5, 6],\n",
    "#          [7, 8]],\n",
    "\n",
    "#         [[1, 2],\n",
    "#          [3, 4],\n",
    "#          [5, 6],\n",
    "#          [7, 8]]])\n",
    "\n",
    "# Splitting tensors\n",
    "a,b = x.unbind(dim=1)\n",
    "print(a,b)\n",
    "# out:\n",
    "#  tensor([1, 3, 5, 7]) tensor([2, 4, 6, 8])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E6vwyrpvUPLE"
   },
   "source": [
    "## Automatic Differentiation (Autograd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 292,
     "status": "ok",
     "timestamp": 1612995280088,
     "user": {
      "displayName": "Joe Papa",
      "photoUrl": "",
      "userId": "00487850786587503652"
     },
     "user_tz": 300
    },
    "id": "IBHw-IziUKAa",
    "outputId": "217508e9-3cdb-4ebd-feac-80089f4b760b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 2., 3.],\n",
      "        [4., 5., 6.]], requires_grad=True)\n",
      "tensor(91., grad_fn=<SumBackward0>)\n",
      "tensor([[ 2.,  4.,  6.],\n",
      "        [ 8., 10., 12.]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1,2,3],[4,5,6]], \n",
    "         dtype=torch.float, requires_grad=True)\n",
    "print(x)\n",
    "# out:\n",
    "# tensor([[1., 2., 3.],\n",
    "#         [4., 5., 6.]], requires_grad=True)\n",
    "\n",
    "f = x.pow(2).sum()\n",
    "print(f)\n",
    "# tensor(91., grad_fn=<SumBackward0>)\n",
    "\n",
    "f.backward()\n",
    "print(x.grad) # df/dx = 2x \n",
    "# tensor([[ 2.,  4.,  6.],\n",
    "#         [ 8., 10., 12.]])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyMmZrwpk+TR0fSUOT0KmY+T",
   "collapsed_sections": [],
   "name": "02_Tensors.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}