Aug20/Discussion1/ResNet_Implementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reference - https://github.com/FrancescoSaverioZuppichini/ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "from functools import partial\n",
    "from dataclasses import dataclass\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Conv2dAuto(nn.Conv2d):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super().__init__(*args, **kwargs)\n",
    "        self.padding =  (self.kernel_size[0] // 2, self.kernel_size[1] // 2) # dynamic add padding based on the kernel_size\n",
    "        \n",
    "conv3x3 = partial(Conv2dAuto, kernel_size=3, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conv2dAuto(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n"
     ]
    }
   ],
   "source": [
    "conv = conv3x3(in_channels=32, out_channels=64)\n",
    "print(conv)\n",
    "del conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The residual block takes an input with in_channels, applies some blocks of convolutional layers to reduce it to out_channels and sum it up to the original input. If their sizes mismatch, then the input goes into an identity.\n",
    "\n",
    "class ResidualBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels):\n",
    "        super().__init__()\n",
    "        self.in_channels, self.out_channels =  in_channels, out_channels\n",
    "        self.blocks = nn.Identity()\n",
    "        self.shortcut = nn.Identity()   \n",
    "    \n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "        if self.should_apply_shortcut: residual = self.shortcut(x)\n",
    "        x = self.blocks(x)\n",
    "        x += residual\n",
    "        return x\n",
    "    \n",
    "    @property\n",
    "    def should_apply_shortcut(self):\n",
    "        return self.in_channels != self.out_channels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "\n",
    "class ResNetResidualBlock(ResidualBlock):\n",
    "    def __init__(self, in_channels, out_channels, expansion=1, downsampling=1, conv=conv3x3, *args, **kwargs):\n",
    "        super().__init__(in_channels, out_channels)\n",
    "        self.expansion, self.downsampling, self.conv = expansion, downsampling, conv\n",
    "        self.shortcut = nn.Sequential(OrderedDict(\n",
    "        {\n",
    "            'conv' : nn.Conv2d(self.in_channels, self.expanded_channels, kernel_size=1,\n",
    "                      stride=self.downsampling, bias=False),\n",
    "            'bn' : nn.BatchNorm2d(self.expanded_channels)\n",
    "            \n",
    "        })) if self.should_apply_shortcut else None\n",
    "        \n",
    "        \n",
    "    @property\n",
    "    def expanded_channels(self):\n",
    "        return self.out_channels * self.expansion\n",
    "    \n",
    "    @property\n",
    "    def should_apply_shortcut(self):\n",
    "        return self.in_channels != self.expanded_channels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#A basic ResNet block is composed by two layers of 3x3 convs/batchnorm/relu.\n",
    "\n",
    "from collections import OrderedDict\n",
    "def conv_bn(in_channels, out_channels, conv, *args, **kwargs):\n",
    "    return nn.Sequential(OrderedDict({'conv': conv(in_channels, out_channels, *args, **kwargs), \n",
    "                          'bn': nn.BatchNorm2d(out_channels) }))\n",
    "class ResNetBasicBlock(ResNetResidualBlock):\n",
    "    expansion = 1\n",
    "    def __init__(self, in_channels, out_channels, activation=nn.ReLU, *args, **kwargs):\n",
    "        super().__init__(in_channels, out_channels, *args, **kwargs)\n",
    "        self.blocks = nn.Sequential(\n",
    "            conv_bn(self.in_channels, self.out_channels, conv=self.conv, bias=False, stride=self.downsampling),\n",
    "            activation(),\n",
    "            conv_bn(self.out_channels, self.expanded_channels, conv=self.conv, bias=False),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNetBasicBlock(\n",
      "  (blocks): Sequential(\n",
      "    (0): Sequential(\n",
      "      (conv): Conv2dAuto(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (1): ReLU()\n",
      "    (2): Sequential(\n",
      "      (conv): Conv2dAuto(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (shortcut): Sequential(\n",
      "    (conv): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "dummy = torch.ones((1, 32, 224, 224))\n",
    "\n",
    "block = ResNetBasicBlock(32, 64)\n",
    "block(dummy).shape\n",
    "print(block)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#To increase the network deepths but to decrese the number of parameters, the Authors defined a BottleNeck block that \"The three layers are 1x1, 3x3, and 1x1 convolutions, where the 1×1 layers are responsible for reducing and then increasing (restoring) dimensions, leaving the 3×3 layer a bottleneck with smaller input/output dimensions.\" We can extend the ResNetResidualBlock and create these blocks.\n",
    "\n",
    "class ResNetBottleNeckBlock(ResNetResidualBlock):\n",
    "    expansion = 4\n",
    "    def __init__(self, in_channels, out_channels, activation=nn.ReLU, *args, **kwargs):\n",
    "        super().__init__(in_channels, out_channels, expansion=4, *args, **kwargs)\n",
    "        self.blocks = nn.Sequential(\n",
    "           conv_bn(self.in_channels, self.out_channels, self.conv, kernel_size=1),\n",
    "             activation(),\n",
    "             conv_bn(self.out_channels, self.out_channels, self.conv, kernel_size=3, stride=self.downsampling),\n",
    "             activation(),\n",
    "             conv_bn(self.out_channels, self.expanded_channels, self.conv, kernel_size=1),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNetBottleNeckBlock(\n",
      "  (blocks): Sequential(\n",
      "    (0): Sequential(\n",
      "      (conv): Conv2dAuto(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (1): ReLU()\n",
      "    (2): Sequential(\n",
      "      (conv): Conv2dAuto(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (3): ReLU()\n",
      "    (4): Sequential(\n",
      "      (conv): Conv2dAuto(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (shortcut): Sequential(\n",
      "    (conv): Conv2d(32, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "\n",
    "dummy = torch.ones((1, 32, 10, 10))\n",
    "\n",
    "block = ResNetBottleNeckBlock(32, 64)\n",
    "block(dummy).shape\n",
    "print(block)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResNetLayer(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, block=ResNetBasicBlock, n=1, *args, **kwargs):\n",
    "        super().__init__()\n",
    "        # 'We perform downsampling directly by convolutional layers that have a stride of 2.'\n",
    "        downsampling = 2 if in_channels != out_channels else 1\n",
    "        \n",
    "        self.blocks = nn.Sequential(\n",
    "            block(in_channels , out_channels, *args, **kwargs, downsampling=downsampling),\n",
    "            *[block(out_channels * block.expansion, \n",
    "                    out_channels, downsampling=1, *args, **kwargs) for _ in range(n - 1)]\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.blocks(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResNetLayer(\n",
       "  (blocks): Sequential(\n",
       "    (0): ResNetBasicBlock(\n",
       "      (blocks): Sequential(\n",
       "        (0): Sequential(\n",
       "          (conv): Conv2dAuto(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "          (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (1): ReLU()\n",
       "        (2): Sequential(\n",
       "          (conv): Conv2dAuto(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (shortcut): Sequential(\n",
       "        (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): ResNetBasicBlock(\n",
       "      (blocks): Sequential(\n",
       "        (0): Sequential(\n",
       "          (conv): Conv2dAuto(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (1): ReLU()\n",
       "        (2): Sequential(\n",
       "          (conv): Conv2dAuto(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (shortcut): None\n",
       "    )\n",
       "    (2): ResNetBasicBlock(\n",
       "      (blocks): Sequential(\n",
       "        (0): Sequential(\n",
       "          (conv): Conv2dAuto(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "        (1): ReLU()\n",
       "        (2): Sequential(\n",
       "          (conv): Conv2dAuto(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "          (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (shortcut): None\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy = torch.ones((1, 32, 48, 48))\n",
    "\n",
    "layer = ResNetLayer(64, 128, block=ResNetBasicBlock, n=3)\n",
    "layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The encoder is composed by multiple layer at increasing features size.\n",
    "\n",
    "class ResNetEncoder(nn.Module):\n",
    "    \"\"\"\n",
    "    ResNet encoder composed by increasing different layers with increasing features.\n",
    "    \"\"\"\n",
    "    def __init__(self, in_channels=3, blocks_sizes=[64, 128, 256, 512], deepths=[2,2,2,2], \n",
    "                 activation=nn.ReLU, block=ResNetBasicBlock, *args,**kwargs):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.blocks_sizes = blocks_sizes\n",
    "        \n",
    "        self.gate = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, self.blocks_sizes[0], kernel_size=7, stride=2, padding=3, bias=False),\n",
    "            nn.BatchNorm2d(self.blocks_sizes[0]),\n",
    "            activation(),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        )\n",
    "        \n",
    "        self.in_out_block_sizes = list(zip(blocks_sizes, blocks_sizes[1:]))\n",
    "        self.blocks = nn.ModuleList([ \n",
    "            ResNetLayer(blocks_sizes[0], blocks_sizes[0], n=deepths[0], activation=activation, \n",
    "                        block=block,  *args, **kwargs),\n",
    "            *[ResNetLayer(in_channels * block.expansion, \n",
    "                          out_channels, n=n, activation=activation, \n",
    "                          block=block, *args, **kwargs) \n",
    "              for (in_channels, out_channels), n in zip(self.in_out_block_sizes, deepths[1:])]       \n",
    "        ])\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.gate(x)\n",
    "        for block in self.blocks:\n",
    "            x = block(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The decoder is the last piece we need to create the full network. It is a fully connected layer that maps the features learned by the network to their respective classes.\n",
    "class ResnetDecoder(nn.Module):\n",
    "    \"\"\"\n",
    "    This class represents the tail of ResNet. It performs a global pooling and maps the output to the\n",
    "    correct class by using a fully connected layer.\n",
    "    \"\"\"\n",
    "    def __init__(self, in_features, n_classes):\n",
    "        super().__init__()\n",
    "        self.avg = nn.AdaptiveAvgPool2d((1, 1))\n",
    "        self.decoder = nn.Linear(in_features, n_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.avg(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.decoder(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resnet Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResNet(nn.Module):\n",
    "    \n",
    "    def __init__(self, in_channels, n_classes, *args, **kwargs):\n",
    "        super().__init__()\n",
    "        self.encoder = ResNetEncoder(in_channels, *args, **kwargs)\n",
    "        self.decoder = ResnetDecoder(self.encoder.blocks[-1].blocks[-1].expanded_channels, n_classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.encoder(x)\n",
    "        x = self.decoder(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet18(in_channels, n_classes):\n",
    "    return ResNet(in_channels, n_classes, block=ResNetBasicBlock, deepths=[2, 2, 2, 2])\n",
    "\n",
    "def resnet34(in_channels, n_classes):\n",
    "    return ResNet(in_channels, n_classes, block=ResNetBasicBlock, deepths=[3, 4, 6, 3])\n",
    "\n",
    "def resnet50(in_channels, n_classes):\n",
    "    return ResNet(in_channels, n_classes, block=ResNetBottleNeckBlock, deepths=[3, 4, 6, 3])\n",
    "\n",
    "def resnet101(in_channels, n_classes):\n",
    "    return ResNet(in_channels, n_classes, block=ResNetBottleNeckBlock, deepths=[3, 4, 23, 3])\n",
    "\n",
    "def resnet152(in_channels, n_classes):\n",
    "    return ResNet(in_channels, n_classes, block=ResNetBottleNeckBlock, deepths=[3, 8, 36, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1         [-1, 64, 112, 112]           9,408\n",
      "       BatchNorm2d-2         [-1, 64, 112, 112]             128\n",
      "              ReLU-3         [-1, 64, 112, 112]               0\n",
      "         MaxPool2d-4           [-1, 64, 56, 56]               0\n",
      "            Conv2d-5          [-1, 256, 56, 56]          16,384\n",
      "       BatchNorm2d-6          [-1, 256, 56, 56]             512\n",
      "        Conv2dAuto-7           [-1, 64, 56, 56]           4,096\n",
      "       BatchNorm2d-8           [-1, 64, 56, 56]             128\n",
      "              ReLU-9           [-1, 64, 56, 56]               0\n",
      "       Conv2dAuto-10           [-1, 64, 56, 56]          36,864\n",
      "      BatchNorm2d-11           [-1, 64, 56, 56]             128\n",
      "             ReLU-12           [-1, 64, 56, 56]               0\n",
      "       Conv2dAuto-13          [-1, 256, 56, 56]          16,384\n",
      "      BatchNorm2d-14          [-1, 256, 56, 56]             512\n",
      "ResNetBottleNeckBlock-15          [-1, 256, 56, 56]               0\n",
      "       Conv2dAuto-16           [-1, 64, 56, 56]          16,384\n",
      "      BatchNorm2d-17           [-1, 64, 56, 56]             128\n",
      "             ReLU-18           [-1, 64, 56, 56]               0\n",
      "       Conv2dAuto-19           [-1, 64, 56, 56]          36,864\n",
      "      BatchNorm2d-20           [-1, 64, 56, 56]             128\n",
      "             ReLU-21           [-1, 64, 56, 56]               0\n",
      "       Conv2dAuto-22          [-1, 256, 56, 56]          16,384\n",
      "      BatchNorm2d-23          [-1, 256, 56, 56]             512\n",
      "ResNetBottleNeckBlock-24          [-1, 256, 56, 56]               0\n",
      "       Conv2dAuto-25           [-1, 64, 56, 56]          16,384\n",
      "      BatchNorm2d-26           [-1, 64, 56, 56]             128\n",
      "             ReLU-27           [-1, 64, 56, 56]               0\n",
      "       Conv2dAuto-28           [-1, 64, 56, 56]          36,864\n",
      "      BatchNorm2d-29           [-1, 64, 56, 56]             128\n",
      "             ReLU-30           [-1, 64, 56, 56]               0\n",
      "       Conv2dAuto-31          [-1, 256, 56, 56]          16,384\n",
      "      BatchNorm2d-32          [-1, 256, 56, 56]             512\n",
      "ResNetBottleNeckBlock-33          [-1, 256, 56, 56]               0\n",
      "      ResNetLayer-34          [-1, 256, 56, 56]               0\n",
      "           Conv2d-35          [-1, 512, 28, 28]         131,072\n",
      "      BatchNorm2d-36          [-1, 512, 28, 28]           1,024\n",
      "       Conv2dAuto-37          [-1, 128, 56, 56]          32,768\n",
      "      BatchNorm2d-38          [-1, 128, 56, 56]             256\n",
      "             ReLU-39          [-1, 128, 56, 56]               0\n",
      "       Conv2dAuto-40          [-1, 128, 28, 28]         147,456\n",
      "      BatchNorm2d-41          [-1, 128, 28, 28]             256\n",
      "             ReLU-42          [-1, 128, 28, 28]               0\n",
      "       Conv2dAuto-43          [-1, 512, 28, 28]          65,536\n",
      "      BatchNorm2d-44          [-1, 512, 28, 28]           1,024\n",
      "ResNetBottleNeckBlock-45          [-1, 512, 28, 28]               0\n",
      "       Conv2dAuto-46          [-1, 128, 28, 28]          65,536\n",
      "      BatchNorm2d-47          [-1, 128, 28, 28]             256\n",
      "             ReLU-48          [-1, 128, 28, 28]               0\n",
      "       Conv2dAuto-49          [-1, 128, 28, 28]         147,456\n",
      "      BatchNorm2d-50          [-1, 128, 28, 28]             256\n",
      "             ReLU-51          [-1, 128, 28, 28]               0\n",
      "       Conv2dAuto-52          [-1, 512, 28, 28]          65,536\n",
      "      BatchNorm2d-53          [-1, 512, 28, 28]           1,024\n",
      "ResNetBottleNeckBlock-54          [-1, 512, 28, 28]               0\n",
      "       Conv2dAuto-55          [-1, 128, 28, 28]          65,536\n",
      "      BatchNorm2d-56          [-1, 128, 28, 28]             256\n",
      "             ReLU-57          [-1, 128, 28, 28]               0\n",
      "       Conv2dAuto-58          [-1, 128, 28, 28]         147,456\n",
      "      BatchNorm2d-59          [-1, 128, 28, 28]             256\n",
      "             ReLU-60          [-1, 128, 28, 28]               0\n",
      "       Conv2dAuto-61          [-1, 512, 28, 28]          65,536\n",
      "      BatchNorm2d-62          [-1, 512, 28, 28]           1,024\n",
      "ResNetBottleNeckBlock-63          [-1, 512, 28, 28]               0\n",
      "       Conv2dAuto-64          [-1, 128, 28, 28]          65,536\n",
      "      BatchNorm2d-65          [-1, 128, 28, 28]             256\n",
      "             ReLU-66          [-1, 128, 28, 28]               0\n",
      "       Conv2dAuto-67          [-1, 128, 28, 28]         147,456\n",
      "      BatchNorm2d-68          [-1, 128, 28, 28]             256\n",
      "             ReLU-69          [-1, 128, 28, 28]               0\n",
      "       Conv2dAuto-70          [-1, 512, 28, 28]          65,536\n",
      "      BatchNorm2d-71          [-1, 512, 28, 28]           1,024\n",
      "ResNetBottleNeckBlock-72          [-1, 512, 28, 28]               0\n",
      "      ResNetLayer-73          [-1, 512, 28, 28]               0\n",
      "           Conv2d-74         [-1, 1024, 14, 14]         524,288\n",
      "      BatchNorm2d-75         [-1, 1024, 14, 14]           2,048\n",
      "       Conv2dAuto-76          [-1, 256, 28, 28]         131,072\n",
      "      BatchNorm2d-77          [-1, 256, 28, 28]             512\n",
      "             ReLU-78          [-1, 256, 28, 28]               0\n",
      "       Conv2dAuto-79          [-1, 256, 14, 14]         589,824\n",
      "      BatchNorm2d-80          [-1, 256, 14, 14]             512\n",
      "             ReLU-81          [-1, 256, 14, 14]               0\n",
      "       Conv2dAuto-82         [-1, 1024, 14, 14]         262,144\n",
      "      BatchNorm2d-83         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-84         [-1, 1024, 14, 14]               0\n",
      "       Conv2dAuto-85          [-1, 256, 14, 14]         262,144\n",
      "      BatchNorm2d-86          [-1, 256, 14, 14]             512\n",
      "             ReLU-87          [-1, 256, 14, 14]               0\n",
      "       Conv2dAuto-88          [-1, 256, 14, 14]         589,824\n",
      "      BatchNorm2d-89          [-1, 256, 14, 14]             512\n",
      "             ReLU-90          [-1, 256, 14, 14]               0\n",
      "       Conv2dAuto-91         [-1, 1024, 14, 14]         262,144\n",
      "      BatchNorm2d-92         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-93         [-1, 1024, 14, 14]               0\n",
      "       Conv2dAuto-94          [-1, 256, 14, 14]         262,144\n",
      "      BatchNorm2d-95          [-1, 256, 14, 14]             512\n",
      "             ReLU-96          [-1, 256, 14, 14]               0\n",
      "       Conv2dAuto-97          [-1, 256, 14, 14]         589,824\n",
      "      BatchNorm2d-98          [-1, 256, 14, 14]             512\n",
      "             ReLU-99          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-100         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-101         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-102         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-103          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-104          [-1, 256, 14, 14]             512\n",
      "            ReLU-105          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-106          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-107          [-1, 256, 14, 14]             512\n",
      "            ReLU-108          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-109         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-110         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-111         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-112          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-113          [-1, 256, 14, 14]             512\n",
      "            ReLU-114          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-115          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-116          [-1, 256, 14, 14]             512\n",
      "            ReLU-117          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-118         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-119         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-120         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-121          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-122          [-1, 256, 14, 14]             512\n",
      "            ReLU-123          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-124          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-125          [-1, 256, 14, 14]             512\n",
      "            ReLU-126          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-127         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-128         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-129         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-130          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-131          [-1, 256, 14, 14]             512\n",
      "            ReLU-132          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-133          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-134          [-1, 256, 14, 14]             512\n",
      "            ReLU-135          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-136         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-137         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-138         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-139          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-140          [-1, 256, 14, 14]             512\n",
      "            ReLU-141          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-142          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-143          [-1, 256, 14, 14]             512\n",
      "            ReLU-144          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-145         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-146         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-147         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-148          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-149          [-1, 256, 14, 14]             512\n",
      "            ReLU-150          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-151          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-152          [-1, 256, 14, 14]             512\n",
      "            ReLU-153          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-154         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-155         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-156         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-157          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-158          [-1, 256, 14, 14]             512\n",
      "            ReLU-159          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-160          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-161          [-1, 256, 14, 14]             512\n",
      "            ReLU-162          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-163         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-164         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-165         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-166          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-167          [-1, 256, 14, 14]             512\n",
      "            ReLU-168          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-169          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-170          [-1, 256, 14, 14]             512\n",
      "            ReLU-171          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-172         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-173         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-174         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-175          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-176          [-1, 256, 14, 14]             512\n",
      "            ReLU-177          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-178          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-179          [-1, 256, 14, 14]             512\n",
      "            ReLU-180          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-181         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-182         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-183         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-184          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-185          [-1, 256, 14, 14]             512\n",
      "            ReLU-186          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-187          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-188          [-1, 256, 14, 14]             512\n",
      "            ReLU-189          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-190         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-191         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-192         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-193          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-194          [-1, 256, 14, 14]             512\n",
      "            ReLU-195          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-196          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-197          [-1, 256, 14, 14]             512\n",
      "            ReLU-198          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-199         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-200         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-201         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-202          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-203          [-1, 256, 14, 14]             512\n",
      "            ReLU-204          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-205          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-206          [-1, 256, 14, 14]             512\n",
      "            ReLU-207          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-208         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-209         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-210         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-211          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-212          [-1, 256, 14, 14]             512\n",
      "            ReLU-213          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-214          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-215          [-1, 256, 14, 14]             512\n",
      "            ReLU-216          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-217         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-218         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-219         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-220          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-221          [-1, 256, 14, 14]             512\n",
      "            ReLU-222          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-223          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-224          [-1, 256, 14, 14]             512\n",
      "            ReLU-225          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-226         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-227         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-228         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-229          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-230          [-1, 256, 14, 14]             512\n",
      "            ReLU-231          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-232          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-233          [-1, 256, 14, 14]             512\n",
      "            ReLU-234          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-235         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-236         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-237         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-238          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-239          [-1, 256, 14, 14]             512\n",
      "            ReLU-240          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-241          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-242          [-1, 256, 14, 14]             512\n",
      "            ReLU-243          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-244         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-245         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-246         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-247          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-248          [-1, 256, 14, 14]             512\n",
      "            ReLU-249          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-250          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-251          [-1, 256, 14, 14]             512\n",
      "            ReLU-252          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-253         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-254         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-255         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-256          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-257          [-1, 256, 14, 14]             512\n",
      "            ReLU-258          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-259          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-260          [-1, 256, 14, 14]             512\n",
      "            ReLU-261          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-262         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-263         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-264         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-265          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-266          [-1, 256, 14, 14]             512\n",
      "            ReLU-267          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-268          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-269          [-1, 256, 14, 14]             512\n",
      "            ReLU-270          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-271         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-272         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-273         [-1, 1024, 14, 14]               0\n",
      "      Conv2dAuto-274          [-1, 256, 14, 14]         262,144\n",
      "     BatchNorm2d-275          [-1, 256, 14, 14]             512\n",
      "            ReLU-276          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-277          [-1, 256, 14, 14]         589,824\n",
      "     BatchNorm2d-278          [-1, 256, 14, 14]             512\n",
      "            ReLU-279          [-1, 256, 14, 14]               0\n",
      "      Conv2dAuto-280         [-1, 1024, 14, 14]         262,144\n",
      "     BatchNorm2d-281         [-1, 1024, 14, 14]           2,048\n",
      "ResNetBottleNeckBlock-282         [-1, 1024, 14, 14]               0\n",
      "     ResNetLayer-283         [-1, 1024, 14, 14]               0\n",
      "          Conv2d-284           [-1, 2048, 7, 7]       2,097,152\n",
      "     BatchNorm2d-285           [-1, 2048, 7, 7]           4,096\n",
      "      Conv2dAuto-286          [-1, 512, 14, 14]         524,288\n",
      "     BatchNorm2d-287          [-1, 512, 14, 14]           1,024\n",
      "            ReLU-288          [-1, 512, 14, 14]               0\n",
      "      Conv2dAuto-289            [-1, 512, 7, 7]       2,359,296\n",
      "     BatchNorm2d-290            [-1, 512, 7, 7]           1,024\n",
      "            ReLU-291            [-1, 512, 7, 7]               0\n",
      "      Conv2dAuto-292           [-1, 2048, 7, 7]       1,048,576\n",
      "     BatchNorm2d-293           [-1, 2048, 7, 7]           4,096\n",
      "ResNetBottleNeckBlock-294           [-1, 2048, 7, 7]               0\n",
      "      Conv2dAuto-295            [-1, 512, 7, 7]       1,048,576\n",
      "     BatchNorm2d-296            [-1, 512, 7, 7]           1,024\n",
      "            ReLU-297            [-1, 512, 7, 7]               0\n",
      "      Conv2dAuto-298            [-1, 512, 7, 7]       2,359,296\n",
      "     BatchNorm2d-299            [-1, 512, 7, 7]           1,024\n",
      "            ReLU-300            [-1, 512, 7, 7]               0\n",
      "      Conv2dAuto-301           [-1, 2048, 7, 7]       1,048,576\n",
      "     BatchNorm2d-302           [-1, 2048, 7, 7]           4,096\n",
      "ResNetBottleNeckBlock-303           [-1, 2048, 7, 7]               0\n",
      "      Conv2dAuto-304            [-1, 512, 7, 7]       1,048,576\n",
      "     BatchNorm2d-305            [-1, 512, 7, 7]           1,024\n",
      "            ReLU-306            [-1, 512, 7, 7]               0\n",
      "      Conv2dAuto-307            [-1, 512, 7, 7]       2,359,296\n",
      "     BatchNorm2d-308            [-1, 512, 7, 7]           1,024\n",
      "            ReLU-309            [-1, 512, 7, 7]               0\n",
      "      Conv2dAuto-310           [-1, 2048, 7, 7]       1,048,576\n",
      "     BatchNorm2d-311           [-1, 2048, 7, 7]           4,096\n",
      "ResNetBottleNeckBlock-312           [-1, 2048, 7, 7]               0\n",
      "     ResNetLayer-313           [-1, 2048, 7, 7]               0\n",
      "   ResNetEncoder-314           [-1, 2048, 7, 7]               0\n",
      "AdaptiveAvgPool2d-315           [-1, 2048, 1, 1]               0\n",
      "          Linear-316                 [-1, 1000]       2,049,000\n",
      "   ResnetDecoder-317                 [-1, 1000]               0\n",
      "================================================================\n",
      "Total params: 44,549,160\n",
      "Trainable params: 44,549,160\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.57\n",
      "Forward/backward pass size (MB): 373.85\n",
      "Params size (MB): 169.94\n",
      "Estimated Total Size (MB): 544.36\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from torchsummary import summary\n",
    "\n",
    "model = resnet101(3, 1000)\n",
    "summary(model.cuda(), (3, 224, 224))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}