week13/4-3.nn.functional和nn.Module.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "Python 3.7.0 64-bit ('pytorch1.6': conda)",
   "display_name": "Python 3.7.0 64-bit ('pytorch1.6': conda)",
   "metadata": {
    "interpreter": {
     "hash": "32e20fa419d82ae44b3cd2548dcb635120fe40ded07472dbee097e90a87506aa"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# 4-3 nn.functional和nn.Module"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## 一、nn.functional和nn.Module\n",
    "\n",
    "前面介绍了Pytorch的低阶API, 可以利用这些API构建出神经网络相关的组件。\n",
    "\n",
    "而Pytorch和神经网络相关的功能组件大多都封装在torch.nn模块下。\n",
    "\n",
    "这些功能组件的绝大部分既有函数形式实现, 也有类形式实现。\n",
    "\n",
    "其中nn.functional有各种功能的函数实现。\n",
    "\n",
    "激活函数：\n",
    "* F.relu\n",
    "* F.sigmoid\n",
    "* F.tanh\n",
    "* F.softmax\n",
    "\n",
    "模型层:\n",
    "* F.linear\n",
    "* F.conv2d\n",
    "* F.max_pool2d\n",
    "* F.dropout2d\n",
    "* F.embedding\n",
    "\n",
    "损失函数:\n",
    "* F.binary_cross_entropy\n",
    "* F.mse_loss\n",
    "* F.cross_entropy\n",
    "\n",
    "为了方便管理, 一般通过继承nn.Module转换成类的实现形式, 并直接封装在nn模块之间。\n",
    "\n",
    "激活函数:\n",
    "* nn.ReLU\n",
    "* nn.Sigmoid\n",
    "* nn.Tanh\n",
    "* nn.Softmax\n",
    "\n",
    "模型层:\n",
    "* nn.Linear\n",
    "* nn.Conv2d\n",
    "* nn.MaxPool2d\n",
    "* nn.Dropout2d\n",
    "* nn.Embedding\n",
    "\n",
    "损失函数:\n",
    "* nn.BCELoss\n",
    "* nn.MSELoss\n",
    "* nn.CrossEntropyLoss"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## 二、使用nn.Module来管理参数\n",
    "在Pytorch中, 模型的参数是需要被优化器训练的, 因此, 通常需要设置参数为requires_grad=True的张量。\n",
    "\n",
    "同时, 在一个模型中, 往往有许多参数, 要手动管理这些参数并不是一个统一的事情。\n",
    "\n",
    "Pytorch一般讲参数用nn.Parameter来表示"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Parameter containing:\ntensor([[ 0.3199,  0.3305],\n        [-0.5942,  0.0669]], requires_grad=True)\nTrue\n"
    }
   ],
   "source": [
    "w = nn.Parameter(torch.randn(2, 2))\n",
    "print(w)\n",
    "print(w.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "ParameterList(\n    (0): Parameter containing: [torch.FloatTensor of size 8x1]\n    (1): Parameter containing: [torch.FloatTensor of size 8x2]\n)\nTrue\n"
    }
   ],
   "source": [
    "# nn.ParameterList可以将多个nn.Parameter组成一个列表\n",
    "params_list = nn.ParameterList([nn.Parameter(torch.rand(8, i)) for i in range(1, 3)])\n",
    "print(params_list)\n",
    "print(params_list[0].requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "ParameterDict(\n    (a): Parameter containing: [torch.FloatTensor of size 2x2]\n    (b): Parameter containing: [torch.FloatTensor of size 2]\n)\nTrue\n"
    }
   ],
   "source": [
    "# nn.ParameterDict可以将多个nn.Parameter组成一个字典\n",
    "params_dict = nn.ParameterDict({\"a\": nn.Parameter(torch.rand(2, 2)), \"b\": nn.Parameter(torch.zeros(2))})\n",
    "print(params_dict)\n",
    "print(params_dict[\"a\"].requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Parameter containing:\ntensor([[ 0.3199,  0.3305],\n        [-0.5942,  0.0669]], requires_grad=True) \n\nParameter containing:\ntensor([[0.8650],\n        [0.6233],\n        [0.4305],\n        [0.7132],\n        [0.7377],\n        [0.5824],\n        [0.9581],\n        [0.9803]], requires_grad=True) \n\nParameter containing:\ntensor([[0.9732, 0.6801],\n        [0.7689, 0.0200],\n        [0.5271, 0.5367],\n        [0.0412, 0.6533],\n        [0.6310, 0.8141],\n        [0.1876, 0.5978],\n        [0.3673, 0.5978],\n        [0.7242, 0.0620]], requires_grad=True) \n\nParameter containing:\ntensor([[0.2918, 0.9623],\n        [0.5477, 0.3819]], requires_grad=True) \n\nParameter containing:\ntensor([0., 0.], requires_grad=True) \n\nnumber of Parameters =  5\n"
    }
   ],
   "source": [
    "# 可以用Module把它们管理起来\n",
    "module = nn.Module()\n",
    "module.w = w\n",
    "module.params_list = params_list\n",
    "module.params_dict = params_dict\n",
    "\n",
    "num_param = 0\n",
    "for param in module.parameters():\n",
    "    print(param, \"\\n\")\n",
    "    num_param += 1\n",
    "print(\"number of Parameters = \", num_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 实践当中, 一般通过继承nn.Module来构建模块类, 并将所有需要学习的参数的部分放在构造函数中。\n",
    "\n",
    "# 以下为Pytorch中nn.Linear的源码简化版本\n",
    "\n",
    "class Linear(nn.Module):\n",
    "    __constants__ = [\"in_features\", \"out_features\"]\n",
    "\n",
    "    def __init__(self, in_features, out_features, bias=True):\n",
    "        super().__init__()\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.weight = nn.Parameter(torch.Tensor(out_features, in_features))\n",
    "        if bias:\n",
    "            self.bias = nn.Parameter(torch.Tensor(out_features))\n",
    "        else:\n",
    "            self.register_parameter(\"bias\", None)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return F.linear(x, self.weight, self.bias)"
   ]
  },
  {
   "source": [
    "## 三、使用nn.Module来管理子模块\n",
    "一般情况下, 我们很少直接使用nn.Parameter来定义参数构建模型, 而是通过一些拼装一些常用模型来构造模型。\n",
    "\n",
    "这些模型也是继承自nn.Module的对象, 本身也包括参数, 属于我们定义的模块的子模块。\n",
    "\n",
    "nn.Module提供了一下方法可以来管理这些模块：\n",
    "* children()方法, 返回生成器, 包括模块下所有的子模块\n",
    "* named_children(), 包括模块下所有的子模块, 以及他们的名字\n",
    "* modules(), 包括模块下的所有各个层级的模块, 包括模块本身\n",
    "* named_modules(), 包括各个层级的模块以及它们的名字。"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(num_embeddings = 10000,embedding_dim = 3,padding_idx = 1)\n",
    "        self.conv = nn.Sequential()\n",
    "        self.conv.add_module(\"conv_1\",nn.Conv1d(in_channels = 3,out_channels = 16,kernel_size = 5))\n",
    "        self.conv.add_module(\"pool_1\",nn.MaxPool1d(kernel_size = 2))\n",
    "        self.conv.add_module(\"relu_1\",nn.ReLU())\n",
    "        self.conv.add_module(\"conv_2\",nn.Conv1d(in_channels = 16,out_channels = 128,kernel_size = 2))\n",
    "        self.conv.add_module(\"pool_2\",nn.MaxPool1d(kernel_size = 2))\n",
    "        self.conv.add_module(\"relu_2\",nn.ReLU())\n",
    "        \n",
    "        self.dense = nn.Sequential()\n",
    "        self.dense.add_module(\"flatten\",nn.Flatten())\n",
    "        self.dense.add_module(\"linear\",nn.Linear(6144,1))\n",
    "        self.dense.add_module(\"sigmoid\",nn.Sigmoid())\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x = self.embedding(x).transpose(1,2)\n",
    "        x = self.conv(x)\n",
    "        y = self.dense(x)\n",
    "        return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Embedding(10000, 3, padding_idx=1) \n\nSequential(\n  (conv_1): Conv1d(3, 16, kernel_size=(5,), stride=(1,))\n  (pool_1): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  (relu_1): ReLU()\n  (conv_2): Conv1d(16, 128, kernel_size=(2,), stride=(1,))\n  (pool_2): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  (relu_2): ReLU()\n) \n\nSequential(\n  (flatten): Flatten()\n  (linear): Linear(in_features=6144, out_features=1, bias=True)\n  (sigmoid): Sigmoid()\n) \n\n"
    }
   ],
   "source": [
    "net = Net()\n",
    "for child in net.children():\n",
    "    print(child, \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "embedding : Embedding(10000, 3, padding_idx=1) \n\nconv : Sequential(\n  (conv_1): Conv1d(3, 16, kernel_size=(5,), stride=(1,))\n  (pool_1): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  (relu_1): ReLU()\n  (conv_2): Conv1d(16, 128, kernel_size=(2,), stride=(1,))\n  (pool_2): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  (relu_2): ReLU()\n) \n\ndense : Sequential(\n  (flatten): Flatten()\n  (linear): Linear(in_features=6144, out_features=1, bias=True)\n  (sigmoid): Sigmoid()\n) \n\n"
    }
   ],
   "source": [
    "for name, child in net.named_children():\n",
    "    print(name, \":\", child, \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Net(\n  (embedding): Embedding(10000, 3, padding_idx=1)\n  (conv): Sequential(\n    (conv_1): Conv1d(3, 16, kernel_size=(5,), stride=(1,))\n    (pool_1): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n    (relu_1): ReLU()\n    (conv_2): Conv1d(16, 128, kernel_size=(2,), stride=(1,))\n    (pool_2): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n    (relu_2): ReLU()\n  )\n  (dense): Sequential(\n    (flatten): Flatten()\n    (linear): Linear(in_features=6144, out_features=1, bias=True)\n    (sigmoid): Sigmoid()\n  )\n) \n\nEmbedding(10000, 3, padding_idx=1) \n\nSequential(\n  (conv_1): Conv1d(3, 16, kernel_size=(5,), stride=(1,))\n  (pool_1): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  (relu_1): ReLU()\n  (conv_2): Conv1d(16, 128, kernel_size=(2,), stride=(1,))\n  (pool_2): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n  (relu_2): ReLU()\n) \n\nConv1d(3, 16, kernel_size=(5,), stride=(1,)) \n\nMaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) \n\nReLU() \n\nConv1d(16, 128, kernel_size=(2,), stride=(1,)) \n\nMaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) \n\nReLU() \n\nSequential(\n  (flatten): Flatten()\n  (linear): Linear(in_features=6144, out_features=1, bias=True)\n  (sigmoid): Sigmoid()\n) \n\nFlatten() \n\nLinear(in_features=6144, out_features=1, bias=True) \n\nSigmoid() \n\n"
    }
   ],
   "source": [
    "for module in net.modules():\n",
    "    print(module, \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Embedding(10000, 3, padding_idx=1)"
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "children_dict = {name: module for name, module in net.named_children()}\n",
    "\n",
    "embedding = children_dict[\"embedding\"]\n",
    "embedding.requires_grad_(False)  # 冻结参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "False\n30000\n"
    }
   ],
   "source": [
    "#可以看到其第一层的参数已经不可以被训练了。\n",
    "for param in embedding.parameters():\n",
    "    print(param.requires_grad)\n",
    "    print(param.numel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "----------------------------------------------------------------\n        Layer (type)               Output Shape         Param #\n================================================================\n         Embedding-1               [-1, 200, 3]          30,000\n            Conv1d-2              [-1, 16, 196]             256\n         MaxPool1d-3               [-1, 16, 98]               0\n              ReLU-4               [-1, 16, 98]               0\n            Conv1d-5              [-1, 128, 97]           4,224\n         MaxPool1d-6              [-1, 128, 48]               0\n              ReLU-7              [-1, 128, 48]               0\n           Flatten-8                 [-1, 6144]               0\n            Linear-9                    [-1, 1]           6,145\n          Sigmoid-10                    [-1, 1]               0\n================================================================\nTotal params: 40,625\nTrainable params: 10,625\nNon-trainable params: 30,000\n----------------------------------------------------------------\nInput size (MB): 0.000763\nForward/backward pass size (MB): 0.287796\nParams size (MB): 0.154972\nEstimated Total Size (MB): 0.443531\n----------------------------------------------------------------\n"
    }
   ],
   "source": [
    "from torchkeras import summary\n",
    "summary(net, input_shape=(200,), input_dtype=torch.LongTensor)"
   ]
  }
 ]
}