basis/参数初始化和使用预训练模型.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 参数初始化\n",
    "\n",
    "**本章内容作为补充内容，可以后期有需要使用别人已经预训练好了的模型再回来翻**\n",
    "\n",
    "\n",
    "对于PyTorch中的各种网络层，我们会需要对参数进行初始化的操作，这里我们以 nn.Linear() 为例子：\n",
    "\n",
    "```\n",
    "class Linear(Module):\n",
    "    def __init__(self, in_features, out_features, bias=True):\n",
    "    ...\n",
    "    ...\n",
    "    ...\n",
    "        self.reset_parameters()\n",
    "    ...\n",
    "    def reset_parameters(self):\n",
    "        stdv = 1. / math.sqrt(self.weight.size(1))\n",
    "        self.weight.data.uniform_(-stdv, stdv)\n",
    "        if self.bias is not None:\n",
    "            self.bias.data.uniform_(-stdv, stdv)\n",
    "```\n",
    "\n",
    "上述代码摘自 PyTorch [torch.nn.Linear()的源码](http://pytorch.org/docs/master/_modules/torch/nn/modules/linear.html#Linear)\n",
    "\n",
    "从源代码中可以看到，PyTorch在我们实例化某层的时候，就帮我们自动完成了初始化的操作。但是，有时候，我们可能需要一些不一样的初始化。\n",
    "\n",
    "比如使用 xavier 方法 来进行初始化，我们该怎么做呢？\n",
    "\n",
    "在PyTorch中有一个 torch.nn.init 这个包就是用来帮我们完成初始化的。\n",
    "\n",
    "这个包提供了很多初始化的方法：\n",
    "\n",
    "* torch.nn.init.constant(tensor, val)\n",
    "* torch.nn.init.normal(tensor, mean=0, std=1)\n",
    "* torch.nn.init.xavier_uniform(tensor, gain=1)\n",
    "\n",
    "详情 http://pytorch.org/docs/master/nn.html#torch-nn-init\n",
    "\n",
    "\n",
    "**注意： PyTorch 中各个层默认的初始化是 uniform —— 均匀分布初始化**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "class LeNet5(nn.Module):\n",
    "    def __init__(self, in_dim, n_class):\n",
    "        super(LeNet5, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_dim, 6, 5, padding=2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16*5*5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, n_class)\n",
    "        \n",
    "        # 参数初始化函数\n",
    "        for p in self.modules():\n",
    "            if isinstance(p, nn.Conv2d):\n",
    "                nn.init.xavier_normal(p.weight.data)\n",
    "            elif isinstance(p, nn.Linear):\n",
    "                nn.init.normal(p.weight.data)\n",
    "\n",
    "    # 向前传播\n",
    "    def forward(self, x):\n",
    "        x = F.max_pool2d(F.relu(self.conv1(x)), 2)\n",
    "        x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
    "        x = x.view(-1, self.num_flat_features(x))\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "    \n",
    "    def num_flat_features(self, x):\n",
    "        size = x.size()[1:]\n",
    "        num_features = 1\n",
    "        for s in size:\n",
    "            num_features *= s\n",
    "        return num_features\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 使用预训练的AlexNet模型来对CIFAR10进行训练\n",
    "\n",
    "现在，我们已经知道AlexNet是个好用的模型，我们想用它来对CIFAR10进行训练，但是我们不想自己训练，想用别人训练好的，我们该怎么做呢？\n",
    "\n",
    "分三步：\n",
    "\n",
    "1、加载预训练模型\n",
    "\n",
    "2、对模型结构进行更改\n",
    "\n",
    "3、重新训练，我们需要进行训练的那几层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import models, transforms, datasets\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -----------------get Alexnet model-------------------------\n",
    "def getAlexNet(DOWNLOAD=True):\n",
    "    alexnet = models.alexnet(pretrained=DOWNLOAD)\n",
    "    return alexnet\n",
    "\n",
    "\n",
    "# 把 tensor 变成 Variable\n",
    "def to_var(x):\n",
    "    x = Variable(x)\n",
    "    if torch.cuda.is_available():\n",
    "        x = x.cuda()\n",
    "    return x\n",
    "\n",
    "# 是否下载\n",
    "DOWNLOAD = True\n",
    "\n",
    "# 需要分类的类别\n",
    "CLASS_NUM = 10\n",
    "\n",
    "# 下载预训练好的AlexNet模型\n",
    "alexnet = getAlexNet(DOWNLOAD)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -----------------修改预训练好的 Alexnet 模型中的分类层-------------------------\n",
    "alexnet.classifier = nn.Sequential(\n",
    "    nn.Dropout(p=0.5),\n",
    "    nn.Linear(in_features=9216, out_features=4096, bias=True),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(p=0.5),\n",
    "    nn.Linear(in_features=4096, out_features=4096, bias=True),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=4096, out_features=CLASS_NUM, bias=True)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "epoch 1\n",
      "Acc: 0.740700\n",
      "epoch 2\n",
      "Acc: 0.821440\n",
      "epoch 3\n",
      "Acc: 0.842560\n",
      "epoch 4\n",
      "Acc: 0.863100\n",
      "epoch 5\n",
      "Acc: 0.877080\n",
      "Acc: 0.857800\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# 使用GPU\n",
    "if torch.cuda.is_available():\n",
    "    alexnet = alexnet.cuda()\n",
    "\n",
    "# 数据预处理\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                         std=[0.229, 0.224, 0.225])\n",
    "])\n",
    "\n",
    "# 加载数据\n",
    "train_data = datasets.CIFAR10('./data', train=True, transform=transform, download=True)\n",
    "test_data = datasets.CIFAR10('./data', train=False, transform=transform, download=False)\n",
    "\n",
    "train_data_loader = DataLoader(train_data, batch_size=256, shuffle=True)\n",
    "test_data_loader = DataLoader(test_data, batch_size=256, shuffle=True)\n",
    "\n",
    "# optim\n",
    "learning_rate = 0.0001\n",
    "num_epoches = 5\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# 训练的时候，我们只更新 classifier 层的参数\n",
    "optimizer = optim.Adam(alexnet.classifier.parameters(), lr=learning_rate)\n",
    "\n",
    "# training\n",
    "alexnet.train()\n",
    "for epoch in range(num_epoches):\n",
    "    print('epoch {}'.format(epoch + 1))\n",
    "    runnin_acc = 0.0\n",
    "    running_loss = 0.0\n",
    "    for data, label in train_data_loader:\n",
    "        img = to_var(data)\n",
    "        label = to_var(label)\n",
    "        out = alexnet(img)\n",
    "        loss = criterion(out, label)\n",
    "        running_loss += loss.data[0] * label.size(0)\n",
    "        _, pred = torch.max(out, 1)\n",
    "        num_correct = (pred == label).sum()\n",
    "        accuracy = (pred == label).float().mean()\n",
    "        runnin_acc += num_correct.data[0]\n",
    "        # 向后传播\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    print(\"Acc: {:.6f}\".format(runnin_acc / len(train_data)))\n",
    "\n",
    "# evaluation\n",
    "alexnet.eval()\n",
    "runnin_acc = 0.0\n",
    "for data, label in test_data_loader:\n",
    "    img = to_var(data)\n",
    "    label = to_var(label)\n",
    "    out = alexnet(img)\n",
    "    loss = criterion(out, label)\n",
    "    _, pred = torch.max(out, 1)\n",
    "    num_correct = (pred == label).sum()\n",
    "    accuracy = (pred == label).float().mean()\n",
    "    runnin_acc += num_correct.data[0]\n",
    "print(\"Acc: {:.6f}\".format(runnin_acc / len(test_data)))\n",
    "\n",
    "\n",
    "print(\"Done!\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 改变预训练模型中的某一层\n",
    "\n",
    "这里我们以Alexnet为例\n",
    "\n",
    "```\n",
    "AlexNet(\n",
    "  (features): Sequential(\n",
    "    (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))\n",
    "    (1): ReLU(inplace)\n",
    "    (2): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
    "    (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
    "    (4): ReLU(inplace)\n",
    "    (5): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
    "    (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
    "    (7): ReLU(inplace)\n",
    "    (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
    "    (9): ReLU(inplace)\n",
    "    (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
    "    (11): ReLU(inplace)\n",
    "    (12): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
    "  )\n",
    "  (classifier): Sequential(\n",
    "    (0): Dropout(p=0.5)\n",
    "    (1): Linear(in_features=9216, out_features=4096, bias=True)\n",
    "    (2): ReLU(inplace)\n",
    "    (3): Dropout(p=0.5)\n",
    "    (4): Linear(in_features=4096, out_features=4096, bias=True)\n",
    "    (5): ReLU(inplace)\n",
    "    (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
    "  )\n",
    ")\n",
    "```\n",
    "\n",
    "某一天，我们突然发现了 AvgPool 的效果比 Maxpool 要好，于是我们想要改变把 AlextNet中的所有 Maxpool 用 AvgPool来代替该怎么办呢？\n",
    "\n",
    "> 查看 Alexnet 模型中的 Maxpool 的位置，我们可以清晰的发现 Maxpool 处于 features 下面的 (2)、(5)、(12)的位置上\n",
    "\n",
    "\n",
    "于是，我们改变 features 层下面的 (2)、(5)、(12) 就行了，具体操作如下所示\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "alexnet = models.alexnet(pretrained=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlexNet(\n",
      "  (features): Sequential(\n",
      "    (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))\n",
      "    (1): ReLU(inplace)\n",
      "    (2): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
      "    (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "    (4): ReLU(inplace)\n",
      "    (5): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
      "    (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (7): ReLU(inplace)\n",
      "    (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (9): ReLU(inplace)\n",
      "    (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (11): ReLU(inplace)\n",
      "    (12): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.5)\n",
      "    (1): Linear(in_features=9216, out_features=4096, bias=True)\n",
      "    (2): ReLU(inplace)\n",
      "    (3): Dropout(p=0.5)\n",
      "    (4): Linear(in_features=4096, out_features=4096, bias=True)\n",
      "    (5): ReLU(inplace)\n",
      "    (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# 改变前的 alexnet\n",
    "print(alexnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 开始改变\n",
    "\n",
    "alexnet.features._modules['2'] = nn.AvgPool2d(kernel_size=(3, 3), stride=(2, 2), ceil_mode=False)\n",
    "alexnet.features._modules['5'] = nn.AvgPool2d(kernel_size=(3, 3), stride=(2, 2), ceil_mode=False)\n",
    "alexnet.features._modules['12'] = nn.AvgPool2d(kernel_size=(3, 3), stride=(2, 2), ceil_mode=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlexNet(\n",
      "  (features): Sequential(\n",
      "    (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))\n",
      "    (1): ReLU(inplace)\n",
      "    (2): AvgPool2d(kernel_size=(3, 3), stride=(2, 2), padding=0, ceil_mode=False, count_include_pad=True)\n",
      "    (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "    (4): ReLU(inplace)\n",
      "    (5): AvgPool2d(kernel_size=(3, 3), stride=(2, 2), padding=0, ceil_mode=False, count_include_pad=True)\n",
      "    (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (7): ReLU(inplace)\n",
      "    (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (9): ReLU(inplace)\n",
      "    (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (11): ReLU(inplace)\n",
      "    (12): AvgPool2d(kernel_size=(3, 3), stride=(2, 2), padding=0, ceil_mode=False, count_include_pad=True)\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.5)\n",
      "    (1): Linear(in_features=9216, out_features=4096, bias=True)\n",
      "    (2): ReLU(inplace)\n",
      "    (3): Dropout(p=0.5)\n",
      "    (4): Linear(in_features=4096, out_features=4096, bias=True)\n",
      "    (5): ReLU(inplace)\n",
      "    (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    " # 改变后的 alexnet\n",
    "print(alexnet)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}