modify_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "665e1593-5572-466e-9ae4-bd31130f0486",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.models as models\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c92207ac",
   "metadata": {},
   "source": [
    "# pre-trained model weight 讀取並查看\n",
    "可使用 pytorch 的 torchvision.models 中所提供的模型權重，也可以使用自己訓練或下載的模型權重檔。\n",
    "\n",
    "## 使用 pytorch 提供的 pre-trained model weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6e034fee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_state type: <class 'collections.OrderedDict'>\n"
     ]
    }
   ],
   "source": [
    "model = models.resnet18(pretrained=True)\n",
    "model_state = model.state_dict()\n",
    "\n",
    "print(\"model_state type:\", type(model_state))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d1726f5a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: conv1.weight\n",
      "value: torch.Size([64, 3, 7, 7])\n",
      "name: bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: bn1.bias\n",
      "value: torch.Size([64])\n",
      "name: bn1.running_mean\n",
      "value: torch.Size([64])\n",
      "name: bn1.running_var\n",
      "value: torch.Size([64])\n",
      "name: bn1.num_batches_tracked\n",
      "value: torch.Size([])\n",
      "name: layer1.0.conv1.weight\n",
      "value: torch.Size([64, 64, 3, 3])\n",
      "name: layer1.0.bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.bn1.bias\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.bn1.running_mean\n",
      "value: torch.Size([64])\n"
     ]
    }
   ],
   "source": [
    "cnt = 0\n",
    "\n",
    "for param_tensor in model_state:\n",
    "    print(\"name:\", param_tensor)\n",
    "    print(\"value:\", model_state[param_tensor].shape)\n",
    "    \n",
    "    cnt += 1\n",
    "    if cnt == 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "375c0ce0",
   "metadata": {},
   "source": [
    "## 使用自己訓練或下載的權重檔"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cc94fa31",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading...\n",
      "From: https://download.pytorch.org/models/resnet18-5c106cde.pth\n",
      "To: C:\\Users\\joyle\\pythonwork\\torch_work\\resnet-5c106cde.pth\n",
      "100%|█████████████████████████████████████████████████████████████████████████████| 46.8M/46.8M [00:05<00:00, 8.63MB/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model has been downloaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import gdown\n",
    "\n",
    "resnet_model = 'https://download.pytorch.org/models/resnet18-5c106cde.pth'\n",
    "gdown.download(resnet_model, \"resnet-5c106cde.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d2178189",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint type: <class 'collections.OrderedDict'>\n"
     ]
    }
   ],
   "source": [
    "checkpoint = torch.load('resnet-5c106cde.pth')\n",
    "print(\"checkpoint type:\", type(checkpoint))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e09eb118",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: conv1.weight\n",
      "value: torch.Size([64, 3, 7, 7])\n",
      "name: bn1.running_mean\n",
      "value: torch.Size([64])\n",
      "name: bn1.running_var\n",
      "value: torch.Size([64])\n",
      "name: bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: bn1.bias\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.conv1.weight\n",
      "value: torch.Size([64, 64, 3, 3])\n",
      "name: layer1.0.bn1.running_mean\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.bn1.running_var\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.bn1.bias\n",
      "value: torch.Size([64])\n"
     ]
    }
   ],
   "source": [
    "cnt = 0\n",
    "for k, v in checkpoint.items():\n",
    "    print(\"name:\", k)\n",
    "    print(\"value:\", v.shape)\n",
    "    \n",
    "    cnt += 1\n",
    "    if cnt == 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d33a136-7fbc-40a8-bf8b-5a35581d1d27",
   "metadata": {},
   "source": [
    "# 修改layer\n",
    "這部分會分為修改 layer 的參數和名稱\n",
    "\n",
    "## 參數\n",
    "### 最後一層的輸出值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a9928ec2-2971-44cf-8881-2913066f899e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Linear(in_features=512, out_features=1000, bias=True)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = models.resnet18(pretrained=True)\n",
    "model.fc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1bdbd8e7-e969-4b54-b256-6a6ab68179e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear(in_features=512, out_features=10, bias=True)\n"
     ]
    }
   ],
   "source": [
    "in_features = model.fc.in_features\n",
    "num_class = 10\n",
    "\n",
    "model.fc = nn.Linear(in_features, num_class)\n",
    "print(model.fc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "30a5a987",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight:  torch.Size([10, 512])\n",
      "bias:  torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "model_state = model.state_dict()\n",
    "print(\"weight: \", model_state['fc.weight'].shape)\n",
    "print(\"bias: \", model_state['fc.bias'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9e126f23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight:  tensor([[-0.0178,  0.0195,  0.0206,  ..., -0.0246, -0.0031,  0.0218],\n",
      "        [ 0.0218, -0.0324, -0.0375,  ..., -0.0089,  0.0187, -0.0119],\n",
      "        [ 0.0048, -0.0325, -0.0199,  ..., -0.0054, -0.0382, -0.0221],\n",
      "        ...,\n",
      "        [-0.0324, -0.0263, -0.0365,  ...,  0.0139,  0.0190,  0.0053],\n",
      "        [ 0.0231, -0.0329,  0.0187,  ...,  0.0001, -0.0297,  0.0108],\n",
      "        [ 0.0147,  0.0201, -0.0160,  ...,  0.0025,  0.0128,  0.0210]])\n",
      "bias:  tensor([ 0.0158,  0.0223,  0.0025, -0.0293,  0.0103,  0.0068, -0.0419,  0.0389,\n",
      "         0.0371, -0.0370])\n"
     ]
    }
   ],
   "source": [
    "print(\"weight: \", model_state['fc.weight'])\n",
    "print(\"bias: \", model_state['fc.bias'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4f73cc7-25da-4799-8419-c5e9e16904a0",
   "metadata": {},
   "source": [
    "### 某層參數 \n",
    "* 使用 pytorch 提供的 pre-trained model 權重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5c18f96b-3352-4c31-9cb6-a89969d93ceb",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.resnet18(pretrained=True)\n",
    "model_state = model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3d1fcf24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: fc.weight\n",
      "name: fc.bias\n"
     ]
    }
   ],
   "source": [
    "for param_tensor in model_state:\n",
    "    if param_tensor.startswith('fc'):\n",
    "        print(\"name:\", param_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "aa61a9c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "org:  torch.Size([1000, 512])\n",
      "now:  torch.Size([10, 512])\n"
     ]
    }
   ],
   "source": [
    "print(\"org: \", model_state['fc.weight'].shape)\n",
    "model_state['fc.weight'] = torch.rand((10, 512))\n",
    "print(\"now: \", model_state['fc.weight'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e4145ccf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "org:  torch.Size([1000])\n",
      "now:  torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "print(\"org: \", model_state['fc.bias'].shape)\n",
    "model_state['fc.bias'] = torch.ones(10)\n",
    "print(\"now: \", model_state['fc.bias'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b5d9ca27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: fc.weight\n",
      "value: tensor([[0.4836, 0.1596, 0.4073,  ..., 0.1078, 0.6504, 0.2826],\n",
      "        [0.6172, 0.7873, 0.0915,  ..., 0.9723, 0.0903, 0.8155],\n",
      "        [0.1876, 0.0691, 0.0087,  ..., 0.6894, 0.4616, 0.0709],\n",
      "        ...,\n",
      "        [0.7497, 0.3456, 0.3441,  ..., 0.1699, 0.3600, 0.4801],\n",
      "        [0.1419, 0.5504, 0.0085,  ..., 0.6065, 0.9412, 0.6216],\n",
      "        [0.4666, 0.0663, 0.2911,  ..., 0.0243, 0.8549, 0.3349]])\n",
      "name: fc.bias\n",
      "value: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])\n"
     ]
    }
   ],
   "source": [
    "for param_tensor in model_state:\n",
    "    if param_tensor in ['fc.weight', 'fc.bias']:\n",
    "        print(\"name:\", param_tensor)\n",
    "        print(\"value:\", model_state[param_tensor])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd2221a",
   "metadata": {},
   "source": [
    "* 使用自己訓練或下載的權重檔"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7de23667-454c-4797-a09e-b3a7a1010c6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load('resnet-5c106cde.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1c3767c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: fc.weight\n",
      "value: torch.Size([1000, 512])\n",
      "name: fc.bias\n",
      "value: torch.Size([1000])\n"
     ]
    }
   ],
   "source": [
    "for k, v in checkpoint.items():\n",
    "    if k in ['fc.weight', 'fc.bias']:\n",
    "        print(\"name:\", k)\n",
    "        print(\"value:\", v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bd7b94ef-0e99-4aa7-813a-2d74b69fc4c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "org:  torch.Size([1000, 512])\n",
      "now:  torch.Size([10, 512])\n"
     ]
    }
   ],
   "source": [
    "print(\"org: \", checkpoint['fc.weight'].shape)\n",
    "checkpoint['fc.weight'] = torch.rand((10, 512))\n",
    "print(\"now: \", checkpoint['fc.weight'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ad656f33-8137-4c99-8de8-4de226503a5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "org:  torch.Size([1000])\n",
      "now:  torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "print(\"org: \", checkpoint['fc.bias'].shape)\n",
    "checkpoint['fc.bias'] = torch.ones(10)\n",
    "print(\"now: \", checkpoint['fc.bias'].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "86ff395d-f5ef-40d0-a792-064d07b0000f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: fc.weight\n",
      "value: tensor([[0.2027, 0.7287, 0.3048,  ..., 0.4458, 0.6887, 0.3502],\n",
      "        [0.4577, 0.7108, 0.3471,  ..., 0.6735, 0.6224, 0.3283],\n",
      "        [0.9822, 0.5041, 0.3341,  ..., 0.7837, 0.7394, 0.6175],\n",
      "        ...,\n",
      "        [0.3742, 0.1213, 0.6600,  ..., 0.7086, 0.8760, 0.8814],\n",
      "        [0.0372, 0.5985, 0.8533,  ..., 0.3187, 0.7410, 0.6001],\n",
      "        [0.6699, 0.5557, 0.1950,  ..., 0.1332, 0.1001, 0.1326]])\n",
      "name: fc.bias\n",
      "value: tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])\n"
     ]
    }
   ],
   "source": [
    "for k, v in checkpoint.items():\n",
    "    if k in ['fc.weight', 'fc.bias']:\n",
    "        print(\"name:\", k)\n",
    "        print(\"value:\", v)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdfd1d12-8bbc-4eb7-903c-a48dc6b29c40",
   "metadata": {},
   "source": [
    "## layer 名稱"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5179bf59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: conv1.weight\n",
      "value: torch.Size([64, 3, 7, 7])\n",
      "name: bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: bn1.bias\n",
      "value: torch.Size([64])\n",
      "name: bn1.running_mean\n",
      "value: torch.Size([64])\n",
      "name: bn1.running_var\n",
      "value: torch.Size([64])\n",
      "name: bn1.num_batches_tracked\n",
      "value: torch.Size([])\n",
      "name: layer1.0.conv1.weight\n",
      "value: torch.Size([64, 64, 3, 3])\n",
      "name: layer1.0.bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.bn1.bias\n",
      "value: torch.Size([64])\n",
      "name: layer1.0.bn1.running_mean\n",
      "value: torch.Size([64])\n"
     ]
    }
   ],
   "source": [
    "model = models.resnet18()\n",
    "\n",
    "cnt = 0\n",
    "for param_tensor in model.state_dict():\n",
    "    print(\"name:\", param_tensor)\n",
    "    print(\"value:\", model.state_dict()[param_tensor].size())\n",
    "    \n",
    "    cnt += 1\n",
    "    if cnt == 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8ffb0d3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: resnet.conv1.weight\n",
      "value: torch.Size([64, 3, 7, 7])\n",
      "name: resnet.bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: resnet.bn1.bias\n",
      "value: torch.Size([64])\n",
      "name: resnet.bn1.running_mean\n",
      "value: torch.Size([64])\n",
      "name: resnet.bn1.running_var\n",
      "value: torch.Size([64])\n",
      "name: resnet.bn1.num_batches_tracked\n",
      "value: torch.Size([])\n",
      "name: resnet.layer1.0.conv1.weight\n",
      "value: torch.Size([64, 64, 3, 3])\n",
      "name: resnet.layer1.0.bn1.weight\n",
      "value: torch.Size([64])\n",
      "name: resnet.layer1.0.bn1.bias\n",
      "value: torch.Size([64])\n",
      "name: resnet.layer1.0.bn1.running_mean\n",
      "value: torch.Size([64])\n"
     ]
    }
   ],
   "source": [
    "checkpoint = torch.load('resnet_weights.pth')\n",
    "\n",
    "cnt = 0\n",
    "for k, v in checkpoint.items():\n",
    "    print(\"name:\", k)\n",
    "    print(\"value:\", v.size())\n",
    "    \n",
    "    cnt += 1\n",
    "    if cnt == 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ccf0ece4",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict = OrderedDict()\n",
    "\n",
    "for k, v in checkpoint.items():\n",
    "    state_dict[k[len('resnet.'):]] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "435241c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['conv1.weight', 'bn1.weight', 'bn1.bias', 'bn1.running_mean', 'bn1.running_var', 'bn1.num_batches_tracked', 'layer1.0.conv1.weight', 'layer1.0.bn1.weight', 'layer1.0.bn1.bias', 'layer1.0.bn1.running_mean', 'layer1.0.bn1.running_var', 'layer1.0.bn1.num_batches_tracked', 'layer1.0.conv2.weight', 'layer1.0.bn2.weight', 'layer1.0.bn2.bias', 'layer1.0.bn2.running_mean', 'layer1.0.bn2.running_var', 'layer1.0.bn2.num_batches_tracked', 'layer1.1.conv1.weight', 'layer1.1.bn1.weight', 'layer1.1.bn1.bias', 'layer1.1.bn1.running_mean', 'layer1.1.bn1.running_var', 'layer1.1.bn1.num_batches_tracked', 'layer1.1.conv2.weight', 'layer1.1.bn2.weight', 'layer1.1.bn2.bias', 'layer1.1.bn2.running_mean', 'layer1.1.bn2.running_var', 'layer1.1.bn2.num_batches_tracked', 'layer2.0.conv1.weight', 'layer2.0.bn1.weight', 'layer2.0.bn1.bias', 'layer2.0.bn1.running_mean', 'layer2.0.bn1.running_var', 'layer2.0.bn1.num_batches_tracked', 'layer2.0.conv2.weight', 'layer2.0.bn2.weight', 'layer2.0.bn2.bias', 'layer2.0.bn2.running_mean', 'layer2.0.bn2.running_var', 'layer2.0.bn2.num_batches_tracked', 'layer2.0.downsample.0.weight', 'layer2.0.downsample.1.weight', 'layer2.0.downsample.1.bias', 'layer2.0.downsample.1.running_mean', 'layer2.0.downsample.1.running_var', 'layer2.0.downsample.1.num_batches_tracked', 'layer2.1.conv1.weight', 'layer2.1.bn1.weight', 'layer2.1.bn1.bias', 'layer2.1.bn1.running_mean', 'layer2.1.bn1.running_var', 'layer2.1.bn1.num_batches_tracked', 'layer2.1.conv2.weight', 'layer2.1.bn2.weight', 'layer2.1.bn2.bias', 'layer2.1.bn2.running_mean', 'layer2.1.bn2.running_var', 'layer2.1.bn2.num_batches_tracked', 'layer3.0.conv1.weight', 'layer3.0.bn1.weight', 'layer3.0.bn1.bias', 'layer3.0.bn1.running_mean', 'layer3.0.bn1.running_var', 'layer3.0.bn1.num_batches_tracked', 'layer3.0.conv2.weight', 'layer3.0.bn2.weight', 'layer3.0.bn2.bias', 'layer3.0.bn2.running_mean', 'layer3.0.bn2.running_var', 'layer3.0.bn2.num_batches_tracked', 'layer3.0.downsample.0.weight', 'layer3.0.downsample.1.weight', 'layer3.0.downsample.1.bias', 'layer3.0.downsample.1.running_mean', 'layer3.0.downsample.1.running_var', 'layer3.0.downsample.1.num_batches_tracked', 'layer3.1.conv1.weight', 'layer3.1.bn1.weight', 'layer3.1.bn1.bias', 'layer3.1.bn1.running_mean', 'layer3.1.bn1.running_var', 'layer3.1.bn1.num_batches_tracked', 'layer3.1.conv2.weight', 'layer3.1.bn2.weight', 'layer3.1.bn2.bias', 'layer3.1.bn2.running_mean', 'layer3.1.bn2.running_var', 'layer3.1.bn2.num_batches_tracked', 'layer4.0.conv1.weight', 'layer4.0.bn1.weight', 'layer4.0.bn1.bias', 'layer4.0.bn1.running_mean', 'layer4.0.bn1.running_var', 'layer4.0.bn1.num_batches_tracked', 'layer4.0.conv2.weight', 'layer4.0.bn2.weight', 'layer4.0.bn2.bias', 'layer4.0.bn2.running_mean', 'layer4.0.bn2.running_var', 'layer4.0.bn2.num_batches_tracked', 'layer4.0.downsample.0.weight', 'layer4.0.downsample.1.weight', 'layer4.0.downsample.1.bias', 'layer4.0.downsample.1.running_mean', 'layer4.0.downsample.1.running_var', 'layer4.0.downsample.1.num_batches_tracked', 'layer4.1.conv1.weight', 'layer4.1.bn1.weight', 'layer4.1.bn1.bias', 'layer4.1.bn1.running_mean', 'layer4.1.bn1.running_var', 'layer4.1.bn1.num_batches_tracked', 'layer4.1.conv2.weight', 'layer4.1.bn2.weight', 'layer4.1.bn2.bias', 'layer4.1.bn2.running_mean', 'layer4.1.bn2.running_var', 'layer4.1.bn2.num_batches_tracked', 'fc.weight', 'fc.bias'])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "81d16552",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[[-0.0322, -0.0509, -0.0117, -0.0062,  0.0003, -0.0347,  0.0073],\n",
       "          [-0.0072, -0.0488, -0.0295, -0.0035, -0.0362, -0.0497, -0.0226],\n",
       "          [ 0.0087,  0.0136,  0.0176,  0.0150, -0.0127,  0.0358,  0.0585],\n",
       "          [-0.0243,  0.0452,  0.0083,  0.0163, -0.0355,  0.0162, -0.0159],\n",
       "          [-0.0291,  0.0263,  0.0014,  0.0211, -0.0300,  0.0307,  0.0133],\n",
       "          [ 0.0156, -0.0002,  0.0679,  0.0492, -0.0200, -0.0276,  0.0333],\n",
       "          [-0.0059, -0.0139,  0.0266, -0.0367, -0.0117,  0.0113, -0.0111]],\n",
       "\n",
       "         [[-0.0222, -0.0303,  0.0024,  0.0153, -0.0004,  0.0151,  0.0063],\n",
       "          [ 0.0276,  0.0107,  0.0152,  0.0082, -0.0362,  0.0104,  0.0325],\n",
       "          [ 0.0083, -0.0053, -0.0374, -0.0166,  0.0003, -0.0019,  0.0062],\n",
       "          [ 0.0144,  0.0045,  0.0279, -0.0087,  0.0057, -0.0145,  0.0090],\n",
       "          [ 0.0215,  0.0047, -0.0268,  0.0387,  0.0050, -0.0034,  0.0031],\n",
       "          [-0.0269, -0.0255,  0.0234, -0.0368,  0.0168,  0.0030,  0.0048],\n",
       "          [-0.0396, -0.0137,  0.0259, -0.0556,  0.0247,  0.0295, -0.0559]],\n",
       "\n",
       "         [[ 0.0167,  0.0488,  0.0255,  0.0004,  0.0625, -0.0064,  0.0254],\n",
       "          [-0.0048,  0.0275, -0.0244, -0.0212,  0.0144, -0.0172, -0.0117],\n",
       "          [ 0.0123,  0.0283,  0.0245,  0.0116, -0.0126,  0.0103,  0.0128],\n",
       "          [ 0.0072, -0.0020,  0.0022,  0.0124, -0.0031, -0.0144,  0.0220],\n",
       "          [-0.0092,  0.0358,  0.0187,  0.0149,  0.0051, -0.0064, -0.0024],\n",
       "          [ 0.0042, -0.0257, -0.0124,  0.0215, -0.0191,  0.0076,  0.0157],\n",
       "          [ 0.0128,  0.0378,  0.0356, -0.0197,  0.0204, -0.0041,  0.0058]]]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_dict['conv1.weight'][:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f4a5ecc0-aff3-44d1-9e70-ef9a7996b232",
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(state_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60865313-f79d-410b-8b4c-9b0479ef3c46",
   "metadata": {},
   "source": [
    "# 新增 layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a0af96f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyResNet18(nn.Module):\n",
    "    def __init__(self, net_block, layers, num_classes=10):\n",
    "        super(MyResNet18, self).__init__()\n",
    "        self.in_channels = 64\n",
    "        self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(64)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.maxpooling = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "\n",
    "        self.layer1 = self.net_block_layer(net_block, 64, layers[0])\n",
    "        self.layer2 = self.net_block_layer(net_block, 128, layers[1], stride=2)\n",
    "        self.layer3 = self.net_block_layer(net_block, 256, layers[2], stride=2)\n",
    "        self.layer4 = self.net_block_layer(net_block, 512, layers[3], stride=2)\n",
    "\n",
    "        ## ============== 新增的網路層 ============ ##\n",
    "        self.layer5 = nn.Sequential(nn.Conv2d(layers[3], 128, kernel_size=3, stride=2, padding=1),\n",
    "                                    nn.BatchNorm2d(128),\n",
    "                                    nn.ReLU(inplace=True))\n",
    "                \n",
    "        ## ======================================= ##\n",
    "        \n",
    "        self.avgpooling = nn.AvgPool2d(7, stride=1)\n",
    "        self.fc = nn.Linear(512 * net_block.expansion, num_classes)\n",
    "\n",
    "        # 參數初始化\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n",
    "\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                nn.init.constant_(m.weight, 1)\n",
    "                nn.init.constant_(m.bias, 0)        \n",
    "\n",
    "    def net_block_layer(self, net_block, out_channels, num_blocks, stride=1):\n",
    "        downsample = None\n",
    "\n",
    "      # 在 shortcut 時，若維度不一樣，要更改維度\n",
    "        if stride != 1 or self.in_channels != out_channels * net_block.expansion:\n",
    "            downsample = nn.Sequential(nn.Conv2d(self.in_channels, out_channels * net_block.expansion, kernel_size=1, stride=stride, bias=False),\n",
    "                      nn.BatchNorm2d(out_channels * net_block.expansion))\n",
    "\n",
    "        layers = []\n",
    "        layers.append(net_block(self.in_channels, out_channels, stride, downsample))\n",
    "        if net_block.expansion != 1:\n",
    "            self.in_channels = out_channels * net_block.expansion\n",
    "\n",
    "        else:\n",
    "            self.in_channels = out_channels\n",
    "\n",
    "        for i in range(1, num_blocks):\n",
    "            layers.append(net_block(self.in_channels, out_channels, 1, None))\n",
    "\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.bn1(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.maxpooling(x)\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        x = self.layer4(x)\n",
    "        \n",
    "        x = self.layer5(x)\n",
    "        \n",
    "        x = self.avgpooling(x)\n",
    "        x = torch.flatten(x, start_dim=1)\n",
    "        x = self.fc(x)\n",
    "\n",
    "        return x\n",
    "    \n",
    "class basic_block(nn.Module):\n",
    "    # 輸出通道乘的倍數\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, stride, downsample):\n",
    "        super(basic_block, self).__init__()      \n",
    "        self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv2 = nn.Conv2d(in_channels=out_channels, out_channels=out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "\n",
    "        # 在 shortcut 時，若維度不一樣，要更改維度\n",
    "        self.downsample = downsample \n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "\n",
    "        if self.downsample is not None:\n",
    "            residual = self.downsample(x)\n",
    "\n",
    "        out += residual\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ed6ccc21",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 10\n",
    "\n",
    "model = MyResNet18(basic_block, [2, 2, 2, 2], num_classes)\n",
    "model_state = model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "65a43eb2-875c-4b98-ba8a-2891e49c1c29",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load('resnet-5c106cde.pth')\n",
    "\n",
    "checkpoint['fc.weight'] = torch.zeros((num_classes, 512))\n",
    "checkpoint['fc.bias'] = torch.zeros(num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "cf26d3ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained_dict = {k: v for k, v in checkpoint.items() if k in model_state}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "4ef0701a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_state.update(pretrained_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f94a437d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(model_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28cf03d4",
   "metadata": {},
   "source": [
    "# 刪除 layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d0aa4840",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load('resnet-5c106cde.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2c8110cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer4.1.conv1.weight\n",
      "layer4.1.bn1.running_mean\n",
      "layer4.1.bn1.running_var\n",
      "layer4.1.bn1.weight\n",
      "layer4.1.bn1.bias\n",
      "layer4.1.conv2.weight\n",
      "layer4.1.bn2.running_mean\n",
      "layer4.1.bn2.running_var\n",
      "layer4.1.bn2.weight\n",
      "layer4.1.bn2.bias\n"
     ]
    }
   ],
   "source": [
    "for k in list(checkpoint.keys()):\n",
    "    if k.startswith('layer4.1'):\n",
    "        print(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "43a5a6f0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['None']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "for k in list(checkpoint.keys()):    \n",
    "    if k.startswith('layer4.1'):\n",
    "        del checkpoint[k]\n",
    "\n",
    "# 驗證是否刪除成功\n",
    "for k in list(checkpoint.keys()):\n",
    "    a = [\"None\" if not k.startswith('layer4.1') else \"Exists\" for k in list(checkpoint.keys())]\n",
    "\n",
    "print(np.unique(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bab04fb8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}