Object_Detection/Faster_R_CNN(2015)_미완성.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Faster R-CNN(2015)_미완성.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMh1S/0MONF7wLWJGj5Cx5c",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e1b2bc3f80164e3f9f24ccc6bd88e20c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_528b6aa6473b4923a5c4cb5dcac4646c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a214b1af5e304f6da26d8d5b6d187342",
              "IPY_MODEL_1a21e0b812fe4a6e897f672c1f955843"
            ]
          }
        },
        "528b6aa6473b4923a5c4cb5dcac4646c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a214b1af5e304f6da26d8d5b6d187342": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7795a77310dd40169a518ec327598b41",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 553433881,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 553433881,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0c8a0ae8a664780b607014c72ebc559"
          }
        },
        "1a21e0b812fe4a6e897f672c1f955843": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe52f9921830436bbb126df3f1c86eef",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 528M/528M [20:21&lt;00:00, 453kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_638e4e6f9dfa4b3d8c44aa8a6b3f02f4"
          }
        },
        "7795a77310dd40169a518ec327598b41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0c8a0ae8a664780b607014c72ebc559": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe52f9921830436bbb126df3f1c86eef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "638e4e6f9dfa4b3d8c44aa8a6b3f02f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Seonghoon-Yu/Paper_Review_and_Implementation_in_PyTorch/blob/master/Object_Detection/Faster_R_CNN(2015)_%EB%AF%B8%EC%99%84%EC%84%B1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XdqE84cFdI5K"
      },
      "source": [
        "## 필요한 라이브러리 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wxJQ97CYdMAm",
        "outputId": "bad55f62-d209-4818-acf9-16a0d8292c32"
      },
      "source": [
        "!pip install -U albumentations"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting albumentations\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/03/58/63fb1d742dc42d9ba2800ea741de1f2bc6bb05548d8724aa84794042eaf2/albumentations-0.5.2-py3-none-any.whl (72kB)\n",
            "\r\u001b[K     |████▌                           | 10kB 14.7MB/s eta 0:00:01\r\u001b[K     |█████████                       | 20kB 20.1MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 30kB 17.5MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 40kB 14.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 51kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 61kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 71kB 7.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 81kB 4.8MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: PyYAML in /usr/local/lib/python3.7/dist-packages (from albumentations) (3.13)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.11.1 in /usr/local/lib/python3.7/dist-packages (from albumentations) (1.19.5)\n",
            "Collecting opencv-python-headless>=4.1.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c8/84/72ec52fbac4775c2a5bf0ee5573c922a0cac35eb841907edf56493a5e313/opencv_python_headless-4.5.2.52-cp37-cp37m-manylinux2014_x86_64.whl (38.2MB)\n",
            "\u001b[K     |████████████████████████████████| 38.2MB 1.4MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.7/dist-packages (from albumentations) (1.4.1)\n",
            "Collecting imgaug>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/66/b1/af3142c4a85cba6da9f4ebb5ff4e21e2616309552caca5e8acefe9840622/imgaug-0.4.0-py2.py3-none-any.whl (948kB)\n",
            "\u001b[K     |████████████████████████████████| 952kB 34.6MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: scikit-image>=0.16.1 in /usr/local/lib/python3.7/dist-packages (from albumentations) (0.16.2)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.7/dist-packages (from imgaug>=0.4.0->albumentations) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: Shapely in /usr/local/lib/python3.7/dist-packages (from imgaug>=0.4.0->albumentations) (1.7.1)\n",
            "Requirement already satisfied, skipping upgrade: Pillow in /usr/local/lib/python3.7/dist-packages (from imgaug>=0.4.0->albumentations) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: matplotlib in /usr/local/lib/python3.7/dist-packages (from imgaug>=0.4.0->albumentations) (3.2.2)\n",
            "Requirement already satisfied, skipping upgrade: opencv-python in /usr/local/lib/python3.7/dist-packages (from imgaug>=0.4.0->albumentations) (4.1.2.30)\n",
            "Requirement already satisfied, skipping upgrade: imageio in /usr/local/lib/python3.7/dist-packages (from imgaug>=0.4.0->albumentations) (2.4.1)\n",
            "Requirement already satisfied, skipping upgrade: PyWavelets>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image>=0.16.1->albumentations) (1.1.1)\n",
            "Requirement already satisfied, skipping upgrade: networkx>=2.0 in /usr/local/lib/python3.7/dist-packages (from scikit-image>=0.16.1->albumentations) (2.5.1)\n",
            "Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations) (1.3.1)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations) (2.8.1)\n",
            "Requirement already satisfied, skipping upgrade: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->imgaug>=0.4.0->albumentations) (0.10.0)\n",
            "Requirement already satisfied, skipping upgrade: decorator<5,>=4.3 in /usr/local/lib/python3.7/dist-packages (from networkx>=2.0->scikit-image>=0.16.1->albumentations) (4.4.2)\n",
            "Installing collected packages: opencv-python-headless, imgaug, albumentations\n",
            "  Found existing installation: imgaug 0.2.9\n",
            "    Uninstalling imgaug-0.2.9:\n",
            "      Successfully uninstalled imgaug-0.2.9\n",
            "  Found existing installation: albumentations 0.1.12\n",
            "    Uninstalling albumentations-0.1.12:\n",
            "      Successfully uninstalled albumentations-0.1.12\n",
            "Successfully installed albumentations-0.5.2 imgaug-0.4.0 opencv-python-headless-4.5.2.52\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LS5-Viw00Hpc"
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch\n",
        "from torchvision.models import vgg16\n",
        "import numpy as np\n",
        "from torchvision.datasets import VOCDetection\n",
        "import torchvision.datasets as dataset\n",
        "from torchvision.transforms.functional import to_tensor, to_pil_image\n",
        "import torch.nn.functional as F\n",
        "import torch.nn.init as init\n",
        "\n",
        "from PIL import Image, ImageDraw, ImageFont\n",
        "import os\n",
        "import xml.etree.ElementTree as ET\n",
        "from typing import Any, Callable, Dict, Optional, Tuple, List\n",
        "import warnings\n",
        "import tarfile\n",
        "import collections\n",
        "import numpy as np\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "import cv2\n",
        "from torch import optim\n",
        "import albumentations as A\n",
        "from albumentations.pytorch import ToTensor\n",
        "import os\n",
        "import time\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_SytCWNKdPcX"
      },
      "source": [
        "# VOC dataset 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mgtq-Rq_SO2T"
      },
      "source": [
        "# VOC 2007 dataset을 저장할 위치\n",
        "path2data = '/content/voc'\n",
        "if not os.path.exists(path2data):\n",
        "    os.mkdir(path2data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rqDY6ziCdSpo"
      },
      "source": [
        "# VOC class names\n",
        "classes = [\n",
        "    \"aeroplane\",\n",
        "    \"bicycle\",\n",
        "    \"bird\",\n",
        "    \"boat\",\n",
        "    \"bottle\",\n",
        "    \"bus\",\n",
        "    \"car\",\n",
        "    \"cat\",\n",
        "    \"chair\",\n",
        "    \"cow\",\n",
        "    \"diningtable\",\n",
        "    \"dog\",\n",
        "    \"horse\",\n",
        "    \"motorbike\",\n",
        "    \"person\",\n",
        "    \"pottedplant\",\n",
        "    \"sheep\",\n",
        "    \"sofa\",\n",
        "    \"train\",\n",
        "    \"tvmonitor\"\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YNHZwMJydUWy"
      },
      "source": [
        "# PyTorch에서 제공하는 VOC dataset을 상속받아, custom dataset을 생성합니다.\n",
        "class myVOCDetection(VOCDetection):\n",
        "    def __getitem__(self, index):\n",
        "        img = np.array(Image.open(self.images[index]).convert('RGB'))\n",
        "        target = self.parse_voc_xml(ET.parse(self.annotations[index]).getroot()) # xml파일 분석하여 dict으로 받아오기\n",
        "\n",
        "        targets = [] # 바운딩 박스 좌표\n",
        "        labels = [] # 바운딩 박스 클래스\n",
        "\n",
        "        # 바운딩 박스 정보 받아오기\n",
        "        for t in target['annotation']['object']:\n",
        "            label = np.zeros(5)\n",
        "            label[:] = t['bndbox']['xmin'], t['bndbox']['ymin'], t['bndbox']['xmax'], t['bndbox']['ymax'], classes.index(t['name'])\n",
        "\n",
        "            targets.append(list(label[:4])) # 바운딩 박스 좌표\n",
        "            labels.append(label[4])         # 바운딩 박스 클래스\n",
        "\n",
        "        if self.transforms:\n",
        "            augmentations = self.transforms(image=img, bboxes=targets)\n",
        "            img = augmentations['image']\n",
        "            targets = augmentations['bboxes']\n",
        "\n",
        "        return img, targets, labels\n",
        "\n",
        "    def parse_voc_xml(self, node: ET.Element) -> Dict[str, Any]: # xml 파일을 dictionary로 반환\n",
        "        voc_dict: Dict[str, Any] = {}\n",
        "        children = list(node)\n",
        "        if children:\n",
        "            def_dic: Dict[str, Any] = collections.defaultdict(list)\n",
        "            for dc in map(self.parse_voc_xml, children):\n",
        "                for ind, v in dc.items():\n",
        "                    def_dic[ind].append(v)\n",
        "            if node.tag == \"annotation\":\n",
        "                def_dic[\"object\"] = [def_dic[\"object\"]]\n",
        "            voc_dict = {node.tag: {ind: v[0] if len(v) == 1 else v for ind, v in def_dic.items()}}\n",
        "        if node.text:\n",
        "            text = node.text.strip()\n",
        "            if not children:\n",
        "                voc_dict[node.tag] = text\n",
        "        return voc_dict"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mx5ho76ldWYz"
      },
      "source": [
        "# train, validation dataset을 생성합니다.\n",
        "train_ds = myVOCDetection(path2data, year='2007', image_set='train', download=True)\n",
        "val_ds = myVOCDetection(path2data, year='2007', image_set='test', download=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rb7sncQnfVIe"
      },
      "source": [
        "# DataEncoder 정의하기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fSh9a6sogOHi"
      },
      "source": [
        "class DataEncoder:\n",
        "    def __init__(self):\n",
        "        self.anchor_ratio = [0.5, 1, 2] # w/h\n",
        "        self.anchor_sizes = [8, 16, 32] # scale\n",
        "        self.anchor_number = len(self.anchor_ratio) * len(self.anchor_sizes) # 9\n",
        "        self.stride = 16 # 입력 이미지 크기와 vgg16 마지막 피쳐맵 크기의 비율\n",
        "\n",
        "    def encode_rpn(self, bboxs,labels):\n",
        "        ctr_x = self.stride/2 # 8\n",
        "        ctr_y = self.stride/2 # 8\n",
        "\n",
        "        # 9개의 anchors\n",
        "        anchors_template = np.zeros((9,4))\n",
        "\n",
        "        for i, ratio in enumerate(self.anchor_ratio):\n",
        "            for j, size in enumerate(self.anchor_sizes):\n",
        "                # 앵커박스 너비와 높이\n",
        "                w = size * np.sqrt(ratio) * self.stride\n",
        "                h = size * np.sqrt(ratio) * self.stride\n",
        "\n",
        "                # 앵커박스 좌표\n",
        "                x1 = -w/2\n",
        "                y1 = -h/2\n",
        "                x2 = w/2\n",
        "                y2 = h/2\n",
        "\n",
        "                anchor = [x1,y1,x2,y2]\n",
        "                anchors_template[i*len(self.anchor_ratio)+j]=anchor\n",
        "        \n",
        "        # 50x50 feature map의 모든 위치에 anchor 할당\n",
        "        feature_map_size = 800 // 16\n",
        "\n",
        "        # 중심 좌표\n",
        "        ctr_x_all = np.arange(8, (feature_map_size + 1) * 16 - 8, 16)\n",
        "        ctr_y_all = np.arange(8, (feature_map_size + 1) * 16 - 8, 16)\n",
        "\n",
        "        ctr = np.zeros((feature_map_size, feature_map_size, 2), dtype=np.float32)\n",
        "\n",
        "        for x in range(feature_map_size):\n",
        "            for y in range(feature_map_size):\n",
        "                ctr[x, y] = np.array([ctr_x_all[x], ctr_y_all[y]])\n",
        "\n",
        "\n",
        "        # anchors -> (W//16, H//16, 9, 4)\n",
        "        anchors = np.zeros((feature_map_size, feature_map_size, 9, 4))\n",
        "\n",
        "        for x in range(feature_map_size):\n",
        "            for y in range(feature_map_size):\n",
        "                anchors[x, y] = (ctr[x, y] + anchors_template.reshape(-1, 2, 2)).reshape(-1, 4)\n",
        "\n",
        "        # 앵커 박스에 labels 할당\n",
        "        # 1) gt와 가장 높은 iou를 지닌 앵커박스에 positive 할당\n",
        "        # 2) gt와 0.7 iou 이상을 지닌 앵커박스에 negative 할당\n",
        "\n",
        "        # 입력 이미지를 벗어나는 앵커 제거\n",
        "        anchors = anchors.reshape(-1, 4)\n",
        "\n",
        "        index_inside = np.where((anchors[:, 0] >= 0) &\n",
        "                            (anchors[:, 1] >= 0) &\n",
        "                            (anchors[:, 2] <= 800) &\n",
        "                            (anchors[:, 3] <= 800))[0]\n",
        "\n",
        "        # 빈 레이블 생성\n",
        "        label = np.empty((len(index_inside),), dtype=np.int32)\n",
        "        label.fill(-1)\n",
        "\n",
        "        valid_anchors = torch.Tensor(anchors[index_inside])\n",
        "\n",
        "        # gt와 앵커의 iou 계산\n",
        "        ious = self._box_iou(valid_anchors, bboxs, order='xyxy')\n",
        "\n",
        "        ious = np.array(ious)\n",
        "        bboxs = np.array(bboxs)\n",
        "        labels = np.array(labels)\n",
        "        valid_anchors = np.array(valid_anchors)\n",
        "\n",
        "        # 각 gt에 대해 가장 높은 iou와 그에 해당하는 앵커박스\n",
        "        gt_argmax_ious = ious.argmax(axis=0) # gt와 가장 높은 iou 인덱스\n",
        "        gt_max_ious = ious[gt_argmax_ious, np.arange(len(bboxs))] # iou 추출\n",
        "\n",
        "        # 각 앵커박스에 대해 가장 높은 iou와 그에 해당하는 gt\n",
        "        argmax_ious = ious.argmax(axis=1)\n",
        "        max_ious = ious[np.arange(ious.shape[0]), argmax_ious]\n",
        "\n",
        "        # gt_max_ious를 가진 앵커박스 찾기\n",
        "        gt_argmax_ious = np.where(ious == gt_max_ious)[0]\n",
        "\n",
        "        pos_iou_threshold = 0.7 # positive\n",
        "        neg_iou_threshold = 0.3 # negative\n",
        "\n",
        "\n",
        "        label[max_ious < neg_iou_threshold] = 0\n",
        "        label[gt_argmax_ious] = 1 # gt와 가장 높은 iou를 지닌 앵커박스\n",
        "        label[max_ious >= pos_iou_threshold] = 1 # iou threshold 이상인 앵커박스 \n",
        "\n",
        "        # positive, negative anchors sample\n",
        "        # loss function 계산을 위해 256개의 앵커박스를 sample 합니다.\n",
        "        # positive : negative = 1:1\n",
        "        pos_ratio = 0.5\n",
        "        n_sample = 256\n",
        "        n_pos = pos_ratio * n_sample\n",
        "\n",
        "        # positive\n",
        "        pos_index = np.where(label == 1)[0]\n",
        "        if len(pos_index) > n_pos:\n",
        "            disable_index = np.random.choice(pos_index, size=len(pos_index) - n_pos, replace=False)\n",
        "            label[disable_index] = -1\n",
        "\n",
        "        # negative\n",
        "        n_neg = n_sample - len(np.where(label==1)[0])\n",
        "\n",
        "        neg_index = np.where(label == 0)[0]\n",
        "\n",
        "        if len(neg_index) > n_neg:\n",
        "            disable_index = np.random.choice(neg_index, size=len(neg_index) - n_neg, replace=False)\n",
        "            label[disable_index] = -1\n",
        "\n",
        "        # 앵커 박스 좌표 할당하기\n",
        "        # gt와 앵커 박스의 offset을 계산합니다.\n",
        "        max_iou_bbox = bboxs[argmax_ious]\n",
        "\n",
        "        width = valid_anchors[:, 2] - valid_anchors[:, 0]\n",
        "        height = valid_anchors[:, 3] - valid_anchors[:, 1]\n",
        "        ctr_x = valid_anchors[:, 0] + width*0.5\n",
        "        ctr_y = valid_anchors[:, 1] + height*0.5\n",
        "\n",
        "        base_width = max_iou_bbox[:, 2] - max_iou_bbox[:, 0]\n",
        "        base_height = max_iou_bbox[:, 3] - max_iou_bbox[:, 1]\n",
        "        base_ctr_x = max_iou_bbox[:, 0] + base_height*0.5\n",
        "        base_ctr_y = max_iou_bbox[:, 1] + base_width*0.5\n",
        "\n",
        "        eps = np.finfo(height.dtype).eps\n",
        "        height = np.maximum(height, eps)\n",
        "        width = np.maximum(width, eps)\n",
        "\n",
        "        dy = (base_ctr_y - ctr_y) / height\n",
        "        dx = (base_ctr_x - ctr_x) / width\n",
        "        dh = np.log(base_height / height)\n",
        "        dw = np.log(base_width / width)\n",
        "\n",
        "        anchor_locs = np.vstack((dy, dx, dh, dw)).transpose()\n",
        "\n",
        "        # valid anchor를 제외한 나머지를 채웁니다.\n",
        "        anchor_labels = np.empty((len(anchors),), dtype=np.int32)\n",
        "        anchor_labels.fill(-1)\n",
        "        anchor_labels[index_inside] = label\n",
        "\n",
        "        anchor_locations = np.empty((len(anchors), 4), dtype=np.float32)\n",
        "        anchor_locations.fill(0)\n",
        "        anchor_locations[index_inside] = anchor_locs\n",
        "\n",
        "        return torch.Tensor(anchor_locations), torch.Tensor(anchor_labels)\n",
        "\n",
        "    # encode된 값을 원래대로 복구 및 nms 진행\n",
        "    def decode(self,loc_preds, cls_preds, input_size):\n",
        "        cls_thresh = 0.5\n",
        "        nms_thresh = 0.5\n",
        "\n",
        "        input_size = torch.Tensor([input_size,input_size]) if isinstance(input_size, int) else torch.Tensor(input_size)\n",
        "        anchor_boxes = self._get_anchor_boxes(input_size) # 앵커 박스 생성\n",
        "\n",
        "        loc_xy = loc_preds[:,:2] # 결과값 offset 추출\n",
        "        loc_wh = loc_preds[:,2:]\n",
        "\n",
        "        xy = loc_xy * anchor_boxes[:,2:] + anchor_boxes[:,:2] # offset + anchor\n",
        "        wh = loc_wh.exp() * anchor_boxes[:,2:]\n",
        "        boxes = torch.cat([xy-wh/2, xy+wh/2], 1)\n",
        "\n",
        "        score, labels = cls_preds.sigmoid().max(1)\n",
        "        ids = score > cls_thresh\n",
        "        ids = ids.nonzero().squeeze()\n",
        "        keep = self._box_nms(boxes[ids], score[ids], threshold=nms_thresh) # nms\n",
        "        return boxes[ids][keep], labels[ids][keep]\n",
        "\n",
        "    \n",
        "    # x1,y1,x2,y2 <-> cx,cy,w,h\n",
        "    def _change_box_order(self, boxes, order):\n",
        "        assert order in ['xyxy2xywh','xywh2xyxy']\n",
        "        boxes = np.array(boxes)\n",
        "        a = boxes[:,:2]\n",
        "        b = boxes[:,2:]\n",
        "        a, b = torch.Tensor(a), torch.Tensor(b)\n",
        "        if order == 'xyxy2xywh':\n",
        "            return torch.cat([(a+b)/2,b-a+1],1) # xywh\n",
        "        return torch.cat([a-b/2, a+b/2],1) # xyxy\n",
        "\n",
        "    # 두 박스의 iou 계산\n",
        "    def _box_iou(self, box1, box2, order='xyxy'):\n",
        "        if order == 'xywh':\n",
        "            box1 = self._change_box_order(box1, 'xywh2xyxy')\n",
        "            box2 = self._change_box_order(box2, 'xywh2xyxy')\n",
        "        \n",
        "        N = box1.size(0)\n",
        "        M = box2.size(0)\n",
        "\n",
        "        lt = torch.max(box1[:,None,:2], box2[:,:2])\n",
        "        rb = torch.min(box1[:,None,2:], box2[:,2:])\n",
        "\n",
        "        wh = (rb-lt+1).clamp(min=0)\n",
        "        inter = wh[:,:,0] * wh[:,:,1]\n",
        "\n",
        "        area1 = (box1[:,2]-box1[:,0]+1) * (box1[:,3]-box1[:,1]+1)\n",
        "        area2 = (box2[:,2]-box2[:,0]+1) * (box2[:,3]-box2[:,1]+1)\n",
        "        iou = inter / (area1[:,None] + area2 - inter)\n",
        "        return iou\n",
        "\n",
        "    # nms\n",
        "    def _box_nms(self, bboxes, scores, threshold=0.5, mode='union'):\n",
        "        x1 = bboxes[:,0]\n",
        "        y1 = bboxes[:,1]\n",
        "        x2 = bboxes[:,2]\n",
        "        y2 = bboxes[:,3]\n",
        "\n",
        "        areas = (x2-x1+1) * (y2-y1+1)\n",
        "        _, order = scores.sort(0, descending=True) # confidence 순 정렬\n",
        "        keep = []\n",
        "        while order.numel() > 0:\n",
        "            if order.numel() == 1:\n",
        "                keep.append(order.data)\n",
        "                break\n",
        "            i = order[0] # confidence 가장 높은 anchor 추출\n",
        "            keep.append(i) # 최종 detection에 저장\n",
        "\n",
        "            xx1 = x1[order[1:]].clamp(min=x1[i])\n",
        "            yy1 = y1[order[1:]].clamp(min=y1[i])\n",
        "            xx2 = x2[order[1:]].clamp(max=x2[i])\n",
        "            yy2 = y2[order[1:]].clamp(max=y2[i])\n",
        "\n",
        "            w = (xx2-xx1+1).clamp(min=0)\n",
        "            h = (yy2-yy1+1).clamp(min=0)\n",
        "            inter = w*h\n",
        "\n",
        "            if mode == 'union':\n",
        "                ovr = inter / (areas[i] + areas[order[1:]] - inter)\n",
        "            elif mode == 'min':\n",
        "                ovr = inter / areas[order[1:]].clamp(max=areas[i])\n",
        "            else:\n",
        "                raise TypeError('Unknown nms mode: %s.' % mode)\n",
        "\n",
        "            ids = (ovr<=threshold).nonzero().squeeze()\n",
        "            if ids.numel() == 0:\n",
        "                break\n",
        "            order = order[ids+1]\n",
        "        return torch.LongTensor(keep)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T0pz4_epjbZk",
        "outputId": "6ca3b314-9745-4cd6-b907-1cf65f78f5b8"
      },
      "source": [
        "encoder = DataEncoder()\n",
        "loc_tar, cls_tar = encoder.encode_rpn(loc_targets, cls_targets)\n",
        "\n",
        "print(loc_tar.shape, cls_tar.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([22500, 4]) torch.Size([22500])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1_ms7t2BkxJR"
      },
      "source": [
        "# DataLoader 생성하기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n-pjhAA7b-81"
      },
      "source": [
        "# collate_fn\n",
        "def collate_fn(batch):\n",
        "    encoder = DataEncoder()\n",
        "    imgs = [x[0] for x in batch]\n",
        "    boxes = [torch.Tensor(x[1]) for x in batch]\n",
        "    labels = [torch.Tensor(x[2]) for x in batch]\n",
        "    h,w = 800, 800 # 입력 이미지 크기\n",
        "    num_imgs = len(imgs)\n",
        "    inputs = torch.zeros(num_imgs, 3, h, w)\n",
        "\n",
        "    loc_targets = []\n",
        "    cls_targets = []\n",
        "\n",
        "    # anchor box 생성\n",
        "    for i in range(num_imgs):\n",
        "        inputs[i] = imgs[i]\n",
        "        # loc_target = [x,y,w,h]\n",
        "        loc_target, cls_target = encoder.encode(boxes=boxes[i], labels=labels[i])\n",
        "        loc_targets.append(loc_target)\n",
        "        cls_targets.append(cls_target)\n",
        "    return inputs, torch.stack(loc_targets), torch.stack(cls_targets)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8eiHlSFvqTtI"
      },
      "source": [
        "train_dl = DataLoader(train_ds, batch_size=4, shuffle=True, collate_fn=collate_fn)\n",
        "val_dl = DataLoader(val_ds, batch_size=4, shuffle=True, collate_fn=collate_fn)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 668
        },
        "id": "Pj8EJVVLkZsS",
        "outputId": "9f22a6fc-1691-435d-afb7-ea93429117ff"
      },
      "source": [
        "image = torch.zeros(1, 3, 800, 800)\n",
        "loc_targets = torch.empty(2, 4)\n",
        "cls_targets = torch.empty(2, 1)\n",
        "loc_targets[0] = torch.tensor([300, 200, 500, 500], dtype=float)\n",
        "loc_targets[1] = torch.tensor([100, 400, 150, 450], dtype=float)\n",
        "cls_targets[0], cls_targets[1] = 1, 2\n",
        "torch.tensor(cls_targets, dtype=float)\n",
        "\n",
        "\n",
        "def show(image, loc_targets, cls_targets):\n",
        "    if type(image) == torch.Tensor:\n",
        "        img = image.squeeze(0)\n",
        "    img = to_pil_image(img)\n",
        "    draw = ImageDraw.Draw(img)\n",
        "    loc_targets = np.array(loc_targets)\n",
        "    W, H = img.size\n",
        "\n",
        "    for tg, label in zip(loc_targets, cls_targets):\n",
        "        bbox = tg\n",
        "        label = label\n",
        "\n",
        "        draw.rectangle(((bbox[0], bbox[1]), (bbox[2], bbox[3])), width=3)\n",
        "    plt.imshow(np.array(img))\n",
        "\n",
        "plt.figure(figsize=(10,10))\n",
        "show(image, loc_targets, cls_targets)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 3, 800, 800]) torch.Size([2, 5])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAAJCCAYAAADKjmNEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAaU0lEQVR4nO3df6zdd33f8dc7tmN6U0YS6KwozgYIA+KPEdIIgoIqCqJKsorwB0V41rBQJP9DJ1A7tWGTtlXapPafUlAnq1ZDGybGj6aljhBqmwXQtGkJJBDCj5DGELLYSuLSBrPWqj13n/1xP07e9rz62rnX5yb38ZCOzvf7Od/j87kffK1nvud7DjXGCAAAyy5a9AQAANYTcQQA0IgjAIBGHAEANOIIAKARRwAAzZrEUVXdUFUPV9WBqrp1LV4DAGAt1Gp/z1FVbUry50nekeRgkq8m2TnG+M6qvhAAwBpYizNHb0xyYIzx/THG8SSfTnLzGrwOAMCq27wGf+aVSR5v+weTvOnve0JV+ZpuAOBC++EY46dOH1yLOFqRqtqTZM+iXh8A2PAeO9PgWsTRoSRXtf3tc+wUY4x9SfYlzhwBAOvHWlxz9NUkO6rqFVV1cZL3JrlzDV4HAGDVrfqZozHGiar6xSR/mmRTko+PMb692q8DALAWVv2j/Oc1CW+rAQAX3v1jjGtPH/QN2QAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEBz1jiqqo9X1eGq+lYbu7yq7qqqR+b9ZXO8qupjVXWgqh6sqmvWcvIAAKttJWeOfj/JDaeN3Zrk7jHGjiR3z/0kuTHJjnnbk2Tv6kwTAODCOGscjTH+a5K/Om345iS3z+3bk7yrjX9iLLsnyaVVdcVqTRYAYK2d7zVH28YYT8ztJ5Nsm9tXJnm8HXdwjv0/qmpPVd1XVfed5xwAAFbd5uf6B4wxRlWN83jeviT7kuR8ng8AsBbO98zRUyffLpv3h+f4oSRXteO2zzEAgOeF842jO5Psntu7k+xv4++bn1q7LsmR9vYbAMC6d9a31arqU0nemuRlVXUwyb9N8utJPltVtyR5LMl75uFfSHJTkgNJjiZ5/xrMGQBgzdQYi7/cxzVHAMAC3D/GuPb0Qd+QDQDQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADRnjaOquqqqvlRV36mqb1fVB+f45VV1V1U9Mu8vm+NVVR+rqgNV9WBVXbPWPwQAwGpZyZmjE0l+eYzxuiTXJflAVb0uya1J7h5j7Ehy99xPkhuT7Ji3PUn2rvqsAQDWyOazHTDGeCLJE3P7f1XVQ0muTHJzkrfOw25P8uUkvzrHPzHGGEnuqapLq+qK+ecAz9Ell1yy6Cmwjp04cSLHjh1b9DTgee2scdRV1cuTvCHJvUm2teB5Msm2uX1lksfb0w7OsVPiqKr2ZPnMErBCS0tLefTRR7O0tLToqbBO7d+/P7t27Vr0NOB5bcVxVFU/meQPk3xojPHjqnrmsTHGqKpxLi88xtiXZN/8s8/pubBRVVWWlpaytLSU48ePZ/kELSSbNm3Kli1bsnXr1kVPBZ73VhRHVbUly2H0yTHGH83hp06+XVZVVyQ5PMcPJbmqPX37HANWyfHjx/PqV786R44cWfRUWCd27tyZvXtd4gmr4axxVMuniG5L8tAY4zfbQ3cm2Z3k1+f9/jb+i1X16SRvSnLE9Uaw+o4cOSKOeMbRo0cXPQV4wVjJmaPrk/zzJN+sqgfm2L/KchR9tqpuSfJYkvfMx76Q5KYkB5IcTfL+VZ0xAMAaWsmn1f5bkvr/PPz2Mxw/knzgOc4LAGAhfEM2AEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0Jw1jqrqRVX1lar6RlV9u6p+bY6/oqruraoDVfWZqrp4jm+d+wfm4y9f2x8BAGD1rOTM0bEkbxtjvD7J1UluqKrrkvxGko+MMV6V5Okkt8zjb0ny9Bz/yDwOAOB54axxNJb99dzdMm8jyduS3DHHb0/yrrl989zPfPztVVWrNmMAgDW0omuOqmpTVT2Q5HCSu5J8L8mPxhgn5iEHk1w5t69M8niSzMePJHnpGf7MPVV1X1Xd99x+BACA1bOiOBpj/N0Y4+ok25O8Mclrn+sLjzH2jTGuHWNc+1z/LACA1XJOn1YbY/woyZeSvDnJpVW1eT60PcmhuX0oyVVJMh9/SZK/XJXZAgCssZV8Wu2nqurSuf0TSd6R5KEsR9K752G7k+yf23fO/czHvzjGGKs5aQCAtbL57IfkiiS3V9WmLMfUZ8cYn6+q7yT5dFX9+yRfT3LbPP62JP+pqg4k+ask712DeQMArImzxtEY48EkbzjD+PezfP3R6eN/m+QXVmV2AAAXmG/IBgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AAJoVx1FVbaqqr1fV5+f+K6rq3qo6UFWfqaqL5/jWuX9gPv7ytZk6AMDqO5czRx9M8lDb/40kHxljvCrJ00lumeO3JHl6jn9kHgcA8LyweSUHVdX2JP80yX9I8ktVVUneluSfzUNuT/LvkuxNcvPcTpI7kvx2VdUYY6zetF/YLrnkkkVPIUly4sSJHDt2bNHTAIALakVxlOS3kvxKkhfP/Zcm+dEY48TcP5jkyrl9ZZLHk2SMcaKqjszjf9j/wKrak2TP+U/9hWlpaSmPPvpolpaWFj2V7N+/P7t27Vr0NADggjprHFXVzyc5PMa4v6reulovPMbYl2TffA1nlaaqytLSUpaWlnL8+PEs4oTbpk2bsmXLlmzduvWCvzYALNpKzhxdn+SdVXVTkhcl+QdJPprk0qraPM8ebU9yaB5/KMlVSQ5W1eYkL0nyl6s+8xe448eP59WvfnWOHDlywV97586d2bt37wV/XQBYD84aR2OMDyf5cJLMM0f/coyxq6r+IMm7k3w6ye4k++dT7pz7/2M+/kXXG52fI0eOLCSOjh49esFfEwDWi+fyPUe/muWLsw9k+Zqi2+b4bUleOsd/Kcmtz22KAAAXzkovyE6SjDG+nOTLc/v7Sd54hmP+NskvrMLcAAAuON+QDQDQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgGbzoifAmV100UXZuXNnjh49esFf+/rrr7/grwkA64U4Wqe2bNmSvXv3LnoaALDhiKN15sSJE9m/f3+2bt266KnknnvuWfQUAOCCE0frzLFjx7Jr165FTwMANiwXZAMANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAs6I4qqofVNU3q+qBqrpvjl1eVXdV1SPz/rI5XlX1sao6UFUPVtU1a/kDAACsps3ncOzPjjF+2PZvTXL3GOPXq+rWuf+rSW5MsmPe3pRk77wHVslFF12UnTt35ujRo4ueCuvE9ddfv+gpwAvGucTR6W5O8ta5fXuSL2c5jm5O8okxxkhyT1VdWlVXjDGeeC4TBZ61ZcuW7N27d9HTAHhBWmkcjSR/VlUjye+MMfYl2daC58kk2+b2lUkeb889OMdOiaOq2pNkz/lOHDaiEydOZP/+/dm6deuip8I6dc899yx6CvC8t9I4essY41BV/cMkd1XVd/uDY4wxw2nFZmDtS5JzfS5sVMeOHcuuXbsWPQ2AF7QVXZA9xjg07w8n+VySNyZ5qqquSJJ5f3gefijJVe3p2+cYAMC6d9Y4qqpLqurFJ7eT/FySbyW5M8nuedjuJPvn9p1J3jc/tXZdkiOuNwIAni9W8rbatiSfq6qTx//nMcafVNVXk3y2qm5J8liS98zjv5DkpiQHkhxN8v5VnzUAwBqp5Q+VLXgSrjkCAC68+8cY154+6BuyAQAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgGZFcVRVl1bVHVX13ap6qKreXFWXV9VdVfXIvL9sHltV9bGqOlBVD1bVNWv7IwAArJ6Vnjn6aJI/GWO8NsnrkzyU5NYkd48xdiS5e+4nyY1JdszbniR7V3XGAABr6KxxVFUvSfIzSW5LkjHG8THGj5LcnOT2edjtSd41t29O8omx7J4kl1bVFas+cwCANbCSM0evSPIXSX6vqr5eVb9bVZck2TbGeGIe82SSbXP7yiSPt+cfnGOnqKo9VXVfVd13/tMHAFhdK4mjzUmuSbJ3jPGGJH+TZ99CS5KMMUaScS4vPMbYN8a4doxx7bk8DwBgLa0kjg4mOTjGuHfu35HlWHrq5Ntl8/7wfPxQkqva87fPMQCAde+scTTGeDLJ41X1mjn09iTfSXJnkt1zbHeS/XP7ziTvm59auy7Jkfb2GwDAurZ5hcf9iySfrKqLk3w/yfuzHFafrapbkjyW5D3z2C8kuSnJgSRH57EAAM8LtXy50IInUbX4SQAAG839Z7r22TdkAwA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAM1Z46iqXlNVD7Tbj6vqQ1V1eVXdVVWPzPvL5vFVVR+rqgNV9WBVXbP2PwYAwOo4axyNMR4eY1w9xrg6yU8nOZrkc0luTXL3GGNHkrvnfpLcmGTHvO1JsnctJg4AsBbO9W21tyf53hjjsSQ3J7l9jt+e5F1z++YknxjL7klyaVVdsSqzBQBYY+caR+9N8qm5vW2M8cTcfjLJtrl9ZZLH23MOzrFTVNWeqrqvqu47xzkAAKyZFcdRVV2c5J1J/uD0x8YYI8k4lxceY+wbY1w7xrj2XJ4HALCWzuXM0Y1JvjbGeGruP3Xy7bJ5f3iOH0pyVXve9jkGALDunUsc7cyzb6klyZ1Jds/t3Un2t/H3zU+tXZfkSHv7DQBgXavld8TOclDVJUn+Z5JXjjGOzLGXJvlskn+U5LEk7xlj/FVVVZLfTnJDlj/Z9v4xxt97XVFVndNbcgAAq+D+M13es6I4WmviCABYgDPGkW/IBgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA04ggAoBFHAACNOAIAaMQRAEAjjgAAGnEEANCIIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAAAacQQA0IgjAIBGHAEANOIIAKARRwAAjTgCAGjEEQBAI44AABpxBADQiCMAgEYcAQA0mxc9gemvkzy86EmsIy9L8sNFT2IdsR6nsh6nsh6nsh6nsh6nsh6n+sdnGlwvcfTwGOPaRU9ivaiq+6zHs6zHqazHqazHqazHqazHqazHynhbDQCgEUcAAM16iaN9i57AOmM9TmU9TmU9TmU9TmU9TmU9TmU9VqDGGIueAwDAurFezhwBAKwLC4+jqrqhqh6uqgNVdeui53MhVNXHq+pwVX2rjV1eVXdV1SPz/rI5XlX1sbk+D1bVNYub+dqoqquq6ktV9Z2q+nZVfXCOb8g1qaoXVdVXquobcz1+bY6/oqrunT/3Z6rq4jm+de4fmI+/fJHzXwtVtamqvl5Vn5/7G3YtkqSqflBV36yqB6rqvjm2UX9fLq2qO6rqu1X1UFW9eQOvxWvm34mTtx9X1Yc26no8FwuNo6ralOQ/JrkxyeuS7Kyq1y1yThfI7ye54bSxW5PcPcbYkeTuuZ8sr82OeduTZO8FmuOFdCLJL48xXpfkuiQfmH8PNuqaHEvytjHG65NcneSGqrouyW8k+cgY41VJnk5yyzz+liRPz/GPzONeaD6Y5KG2v5HX4qSfHWNc3T6WvVF/Xz6a5E/GGK9N8vos/z3ZkGsxxnh4/p24OslPJzma5HPZoOvxnIwxFnZL8uYkf9r2P5zkw4uc0wX82V+e5Ftt/+EkV8ztK7L83U9J8jtJdp7puBfqLcn+JO+wJiNJlpJ8LcmbsvzFbZvn+DO/O0n+NMmb5/bmeVwteu6ruAbbs/wP+tuSfD5JbdS1aGvygyQvO21sw/2+JHlJkkdP/994I67FGdbm55L8d+txfrdFv612ZZLH2/7BObYRbRtjPDG3n0yybW5vqDWab4O8Icm92cBrMt9GeiDJ4SR3Jflekh+NMU7MQ/rP/Mx6zMePJHnphZ3xmvqtJL+S5P/M/Zdm467FSSPJn1XV/VW1Z45txN+XVyT5iyS/N992/d2quiQbcy1O994kn5rb1uMcLTqOOIOxnPAb7mOEVfWTSf4wyYfGGD/uj220NRlj/N1YPjW+Pckbk7x2wVNaiKr6+SSHxxj3L3ou68xbxhjXZPltkQ9U1c/0BzfQ78vmJNck2TvGeEOSv8mzbxkl2VBr8Yx5Dd47k/zB6Y9txPU4H4uOo0NJrmr72+fYRvRUVV2RJPP+8BzfEGtUVVuyHEafHGP80Rze0GuSJGOMHyX5UpbfOrq0qk7+X/70n/mZ9ZiPvyTJX17gqa6V65O8s6p+kOTTWX5r7aPZmGvxjDHGoXl/OMvXlLwxG/P35WCSg2OMe+f+HVmOpY24Ft2NSb42xnhq7m/09Thni46jrybZMT95cnGWTwPeueA5LcqdSXbP7d1Zvu7m5Pj75qcKrktypJ0efUGoqkpyW5KHxhi/2R7akGtSVT9VVZfO7Z/I8vVXD2U5kt49Dzt9PU6u07uTfHH+1+Hz3hjjw2OM7WOMl2f534cvjjF2ZQOuxUlVdUlVvfjkdpavLflWNuDvyxjjySSPV9Vr5tDbk3wnG3AtTrMzz76llliPc7foi56S3JTkz7N8TcW/XvR8LtDP/KkkTyT531n+L59bsnxdxN1JHknyX5JcPo+tLH+i73tJvpnk2kXPfw3W4y1ZPs37YJIH5u2mjbomSf5Jkq/P9fhWkn8zx1+Z5CtJDmT5dPnWOf6iuX9gPv7KRf8Ma7Qub03y+Y2+FvNn/8a8ffvkv5sb+Pfl6iT3zd+XP05y2UZdi/kzXpLls6UvaWMbdj3O9+YbsgEAmkW/rQYAsK6IIwCARhwBADTiCACgEUcAAI04AgBoxBEAQCOOAACa/wsKH0Vnls3gWQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ypln-nFSqM2j",
        "outputId": "40f354d9-469f-4767-fbda-d0f6ddf4a45e"
      },
      "source": [
        "encoder = DataEncoder()\n",
        "loc_tar, obj = encoder.encode(loc_targets, cls_targets, input_size=(800,800))\n",
        "print(loc_tar.shape, obj.shape)\n",
        "obj = obj[obj>0.7]\n",
        "print(obj)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([22500])\n",
            "torch.Size([22500, 4])\n",
            "torch.Size([22500, 2])\n",
            "torch.Size([22500, 1])\n",
            "torch.Size([22500, 4]) torch.Size([500, 2])\n",
            "tensor([0.7942])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2Lghs21SMYI"
      },
      "source": [
        "# 모델 구축하기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HzGJY-v10YNZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122,
          "referenced_widgets": [
            "e1b2bc3f80164e3f9f24ccc6bd88e20c",
            "528b6aa6473b4923a5c4cb5dcac4646c",
            "a214b1af5e304f6da26d8d5b6d187342",
            "1a21e0b812fe4a6e897f672c1f955843",
            "7795a77310dd40169a518ec327598b41",
            "b0c8a0ae8a664780b607014c72ebc559",
            "fe52f9921830436bbb126df3f1c86eef",
            "638e4e6f9dfa4b3d8c44aa8a6b3f02f4"
          ]
        },
        "outputId": "dc13662b-5eb4-40ba-b427-ca13f8ed113d"
      },
      "source": [
        "class feature_extractor(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "\n",
        "        # pre-trained VGG16\n",
        "        self.backbone = nn.Sequential(*list(vgg16(pretrained=True).features))[:30]\n",
        "    \n",
        "    def forward(self,x):\n",
        "        return self.backbone(x)\n",
        "\n",
        "# check\n",
        "x = torch.randn(1, 3, 800, 800).to(device)\n",
        "model = feature_extractor().to(device)\n",
        "output = model(x)\n",
        "print(output.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/vgg16-397923af.pth\" to /root/.cache/torch/hub/checkpoints/vgg16-397923af.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e1b2bc3f80164e3f9f24ccc6bd88e20c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=553433881.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "torch.Size([1, 512, 50, 50])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8mpuC_MX28rf",
        "outputId": "45e51d99-e0fe-4167-dcfd-630742c094c5"
      },
      "source": [
        "class RPN(nn.Module):\n",
        "    def __init__(self, init_weights=True):\n",
        "        super().__init__()\n",
        "\n",
        "        mid_channels = 512\n",
        "        in_channels = 512\n",
        "        n_anchor = 9 # 앵커 박스 수\n",
        "\n",
        "        # sliding layer\n",
        "        self.conv1 = nn.Conv2d(in_channels, mid_channels,3,stride=1,padding=1)\n",
        "\n",
        "        # RoI 좌표 예측, x1, y1, x2, y2\n",
        "        self.reg_layer = nn.Conv2d(mid_channels, n_anchor*4, 1, stride=1, padding=0)\n",
        "        # RoI 객체 존재 유무 예측\n",
        "        self.cls_layer = nn.Conv2d(mid_channels, n_anchor*2, 1, stride=1, padding=0)\n",
        "    \n",
        "        if init_weights:\n",
        "            self._initialize_weights()\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.conv1(x)\n",
        "        \n",
        "        pred_anchor_locs = self.reg_layer(x)\n",
        "        pred_cls_scores = self.cls_layer(x)\n",
        "\n",
        "        # anchor target와 같은 형태로 reformat\n",
        "        # [batch, #anchor*H*W, coordinates]\n",
        "        pred_anchor_locs = pred_anchor_locs.permute(0,2,3,1).contiguous().view(1,-1,4)\n",
        "\n",
        "        # [batch, H, W, #anchor*object], object 존재 유무\n",
        "        pred_cls_scores = pred_cls_scores.permute(0,2,3,1).contiguous()\n",
        "        # [batch, #anchor*H*W], resion proposals를 만들때 사용\n",
        "        objectness_scores = pred_cls_scores.view(1,50,50,9,2)[:,:,:,:,1].contiguous().view(1,-1)\n",
        "        # [batch, #anchor*H*W, 2]\n",
        "        pred_cls_scores = pred_cls_scores.view(1,-1,2)\n",
        "\n",
        "        return pred_anchor_locs, pred_cls_scores, objectness_scores\n",
        "\n",
        "    # 가중치 초기화\n",
        "    def _initialize_weights(self):\n",
        "        self.conv1.weight.data.normal_(0,0.01)\n",
        "        self.conv1.bias.data.zero_()\n",
        "        self.reg_layer.weight.data.normal_(0, 0.01)\n",
        "        self.reg_layer.bias.data.zero_()\n",
        "        self.cls_layer.weight.data.normal_(0, 0.01)\n",
        "        self.cls_layer.bias.data.zero_()\n",
        "\n",
        "# check\n",
        "x = torch.randn(1,512,50,50).to(device)\n",
        "model = RPN().to(device)\n",
        "loc, cls, score = model(x)\n",
        "print(loc.shape)\n",
        "print(cls.shape)\n",
        "print(score.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 22500, 4])\n",
            "torch.Size([1, 22500, 2])\n",
            "torch.Size([1, 22500])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f7hSSYnbHdn_"
      },
      "source": [
        "# RPN이 생성한 RoI를 입력값으로 취합니다.\n",
        "class head(nn.Module):\n",
        "    def __init__(self, anchors, indices_and_rois):\n",
        "        super().__init__()\n",
        "        self.size = (7,7)\n",
        "\n",
        "\n",
        "        # RoI layer\n",
        "        self.max_pool = nn.AdaptiveAvgPool2d(size)\n",
        "\n",
        "        # Loc and Cls layer\n",
        "        # 512*7*7 = 25088, RoI layer 출력값을 flatten\n",
        "        roi_head_classifier = nn.Sequential(nn.Linear(25088,4096),\n",
        "                                            nn.Linear(4096, 4096))\n",
        "        \n",
        "        # loc\n",
        "        # (voc class 20 + background 1 * 4 coordinates)\n",
        "        self.cls_loc = nn.Linear(4096, 21*4)\n",
        "        \n",
        "        # cls\n",
        "        self.cls_score = nn.Linear(4096,21)\n",
        "\n",
        "    \n",
        "\n",
        "\n",
        "        \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hGhSgdgQiSeN"
      },
      "source": [
        "계획\n",
        "dataloader로 target anchor 박스 생성\n",
        "\n",
        "학습시에 rpn까지 학습해서 예측값 받음\n",
        "\n",
        "train 함수에서 벌어지는 일\n",
        "예측값과 생성한 anchor로 roi 만들고 nms 적용해서 fast r-cnn에 전달\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VNPeJ2hmvsHI"
      },
      "source": [
        "# RPN 출력값으로 RoI 생성하는 함수"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BxthXim9vvr1"
      },
      "source": [
        "# RPN 출력값으로 RoI를 생성합니다.\n",
        "def generating_roI(pred_anchor_locs, pred_cls_scores, objectness_scores):\n",
        "    # pred_anchor_loc, pred_cls_scores, objectness_score: RPN 출력값\n",
        "    \n",
        "    encoder = DataEncoder() # 앵커 박스를 생성하기 위해 DataEncoder 생성\n",
        "    input_size = (800,800)\n",
        "\n",
        "    nms_thresh = 0.7\n",
        "    n_train_pre_nms = 12000 # nms 수행 전 roi 수\n",
        "    n_train_post_nms = 2000 # nms 수행 후 roi 수\n",
        "    n_test_pre_nms = 6000\n",
        "    n_test_post_nms = 300\n",
        "    min_size = 16 # stride\n",
        "\n",
        "    input_size_tr = torch.Tensor([input_size, input_size]) if isinstance(input_size, int) else torch.Tensor(input_size)\n",
        "\n",
        "    # 앵커 박스 생성, [x,y,w,h]\n",
        "    anchors = np.array(encoder._get_anchor_boxes(input_size_tr))\n",
        "    \n",
        "    # 앵커 박스 정보 추출\n",
        "    anc_ctr_x = anchors[:,0]\n",
        "    anc_ctr_y = anchors[:,1]\n",
        "    anc_width = anchors[:,2]\n",
        "    anc_height = anchors[:,3]\n",
        "\n",
        "    pred_anchor_locs_np = pred_anchor_locs[0].data.numpy()\n",
        "    objectness_scores_np = objectness_scores[0].data.numpy()\n",
        "\n",
        "    # RPN 출력값 정보 추출\n",
        "    dx = pred_anchor_locs_np[:,0]\n",
        "    dy = pred_anchor_locs_np[:,1]\n",
        "    dw = pred_anchor_locs_np[:,2]\n",
        "    dh = pred_anchor_locs_np[:,3]\n",
        "\n",
        "    # 앵커 박스와의 offset 계산\n",
        "    ctr_x = dx * anc_width + anc_ctr_x\n",
        "    ctr_y = dy * anc_height + anc_ctr_y\n",
        "    w = np.exp(dw) + anc_width\n",
        "    h = np.exp(dh) + anc_height\n",
        "\n",
        "    # roi 생성, x1,y1,x2,y2\n",
        "    roi = np.zeros(pred_anchor_locs_np.shape, dtype=anchors.dtype)\n",
        "    roi[:,0] = ctr_x - 0.5*w\n",
        "    roi[:,1] = ctr_y - 0.5*h\n",
        "    roi[:,2] = ctr_x + 0.5*w\n",
        "    roi[:,3] = ctr_y + 0.5*h\n",
        "\n",
        "    # roi가 이미지 사이즈를 벗어나지 않도록 clip\n",
        "    roi[:,0:4:2] = np.clip(roi[:,0:4:2],a_min=0,a_max=800)\n",
        "    roi[:,1:4:2] = np.clip(roi[:,1:4:2],a_min=0,a_max=800)\n",
        "\n",
        "    # 너비와 높이가 16보다 작은 roi 제거\n",
        "    keep = np.where((h >= min_size) & (w >= min_size))[0]\n",
        "    roi = roi[keep]\n",
        "    score = objectness_scores_np[keep]\n",
        "\n",
        "    # score가 높은 순서대로 정렬\n",
        "    order = score.ravel().argsort()[::-1]\n",
        "    \n",
        "    # score 순으로 roi 12000개 추출\n",
        "    order = order[:n_train_pre_nms]\n",
        "    roi = torch.tensor(roi[order]) # (12000, 4)\n",
        "    score = torch.tensor(score[order]) # (12000,)\n",
        "\n",
        "\n",
        "    # nms 적용하기\n",
        "    keep = encoder._box_nms(roi,score,threshold=nms_thresh)\n",
        "    # 2000개 추출\n",
        "    keep = keep[:n_train_post_nms]\n",
        "    \n",
        "    roi = roi[keep] # x1,y1,x2,y2\n",
        "    return roi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W1DfT3W36CZI",
        "outputId": "034cdd4a-1eb5-4652-84a1-a1811a12e1f4"
      },
      "source": [
        "# check\n",
        "x = torch.randn(1,512,50,50).to(device)\n",
        "model = RPN().to(device)\n",
        "loc, cls, score = model(x)\n",
        "print(loc.shape)\n",
        "print(cls.shape)\n",
        "print(score.shape)\n",
        "\n",
        "roi = generating_roI(loc,cls,score)\n",
        "print(roi.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 22500, 4])\n",
            "torch.Size([1, 22500, 2])\n",
            "torch.Size([1, 22500])\n",
            "torch.Size([2000, 4])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eEuKQSn_C2x2"
      },
      "source": [
        "# RoI sampling 함수\n",
        "- Faster R-CNN은 2000개의 roi 중에서 positive 32, negative 96개를 sample 합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GRXeUvgTDGA5"
      },
      "source": [
        "def roi_sampling(roi, boxes, labels):\n",
        "    # roi = [x1,y1,x2,y2], loc_targets = [x,y,w,h]\n",
        "\n",
        "    encoder = DataEncoder()\n",
        "\n",
        "\n",
        "    # roi와 target 사이의 iou 계산\n",
        "    rois = torch.tensor(roi)\n",
        "    ious = np.array(encoder._box_iou(rois,boxes,order='xyxy'))\n",
        "    \n",
        "    # roi에 label 할당\n",
        "    gt_assignment = ious.argmax(axis=1)\n",
        "    max_iou = ious.max(axis=1)\n",
        "    gt_roi_labels = labels[gt_assignment]\n",
        "    \n",
        "    # roi sample\n",
        "    n_sample = 128\n",
        "    pos_ratio = 0.25\n",
        "    pos_iou_thresh = 0.5\n",
        "    neg_iou_thresh_hi = 0.5\n",
        "    neg_iou_thresh_lo = 0.0\n",
        "\n",
        "    # positive sample\n",
        "    pos_index = np.where(max_iou>=pos_iou_thresh)[0]\n",
        "    pos_roi_per_this_image = n_sample * pos_ratio # 32\n",
        "    # positive sample이 32개보다 적은 경우도 고려\n",
        "    pos_roi_per_this_image = int(min(pos_roi_per_this_image, pos_index.size))\n",
        "    if pos_index.size > 0:\n",
        "        pos_index = np.random.choice(pos_index, size=pos_roi_per_this_image, replace=False)\n",
        "    \n",
        "    # negative sample\n",
        "    neg_index = np.where((max_iou < neg_iou_thresh_hi) & (max_iou_thresh_lo))[0]\n",
        "    neg_roi_per_this_image = n_sample - pos_roi_per_this_image\n",
        "    if neg_index.size > 0:\n",
        "        neg_index = np.random.choice(neg_index, size=neg_roi_per_this_image, replace=False)\n",
        "\n",
        "    # positive, negative sample 결합\n",
        "    keep_index = np.append(pos_index, neg_index)\n",
        "    gt_roi_labels = gt_roi_labels[keep_index]\n",
        "    gt_roi_labels[pos_roi_per_this_image:] = 0\n",
        "    sample_roi = roi[keep_index]\n",
        "\n",
        "\n",
        "    # 작업하기\n",
        "    # head loss 계산을 위한 gt_sample 만들어야함\n",
        "\n",
        "\n",
        "\n",
        "    return sample_roi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GoBDge0Cvh2R"
      },
      "source": [
        "# 손실 함수"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G0suGLScURHU"
      },
      "source": [
        "# RPN Loss\n",
        "def RPNLoss(loc_targets, cls_targets, loc_preds, cls_preds):\n",
        "    loc_pred = loc_preds[0]\n",
        "    cls_pred = cls_preds[0]\n",
        "\n",
        "    # Cross Entropy Loss\n",
        "    rpn_cls_loss = F.cross_entropy(cls_pred, cls_targets.long(), ignore_index=-1)\n",
        "\n",
        "    # Regression Loss\n",
        "    mask = cls_targets > 0 # 객체가 존재하는 경우\n",
        "    mask_loc_preds = loc_pred[mask]\n",
        "    maxk_loc_targets = loc_targets[mask]\n",
        "\n",
        "    # smooth L1 Loss\n",
        "    x = torch.abs(mask_loc_targets - mask_loc_preds)\n",
        "    rpn_loc_loss = (x<1).float() * -.5 * x**2 + (x>1).float()*(x-0.5)\n",
        "\n",
        "    # cls, reg Loss 결합하기\n",
        "    rpn_lambda = 10\n",
        "    N_reg = (gt_rpn_score>0).float().sum() # positive 수\n",
        "    rpn_loc_loss = rpn_loc_loss.sum()/N_reg\n",
        "    rpn_loss = rpn_cls_loss + rpn_lambda*rpn_loc_loss\n",
        "    return rpn_loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xf1lAnpXCQFK"
      },
      "source": [
        "# head loss\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}