Machikado_evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# detectron2 for まちカドまぞく ～評価編～\n",
    "\n",
    "<img src=\"https://user-images.githubusercontent.com/33882378/79189949-02f72b80-7e5e-11ea-81e4-cdc58a3d33c9.jpg\">\n",
    "\n",
    "detectron2 で独自データセット学習したモデルの AP, mAP を評価してみる。\n",
    "\n",
    "> 参考にさせていただいたホームページ（感謝です）<br/>\n",
    "> https://qiita.com/tmtakashi_dist/items/863e1781b5252e453b47<br/>\n",
    "> https://qiita.com/panchokyutech/items/2c972ede2bc883597a87<br/>\n",
    "> https://qiita.com/mdo4nt6n/items/08e11426e2fac8433fed<br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import cv2\n",
    "import random\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## VoTT Export からの読み込み"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# VoTT のエクスポートファイルや、画像が格納されているディレクトリ\n",
    "BASE_DIRECTORY = './vott-json-export/'\n",
    "# VoTT のエクスポートファイル名\n",
    "EXPORT_FILENAME = 'Machikado-export.json'\n",
    "# 訓練データに使用する割合\n",
    "TRAIN_RATIO = 0.8\n",
    "# 乱数シード\n",
    "RANDOM_STATE = 0  # 訓練と同じにしておかないとズレちゃうよ！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "警告: name: 59.jpg - 画像サイズが不一致であるためスキップ image_size:(268, 201), ./vott-json-export/Machikado-export.json: (600, 600)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Metadata(name='test', thing_classes=['シャミ子', 'ごせん像', 'リリス', 'もも', 'みかん', 'その他'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from detectron2.data import DatasetCatalog, MetadataCatalog\n",
    "from machikado_util.Machikado_vott import get_cat_names, get_cat_color, get_machikado_dicts\n",
    "\n",
    "# vott の読み込み関係は py ファイルにした\n",
    "CAT_NAME2ID, CAT_ID2NAME = get_cat_names(os.path.join(BASE_DIRECTORY, EXPORT_FILENAME))\n",
    "dataset_dicts = get_machikado_dicts(os.path.join(BASE_DIRECTORY, EXPORT_FILENAME), BASE_DIRECTORY, CAT_NAME2ID)\n",
    "\n",
    "# VoTT のカテゴリの色も抽出する\n",
    "CAT_IDS, CAT_COLORS = get_cat_color(os.path.join(BASE_DIRECTORY, EXPORT_FILENAME))\n",
    "CAT_IDS, CAT_COLORS\n",
    "\n",
    "# 日本語表示を試す(CAT_ID2NAME と順番を一致させること！)\n",
    "CAT_NAME_JP = ['シャミ子', 'ごせん像', 'リリス', 'もも', 'みかん', 'その他']\n",
    "    \n",
    "# 訓練用、テスト用に分ける\n",
    "random.seed(RANDOM_STATE)\n",
    "random.shuffle(dataset_dicts)\n",
    "\n",
    "split_idx = int(len(dataset_dicts) * TRAIN_RATIO) + 1\n",
    "\n",
    "# 登録\n",
    "DatasetCatalog.clear()\n",
    "DatasetCatalog.register('train', lambda : dataset_dicts[:split_idx])\n",
    "DatasetCatalog.register('test', lambda : dataset_dicts[split_idx:])\n",
    "\n",
    "# 日本語名はこっち\n",
    "MetadataCatalog.get('train').set(thing_classes=CAT_NAME_JP)\n",
    "MetadataCatalog.get('test').set(thing_classes=CAT_NAME_JP)\n",
    "\n",
    "# VoTT のカテゴリ名はこっち\n",
    "# MetadataCatalog.get('train').set(thing_classes=list(CAT_NAME2ID.keys()))\n",
    "# MetadataCatalog.get('test').set(thing_classes=list(CAT_NAME2ID.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 評価\n",
    "\n",
    "* AP、mAP を計算したいと思いますが、ちょっと自信がないので説明はしません。\n",
    "* コードは ./machikado_util/calc_ap.py にありますので、興味がお有りでしたらご覧下さい。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from detectron2.config import get_cfg\n",
    "\n",
    "cfg = get_cfg()\n",
    "cfg.OUTPUT_DIR = './output'\n",
    "\n",
    "# cfg.merge_from_file(\"../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\") # ここの位置は ipynb ファイルの層泰一なので注意\n",
    "# cfg.SOLVER.IMS_PER_BATCH = 2\n",
    "# cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, 'model_final.pth')\n",
    "\n",
    "cfg.merge_from_file('../configs/COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml')\n",
    "cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, 'model_final.pth')\n",
    "\n",
    "cfg.DATALOADER.NUM_WORKERS = 2\n",
    "cfg.MODEL.ROI_HEADS.NUM_CLASSES = len(CAT_ID2NAME)  # machikado は6\n",
    "\n",
    "cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.6   # set the testing threshold for this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from detectron2.engine import DefaultPredictor\n",
    "from machikado_util.calc_ap import append_masks, get_true_datas, predict_datas, make_info_dict\n",
    "\n",
    "# データセットはテストを使用します\n",
    "catalog_name='test'\n",
    "# データセットにマスクデータを追加する\n",
    "append_masks(dataset_dicts)\n",
    "\n",
    "# 推論器を用意する\n",
    "predictor = DefaultPredictor(cfg)\n",
    "\n",
    "# 教師データから評価に必要な情報を取得する\n",
    "true_dicts = get_true_datas(catalog_name=catalog_name)\n",
    "# データセットを一度に推論して結果を取得する\n",
    "pred_dicts = predict_datas(predictor, catalog_name=catalog_name, verbose=False)\n",
    "# precision, recall を計算する\n",
    "df_dict = make_info_dict(true_dicts=true_dicts, pred_dicts=pred_dicts, classes=list(CAT_ID2NAME.keys()), th=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PR(Precision-Recall)曲線"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABDAAAAI4CAYAAACcFxlBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeZzddXno8c8zk4VhnQgZMGODCzrWumGNVbE6Um1obSxKpdpra8GKtbeLrYRLWrVWLKWN3tbbKoJLva0sck0aTLVGbRyhyBIwSNymkqopQ0KAMISEISQzz/3jnMFJODOZ5Sy/c87n/XrNi/ntz5M585wfz3x/3xOZiSRJkiRJUpF1NDoASZIkSZKkw7GBIUmSJEmSCs8GhiRJkiRJKjwbGJIkSZIkqfBsYEiSJEmSpMKzgSFJkmouIo6OiMsjYmEVzrUoIuYfZp/XR8Sp5e/fGhFnzPW6kiSpscKPUVWzi4jPALcBH87MsSqf+1rgKOALwOczc+sU+34VOKHCpp8CdgHPzsx91YxPkookIjqB44CHgAPAAmARpTr6UWAN8NTMvLC8/9nAv2XmQ+Xl9wH3ZeY/HOY6VwGbMvN/T7HPvwDvBI4Evgz8uBzL9sx81RzSlCRJDeIIDLWCPwdeC3zmcDtGREdEvGAG53498BfAScDXIuI/IuJtlf7yl5mvysznj38BrwC+BnwD6Ld5IakNLAK+CGwuf/038EPgVZQaGE8C3jdh/wuA42dygYh4PvBS4NIp9jmCUtPkeODfKTWRDwAnAsdGxEBE/P5MritJrSIifhQRp0xY/u3yHwTntG+FYzsjYn5ExNyjlkrmNToAabYi4gnArwOfoHRzvPSQ7ScDR2fmdyasTuDzEfHkzDxwuGtk5ihwPXB9RKwCfhF4RWbuP0xs/cDfAe/LzD+eflaS1Lwy8z7gxRGxgFJz+cnA+Zm5HSAi1o+PlCs/StIBPDEiuoBjyvufGBFvAI4GPp2PHyp6MfDuzByZIpRfBe4GfhPYnZnPK19zIDP7yzX6zCqkLEkqi4h/BM4AgtI99yhwBHAt8NYGhqYWYgNDzexoSjepbwfekJk/GN9QHsb8GeCfgYkNjAD+C/jDiDgKeDbwLOCqzLz40AtExErgjZSGPV+ZmV8CvjSN2E4BvpSZ62aTmCQ1uU7glsz8s4krM3Os3Hy+DtgHPIPSiIx7gT1AH3AfpYb0HkoNjtHx4yPiZeVtVxzm+u8EBjPzjyPiNeVH/ACeV/5+EaXmtCSpSjLznEPXlZsaWxoQjlqUDQw1rczcBpxRHgZ86IiIvwYezMzLx1dExFcozUcRwHLgMuAa4HPAV6kgM1dHxEbgLOAbEXE78I7M/PGE8z4FWH/IoYuABRHxK4es/6XM/O+ZZSpJzSEiOjNztDw64tpK+2TmLkrNYyLiVuDtmfmj8vL7mHoOjL8EbgY2RsQdmfmHFWL4VeD+Cdd7RkS8A/g94LeAnwH+AbgoIhb6eJ8kAaURE1Xdt3yPfAbwuFotzZZzYKjpZeY/TLj57YiIjwGnA285ZNczM/OZwJ8AuzJzLfBm4BuZecuh5y0/Q01m3paZf0ppaPOngKFDrv/DzHz2xC/gj4A1h663eSGpxV0cEW+c6UERsTAi+ihNuFlxMuaIeAXwcmAn8Abg9Ih49oTtR5a//T6lRgcRcXpEDADdlEZ0LAFeA/wr8C2bF5IEwLHAcDX3Lc978RHgwvGJmqVqsIGhlhERyygNCe6jNGnm/RO3Z+be8rc3Az9XfsxkITDZHBV/FxE/iIiLI+LUzHw0Mz83nbkzJKndlG9WfwP47CwOn0dp3qBXAF+fZJ93A5dm5qrMvBe4HegpX/sUSpMmk5mDlD4FBeA/gQ8Ar6T0WOF7gM9Talx/ehZxSlIreC+lR/fGLaU06fJc953oPZT+oHj7bAKUJmMDQ00tIo6LiDeVPy7vi5RuUH8hM3dPdkx5krlR4ITMvCAzH55kv9+lNMfGKHBNRHwzIt5SnpxOkjRBebLNPcDvRsSx4+vLs9A/PSKeUeGwh4DnZ+bezPylzPy5QyZeHj/HiynNl/GuCau3U5rTYjFwJfChCudfDvw28PeUJn2+GbgVeHl5RIcktZ3M/KfMfBBKo5cpPSq9Ya77jouI91K6h/514AsR8eYqhq82ZwNDze4C4HcpFdKnZ+al4zPcH8bNwIsPt1Nmfjcz30Ppxvm9wLnADyOi4sf+RcRR5eJ+JDN7llCSWsGrgWcCt0bEjoi4n9Jf7tZR+hSnQ10M/EVE3BURd0fE9ogYiohPHLLfLwPvPeSTRz4F/D6lv+59IjOvqXD+a4F7gJWUHhncROlxkjcDl5cfW5GkdvYB4PuZ+a257hsRvRFxDaV5L34xM6+lNArjooj4s0rHSDMVj/90Mqn1RcTzgQcmTsY5g2OfPD7nRoVtv0Ppr4ALgJVTTEQnSaqR8twY5wMXUXp85B7gEkqPqizPzB9FxOLyoyiS1HbKj959GDieUl18cC77RsTbKN0Df4zSR10/OmHbUyg9HnhBZl5d1UTUdmxgSJKkllIeCXfEZI8ISlK7i4geSo+DXJ6Zo3PdNyJ6gaPL8xBV2r4E2DHNkdLSpGxgSJIkSZKkwnMODEmSJEmSVHg2MCRJkiRJUuHNa3QA09Xd3Z2nnHJKo8Oom71793LUUUc1Ooy6MNfWZK7Tc9ttt92XmYurHFLNtVNN9rXcmsy1NVmPW5uv5dbVTvma6/RMVpObpoFx4okncuuttzY6jLoZGBigv7+/0WHUhbm2JnOdnoiY8SfhFEE71WRfy63JXFuT9bi1+VpuXe2Ur7lOz2Q12UdIJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MCRJkiRJUuHZwJAkSZIkSYVnA0OSJEmSJBWeDQxJkiRJklR48xodQC2t2zzE6g2D3D08wpLuLlYu7+PMU3ub4tih4RF6b9o47WMbHa+5Tu/YmeY6l+s2Y65SPczl9V0UrZADtEYerZCDJFnL1Cxq0sCIiF8DzgZenJlLK2w/Gzgf6AQGMvNd1Y5h3eYhVq3dwsj+UQCGhkdYtXYLwGF/GZvt2GaL11z9d1J9FaEmF0UrvEZbIQdojTxaIQfVl/VYRWQtUzOp1QiMe4HfA7596IaIOBm4CHgRsBu4OiLOysw11Qxg9YbBx34Jx43sH+W9136be3Y/MuWxH/nanU11bLPFO5djmy3eRh1btHhXbxj0DbCxGl6Ti2Ky94YLPncHV92yreIxw8MjXDp4Yz3Cm5bN24Z5dHTsoHWHy2G66plrLfOYjmrkOlkO1jxNwXqswpnsvdFapiKKzKzdySN2ZOZJh6x7O3ByZv5pefl04JzM/M0Kx58HnAewePHin73mmmumfe3f/tLeuYQuqco+fcZRjQ6hZvbs2cPRRx89q2Nf+cpX3paZL6xySBU1siYXxVTvDaccO9m0UKOU/hhaDHfuHpt02+Q5TFf9cq1tHtMx91ynyqFINW8uNarZWI9bm6/l2pjqvbFetcyfbWuqRU1uxBwYxwM7JixvB3oq7ZiZlwOXA/T19WV/f/+0L9J700aGhkcet757wRG8+wWvmPLYD3zz6ww/+vi/NBf12GaLdy7HNlu8jTq2aPH2dncxk9/fZjMwMNDM+dWlJhfFZO8NixZ28UfPOb3iMWNjA3R09Nc2sBl4760beWDfzHKYrnrmWss8pqMauU6WQ9FqXpPXqBlp8lzbqh7PRpP/fGeknrlO9t5Yz1rmz7Y11SLXRnwKyT0cXIxPKq+rqpXL++iaf/BfVuZ3dPLak5/Jws55U3699uRnMr+jeY5ttnjNtb3+nbrmd7JyeR8qrLrU5KKY7L1hxdLmeY2uWNpX8Xe0mXKA1sijUg7WPM1BW9VjFUel90ZrmYqqESMwvgh8NSL+OjMfAs4F1lX7IuPPa41/GsKihV2sWNrHsp7DP8c1vs/6bYM8sK/4xzZbvHM5ttnibdSxjY73M3d+i7FMep3FuhnUpSYXxVzeG4piLr+jRdIKeYzHeuWdd3Agx6x5mqu2qscqjonvjX4KiYqubg2MiLgauCQzb4+Ii4HrIuJR4PpaTU505qm9nHlqLzfcALt2zezYZT29s76JqsaxMx3a2uh453Ksudbmuo3MdcPQDzj52Ee45p21Hwau2WlETS6K8feG9esbHcnszeV3tEhaIY9lPb18455tHH88fPbtL2l0OGpC7VyPVRzj741S0dW0gTFxcqLMfOOE768ArqjltSVJB7MmS1IxWI8laXYaMQeGJEmSJEnSjNjAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MCRJkiRJUuHZwJAkSZIkSYVnA0OSJEmSJBWeDQxJkiRJklR4NjAkSZIkSVLh2cCQJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeDYwJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFd68RgcgSePWbR5i9YZB7h4eYUl3FyuX93Hmqb2NDkuSJElSAdjAkFQI6zYPsWrtFkb2jwIwNDzCqrVbAGxiSJIkSfIREknVtWnnEPeOPMwtO0Y57ZKNrNs8VHG/0bHk3of28d27dzMwuJP3rf/OY82LcSP7R1m9YbAeYUuSJEkqOEdgSKqaTTuHuGrrFsYygdIoipWf+xZfuONuFh21gJ0P7WPn7n3c+9A+du19lNHyflO5e3ik1mFLkiRJagI2MCRVzfptg+wfO3gUxf7R5Cvf28mihUdwzPyFHDP/CE456jiO7V7IsQvKX/MX8qnBb/Lgo/sed84l3V31Cl+SJElSgdnAkFQ1D+ybfLTE+1/4C1Me+6sn/zRXbb2D/WNjj63rmt/JyuV9VYtPkiRJUvNyDgxJVbNoYeXREpOtn2hZTy+vf8qzHlvu7e7ir17/HCfwlCRJkgTYwJBURSuW9jG/o/OgdfM7OlmxdHqjKF5wwhIA3vMrz+KGC0+3eSFJkiTpMT5CIqlqlvWUGg7rtw3ywL4RFi3sYsXSvsfWS5IkSdJs2cCQVFXLenpZ1tPL2NgAHR39jQ5HkiRJUovwERJJkiRJklR4NjAkSZIkSVLh2cCQJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeDYwJEmSJElS4dnAkCRJkiRJhTev0QFIUiOt2zzE6g2D3D08wpLuLlYu7+PMU3sbHZYkSZKkQ9jAkNS21m0eYtXaLYzsHwVgaHiEVWu3ANjEkCRJkgrGBoakpjebURT7Doxy8Re/91jzYtzI/lFWbxi0gSFJkiQVTM0aGBFxNnA+0AkMZOa7Dtl+EfBqYAFwaWZ+vFaxSGpdU42iePWzTuTH9z/Mtl17+dH9D/Pj+x/mx/fv5Uf3Pcz2B0fISc559/BInaKvD+uxJBWHNVmSZq8mDYyIOBm4CHgRsBu4OiLOysw15e2vAp4PvBQ4Avh6RHwtM++sRTySWtfffOn7FUdR/Mk1tzN2SIfi6PkLOOGII1lyxCKe+1NP4vrtP2Lvgf2PO+eS7q4aRlxf1mNJKg5rsiTNTa1GYJwBrMnMBwEi4jLgHGBNefupwL9n5hjwcET8G6VO80HFOSLOA84DWLx4MQMDA7MKZu9eGBub1aENtIexsYFGB1En5tqaZp7rbTsfBeCif/0uH/3q9zjrGfN56ZL5jGXywCPJjr3J9r1jbN87xo69Y+zYm9z/SOVxFGMJv7J0Pics7OCEI4ITjuiga14A+4EHgQdZvDC4+r9g/4T6sKADXrN0dEb1Zs+ePbOuT3VQlXpcPrYqNbkopv++4O9ta6pmriMMD1PY34mC16iqaoJcC3WP3Gya4OdbNe2UK7RXvuY6N7VqYBwP7JiwvB3ombC8GfhfEXEpcCTwGuCzh54kMy8HLgfo6+vL/v7+WQVzww2wa9esDm2YsbEBOjr6Gx1GXZhra5pprpt2DvEvP77jseX7H0k+ueVR/vXHHex6eD+PTBhlcURnJz1dx/CUY47i4f07GRk98LjzLVrYxfKfOn3Ka77oRIgY4pr/+jaPjB6gd5afQjIwMMBs61MdVKUeQ/VqclGsXz+9/fy9bU3VzfVGuruhv/8lVTpfdRW8RlVVE+RaqHvkZtMEP9+qaadcob3yNde5qVUD4x7gKROWTyqvAyAzvxoRzwO+Vl5/C7CtRrFIagLrtw2y/5A/iY8m7HzoUV524smc2HUUPV1HcWLX0Ry7YCERAZQaH1dt3cL+sZ80OOZ3dLJiad+0rrusp5edI3vYcNed3HDh1A2PJmU9lqTisCZL0hx01Oi8XwReFxHHlJfPBa4d3xgRxwLXZuZLgbcAzwX+rUaxSGoCD+yrPHHmgbExznrqs3jZE0/mGd0ncNzCIx5rXkCpAfGmpz2HRQtL81YsWtjFm572HJb1+CkiZdZjSSoOa7IkzUFNRmBk5vaIuBi4LiIeBa7PzDURMQC8sbzb30bEIuAA8GfjzwJKak+LFnZVbGKMNyamsqyn14bFJKzHklQc1mRJmpuafYxqZl4BXHHIuv4JiytqdW1JzWfF0r45PQqiyVmPJak4rMmSNHs1a2BI0kyMj6BYv22QB/aNsGhhFyuW9jmyQpIkSRJgA0NSgfgoiCRJkqTJ1GoST0mSJEmSpKqxgSFJkiRJkgrPR0gkaZbWbR5i9YZBhoZH6L1pIyuX93HmqT4CI0mSJNWCDQxJmoV1m4dYtXYLI/tLn5oyNDzCqrVbAGxiSJIkSTXgIySSNAt/s+H7jzUvxo3sH2X1hsEGRSRJkiS1NkdgSGprm3YOMbD9RyRw2iWPfwwkM9mx+xEGdzzEf97zEN/f8RCDOx7i7uFHKp7v7uGR+gQuSZIktRkbGJLa1qadQ1y1dQv7x37yGMgFa+7g+h/cy8L5naWmxY6HeGjfgceO6V64kJO6jmFhZyf7Rkcfd84l3V11i1+SJElqJzYwJLWt9dsGH2tejHv0wBhrvjnEkfPm8cQjj+V5i5bwxCOPYcmRx/LEI4/mqPkLgMc3PwC65neycnlfXXOQJEmS2oUNDElt64F9kz/uccmLfpGImHT7sp7SYyZX3nkHB3KM3u4uP4VEkiRJqiEbGJLa1qKFXRWbGIsWdk3ZvBi3rKeXm3fexWjs4qsXnl6LECVJkiSV+SkkktrWiqV9zO/oPGjd/I5OViz1MRBJkiSpaByBIaltjT8Gsn7bIA/sG2HRwi5WLO17bL0kSZKk4rCBIamtLevptWEhSZIkNQEfIZEkSZIkSYVnA0OSJEmSJBWeDQxJkiRJklR4NjAkSZIkSVLh2cCQJEmSJEmFZwNDkmZp084htu7exZ3DY5x2yUbWbR5qdEiSJElSy7KBIUmzsGnnEFdt3cKBHANgaHiEVWu32MSQJEmSasQGhiTNwvptg+wfGz1o3cj+UVZvGGxQRJIkSVJrs4EhSbPwwL6RiuvvHq68XpIkSdLc2MCQpFlYtLCr4vol3ZXXS5IkSZobGxiSNAsrlvYxv6PzoHVd8ztZubyvQRFJkiRJrW1eowOQpGa0rKcXgCvvvIMDOUZvdxcrl/dx5qm9DY5MkiRJak02MCRplpb19HLzzrsYjV189cLTGx2OJEmS1NJ8hESSJEmSJBWeDQxJkiRJklR4NjAkSZIkSVLh2cCQJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeNP6GNWIWAK8AThufF1mvr9WQUlSM9i0c4itu3dxIMc47ZKNrFzex5mn9tb8utZkqTg27RziRw8Nc+fu+tYBFYP1WJLqa7ojMK4FjgGGJnxNKSLOjohbIuK2iPjQIds6I+LDEXFTeZ9LI2L+jKOXpAbZtHOIq7Zu4UCOATA0PMKqtVtYt/mw5bEaZlSTrcdSbTS4DqgYvEeWpDqa1ggMYHdmfmC6J42Ik4GLgBcBu4GrI+KszFxT3uWXgd7MfHF5/7XAmcD/m3bkktRA67cNsn9s9KB1I/tHWb1hsB5/fZ12TbYeS7XT4DqgYvAeWZLqaLoNjIGIeC3wpfEVmfnoFPufAazJzAcBIuIy4BxgvDjfBcyLiPERIPuB7x56kog4DzgPYPHixQwMDEwz3IPt3QtjY7M6tIH2MDY20Ogg6sRcW1Nr5/rAvpGK64eGR2Zdq2ZgJjW5KvW4fGxVanJRTP99obVfywcz15locB2Ytj179hQqnlpqQK5NfY/cbHwtt652ytdc52a6DYzfoFRco7ycwFOn2P94YMeE5e1Az/hCZm6OiK8Dl5RXDWTmdw49SWZeDlwO0NfXl/39/dMM92A33AC7ds3q0IYZGxugo6O/0WHUhbm2plbPddHCjRX/56W3u4vZ1qoZmElNrko9Lu9blZpcFOvXT2+/Vn8tT2SuM9PgOjBtAwMDhYqnlhqQa1PfIzcbX8utq53yNde5mdYcGJn505n51Mx8SvlrqsIMcA8TijFwUnkdABHxW8CCzLwgMy8AjomIc2cavCQ1yoqlfczv6DxoXdf8TlYu76v5tWdYk63HUo00sg6oGLxHlqT6mlYDIyKOjIiLI+LLEfE3EXHUYQ75IvC6iDimvHwupUmOxv0MB4/+WAA8fbpBS1KjLevp5U1Pew7zyqN8e7u7+KvXP6den0Iyk5psPZZqpJF1QMXgPbIk1dd0P4XkcmAX8E5Kw94+PtXOmbkduBi4LiJuBu7JzDURMRARJwEfAn4uIjZHxE3AC4APzjYJSWqEZT29PO3YJ3BKdwc3XHh6Pf+nZdo12Xos1daynl6efEw3P/eUJ9S7DqgYvEeWpDqa7hwYvZn55vL3342IjYc7IDOvAK44ZF3/hMXXTvPakqSDzagmW48lqWa8R5akOpruCIwFEbEIICKOpTScTZLUGNZkSSoG67Ek1dF0R2C8H7g5Ir4H9FEaJidJagxrsiQVg/VYkupoWg2MzNwQES+kNInQneOfXS1Jqj9rsiQVg/VYkuprygZGRPRn5kBE/OKE1cdHBJn55RrHJkmawJosScVgPZakxjjcCIxfAAaANx2yPgGLs6S2tmnnEFt37+JAjnHaJRtZubyv1p9AYE2WpGKwHktSA0zZwMjM95T/e059wpGk5rBp5xBXbd3CgRwDYGh4hFVrtwDUrIlhTZakYrAeS1JjTOtTSCLiwoh4UkS8MiJuiQg/j1pSW1u/bZD9Y6MHrRvZP8rqDYM1v7Y1WZKKwXosSfU13Y9RXZ6ZdwG/Bvwc8JLahSRJxffAvpGK6+8erry+yqzJklQM1mNJqqPpNjAWRsQ7gP8EFgL7axeSJBXfooVdFdcv6a68vsqsyZJUDNZjSaqj6TYw3gYE8BHgeODPaxaRJDWBFUv7mN/RedC6rvmdrFzeV4/LW5MlqRisx5JUR4f7FBIAMvM7wHfKi0PlL0lqW8t6ShN1XnnnHRzIMXq7u+rxKSSANVmSisJ6LEn1NWUDIyK+lJlnRMR2Sh8LBaUuc2bmkppHJ0kFtqynl5t33sVo7OKrF55e8+tZkyWpGKzHktQYhxuBcRZAZj6xDrFIkqZmTZakYrAeS1IDTDkHRmbuBYiIUyLiFeXvfykinl+P4CRJP2FNlqRisB5LUmNMdxLPTwL3lr//FvD3tQlHkjQN1mRJKgbrsSTV0XQbGJGZ3wXIzLtrGI8k6fCsyZJUDNZjSaqj6TYwHoyI34iIRRHxW8BwLYOSJE3JmixJxWA9lqQ6mm4D41zgDOA64BfKy5LU1jbtHGLr7l3cOTzGaZdsZN3mun16njVZkorBeixJdXS4TyEBIDPvjYhzgCcBd2fm/tqGJUnFtmnnEFdt3cKBHANgaHiEVWu3AHDmqb01vbY1WZKKwXosSfU1rREYEbEcuAX4V+AFEfHXNY1Kkgpu/bZB9o+NHrRuZP8oqzcM1vza1mRJKgbrsSTV13QfIXkP8HLgvsy8GXhJ7UKSpOJ7YN9IxfV3D1deX2XWZEkqBuuxJNXRdBsY88qfd53jyzWKR5KawqKFXRXXL+muvL7KrMmSVAzWY0mqo+k2ML4QEV8BnhQR/xf4Qg1jkqTCW7G0j/kdnQet65rfycrlffW4vDVZkorBeixJdTTdSTwviojTgVOBOzLzK7UNS5KKbVlPaaLOK++8gwM5Rm93FyuX99V8Ak+wJktSUViPJam+ptXAiIiPZ+bbgI01jkeSmsaynl5u3nkXo7GLr154et2ua02WpGKwHktSfU33EZJNEfHmiDg6IhZExIKaRiVJmoo1WZKKwXosSXU03YmG/qC870Xl5QSeWpOIJEmHY02WpGKwHktSHU13Dozn1DoQSdL0WJMlqRisx5JUX1M+QhIRz4qIayPinyKip15BSVIz2LRziK27d3Hn8BinXbKRdZuHano9a7IkFYP1WJIa43BzYPwf4B+AG4G/rn04ktQcNu0c4qqtWziQYwAMDY+wau2WWjcxrMmSVAzWY0lqgMM1MDoz8yuZeSnwtHoEJEnNYP22QfaPjR60bmT/KKs3DNbystZkSSoG67EkNcDhGhg54fsDtQxEkprJA/tGKq6/e7jy+iqxJktSMViPJakBDjeJ58si4m4ggCdM+D4zc0nNo5Okglq0sKtiE2NJd1ctL2tNlqRisB5LUgNM2cDITD/LWpIqWLG0j6u2bjnoMZKu+Z2sXN5Xs2takyWpGKzHktQY0/oYVUnSwZb19AJw5Z13cCDH6O3uYuXyPs48tbfBkUmSJEmt6XBzYMxaRJwdEbdExG0R8aFDtv16RAxM+BqKiHfWKhZJqoVlPb087dgncEp3BzdceHphmxfWY0kqDmuyJM1eTRoYEXEycBHwauCFwJMi4qzx7Zn52czsz8x+4DXAXcDHaxGLJLUz67EkFYc1WZLmplaPkJwBrMnMBwEi4jLgHGBNhX0vAD6amXsP3RAR5wHnASxevJiBgYFZBbN3L4yNzerQBtrD2NhAo4OoE3NtTe2RazLC2OjorOtTHVSlHpePrUpNLorpvy+0x2u5xFxnZ4ThYQr7O7Fnz57CxlZtTZBroe6Rm00T/Hyrpp1yhfbK11znplYNjOOBHROWtwM9h+4UEYuA1wLLKp0kMy8HLgfo6+vL/v7+WQVzww2wa9esDm2YsbEBOjr6Gx1GXZhra2qXXIOb6ejcxYw/Ec8AACAASURBVGzrUx1UpR5D9WpyUaxfP7392uW1DOY6ezfS3Q39/S+p0vmqa2BgoMg1qqqaINdC3SM3myb4+VZNO+UK7ZWvuc5NrebAuIeDi/FJ5XWHejtwZWb6+dmSms6mnUNs3b2LO4fHOO2SjazbPNTokCqxHktScViTJWkOatXA+CLwuog4prx8LnBthf1+B/jnGsUgSTWzaecQV23dwoEsPYcwNDzCqrVbitjEsB5LUnFYkyVpDmrSwMjM7cDFwHURcTNwT2auKc+mfBJARLwQGM7MHVOdS5KKaP22QfaPjR60bmT/KKs3DDYoosqsx5JUHNZkSZqbWs2BQWZeAVxxyLr+Cd/fSmn2ZUlqOg/sG6m4/u7hyusbyXosScVhTZak2avVIySS1NIWLeyquH5Jd+X1kiRJkubGBoYkzcKKpX3M7+g8aF3X/E5WLu9rUESSJElSa6vZIySS1MqW9fQCcOWdd3Agx+jt7mLl8j7OPLW3wZFJkiRJrckGhiTN0rKeXm7eeRejsYuvXnh6o8ORJEmSWpqPkEiSJEmSpMKzgSFJkiRJkgrPBoYkzdKmnUNs3b2LO4fHOO2SjazbPNTokCTV2aadQ/zooWFu/uEu64AkSTVmA0OSZmHTziGu2rqFAzkGwNDwCKvWbvF/XqQ2Yh2QJKm+bGBI0iys3zbI/rHRg9aN7B9l9YbBBkUkqd6sA5Ik1ZcNDEmahQf2jVRcf/dw5fWSWo91QJKk+rKBIUmzsGhhV8X1S7orr5fUeqwDkiTVlw0MSZqFFUv7mN/RedC6rvmdrFze16CIJNWbdUCSpPqa1+gAJKkZLevpBeDKO+/gQI7R293FyuV9nHlqb4Mjk1Qv43Vg/bZBhveNsMQ6IElSTdnAkKRZWtbTy80772I0dvHVC09vdDiSGmBZTy/LenpZsaLRkUiS1Pp8hESSJEmSJBWeDQxJmqVNO4fYunsXdw6PcdolG1m3eajRIUmSJEktywaGJM3Cpp1DXLV1CwdyDICh4RFWrd1iE0OSJEmqERsYkjQL67cNsn9s9KB1I/tHWb1hsEERSZIkSa3NBoYkzcID+0Yqrr97uPJ6SZIkSXNjA0OSZmHRwq6K65d0V14vSZIkaW5sYEjSLKxY2sf8js6D1nXN72Tl8r4GRSRJkiS1tnmNDkCSmtGynl6gNBfGA/tG6O3uYuXyPs48tbfBkUmSJEmtyQaGJM3Ssp5elvX0cswxA/T39zc6HEmSJKml+QiJJEmSJEkqPBsYkiRJkiSp8GxgSJIkSZKkwrOBIUmSJEmSCs8GhiRJkiRJKjwbGJIkSZIkqfBsYEiSJEmSpMKzgSFJkiRJkgrPBoYkSZIkSSo8GxiSJEmSJKnwbGBIkiRJkqTCs4EhSZIkSZIKr2YNjIg4OyJuiYjbIuJDFbY/JyK+HBEbI+ILEfGUWsUiSe3MeixJxWFNlqTZm1eLk0bEycBFwIuA3cDVEXFWZq4pb+8ELgPekJlDEbEE2FOLWCSpnVmPJak4rMmSNDeRmdU/acTbgZMz80/Ly6cD52Tmb5aXXwy8ExgBTgGuA96bmaOHnOc84DyAxYsX/+w111wzq3j27oUDB2aZTMPsAY5udBB1Yq6tqX1y7ezcw9FHzy7XV77ylbdl5gurHNJjqlWPy/tWpSYXxYMPTnfP9nktm+vsHXdc1U5VdXv2zL5GNZu55FrregzFu0duNr6WW1c75Wuu0zNZTa7JCAzgeGDHhOXtQM+E5aXAacDLgG3Ax4G3AR+beJLMvBy4HKCvry/7+/tnFcwNN8CuXbM6tGHGxgbo6OhvdBh1Ya6tqZ1yPfroAWZbn+qgKvUYqleTi2L9+unt106vZXOdvSL/OgwMFLpGVVUT5Fqoe+Rm0wQ/36ppp1yhvfI117mp1RwY93BwMT6pvG7cMHB9Zv44S0NA1gDLahSLJLUz67EkFYc1WZLmoFYNjC8Cr4uIY8rL5wLXTth+I/DciDixvPwqYHONYpGkdmY9lqTisCZL0hzUpIGRmduBi4HrIuJm4J7MXBMRAxFxUmY+BPwBsCYibgC6KU1YJEmqIuuxJBWHNVmS5qZWc2CQmVcAVxyyrn/C91+j9HyfJKmGrMeSVBzWZEmavVo9QiJJkiRJklQ1NjAkSZIkSVLh2cCQJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeDYwJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MCRJkiRJUuHZwJAkSZIkSYVnA0OSJEmSJBWeDQxJkiRJklR4NjAkSZIkSVLh2cCQJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeDYwJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MCRJkiRJUuHZwJAkSZIkSYVnA0OSJEmSJBWeDQxJkiRJklR4NjAkSZIkSVLh2cCQJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeDYwJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MCRJkiRJUuHZwJAkSZIkSYU3r1YnjoizgfOBTmAgM991yPaBQw65IDNvqVU8ktSurMeSVBzWZEmavZo0MCLiZOAi4EXAbuDqiDgrM9dM2G1hZr6kFteXJJVYjyWpOKzJkjQ3tRqBcQawJjMfBIiIy4BzgDXl5XnAcRFxDXAS8HXgfZk5OvEkEXEecF55cV9EfLtG8RbRCcB9jQ6iTsy1NZnr9JxczUAqqEo9Lu/brjXZ13JrMtfWVOR6DN4jz5Wv5dbVTvma6/RUrMm1amAcD+yYsLwd6JmwfDSlgryKUvf5cuB3gMsmniQzLy9vIyJuzcwX1ijewmmnfM21NZlrYVSlHkP71mRzbU3m2pqaIFfvkefAXFtXO+VrrnNTq0k87+HgYnxSeR0AmTmcme8o/3cMWEtpKJ0kqbqsx5JUHNZkSZqDWjUwvgi8LiKOKS+fC1w7vjEiToqIP42IKK86A/hmjWKRpHZmPZak4rAmS9Ic1KSBkZnbgYuB6yLiZuCezFwTEQMRMd5pPhr4ZkRcDwTlYXBTONz2VtNO+ZprazLXAqhRPWaa+7QKc21N5tqaCp2r98hzZq6tq53yNdc5iMys9jklSZIkSZKqqlaPkEiSJEmSJFWNDQxJkiRJklR4hWtgRMTZEXFLRNwWER+qsP0PI2JTRNweEec3IsZqmka+fxARN0XEjRHx0Ygo3M9sug6X64T9PhkRn65jaFU3jZ/rcyLiyxGxMSK+EBFPaUSc1TBVrhHRGREfLr+Gb4mISyNifqNinauI+LWIuCYitk2yfVqv8WbSTjXZelxxP+txE7EeH7TdetzErMcV97MeN5l2qcl1r8eZWZgv4GRgEDiO0qRFnwXOmrD9NOBGYEH56z+AFzY67hrm+zPAl4HO8vL/A17b6LhrkeuE/X4V+Gfg042OuYY/107gG0BveXkJcGyj465RriuAz01YXgu8odFxzyHfVwAnADtm+m/RjF/tVJOtx9bj8rL1uEm+rMfWY+txc321Uz2eZr4tU5PrXY+L1q08A1iTmQ9mKePLgDMnbP8V4B8z89HMfBT4FKVf6GY1Zb6Z+R1KBXm0vGoeMFL/MKvicD9bIuJEYCXwlw2Ir5oOl+syYBvwgSjNMP4/gb31D7MqDpfrXcC8iOgo/3VkP/DdBsRZFZn59cy8b5LNh32NN6F2qsnW4wmsx03JevwT1mPrcbOwHv9EK9VjaKOaXO96XLQGxvHAjgnL24GeGWxvNofNJzMfiYjuiLgSuD0zv1LPAKtoOj+7jwHnA4/UK6gaOVyuSyn9peR9wMuBE4G31Su4Kpsy18zcDHwduKT8NVC+8WhFrVafoL1qsvX4YNbj5mM9/olWqk3jrMcTWI+bUjvVY7Amj6t6bSpaA+MeDk5o/POwp7u92Rw2n4h4NqWhNh/OzL+oY2zVNmWuEfF24HuZeVO9A6uBw/1ch4HrM/PH5U7kGkpd52Z0uJ/rbwELMvOCzLwAOCYizq1zjPXSavUJ2qsmW4/LrMfW4xbQSrVpnPV4AutxU2qnegzW5HFVr01Fa2B8EXhdRBxTXj4XuHbC9muB34qI+RHRCbwF+HydY6ymKfONiMXA3wFnZ+bNDYivmg73s10OPC8i1gGXA6dHxAfrHGO1HC7XG4HnlocEArwK2FzH+KrpcLn+DKWhneMWAE+vU2z1drh/i2bUTjXZevwT1uPmZD3+Ceux9bhZWI9/opXqMViTx1W/Hs9lAo1afAH/g9KL9Wbgg+V1A8BJ5e/PB74JbALe1eh4a5kv8PvA1vLy+Nd5jY65Vj/bCfs9mSaepGg6uQKvpDTB1g3AJ4H5jY65FrlS6rh+vrz9JkoTFB3f6JirkPOOCd9fDTx/sn+LZv9qp5psPbYeW4+b78t6bD22HjfPVzvV48Pl24o1uV71OMonlSRJkiRJKqyiPUIiSZIkSZL0ODYwJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MNTSIuLRiBiIiK9HxL9HxClVPPeO8n/7I+Lqap1XklqVNVmSisF6rGY1r9EBSDW2KzP7ASLi1cBq4HUNjUiS2pc1WZKKwXqspuQIDLWTpwLbACKiLyI2lrvOV0bEwvL6cyJiU/nrfeV1T4+I6yPihvK+/t5I0txZkyWpGKzHahq+yNTqnlAurDuBnwJWltd/EnhfZr4CuBl4R0Q8A3gn8PLMXAZsj4gFwHHAH2TmacAjwKl1z0KSWoM1WZKKwXqspuQjJGp1uzLz5yPimcA1wN8D9wDPBt4fEQALgFuA5wLXZeYIQGZeBhARxwIrI6ILeDLwT/VOQpJahDVZkorBeqym5AgMtYXM/D5wKfCB8qotwNvLz/79MvBp4A7g5yPiSICIeFO5MP9v4E/L+/4HEHUNXpJajDVZkorBeqxmYwND7eRTwKsi4mnA7wAfiYivA2sBMvM/gQ8D10fETcCLgIeAy4B/iog1wI8oDbOTJM2NNVmSisF6rKYRmdnoGCRJkiRJkqbkCAxJkiRJklR4NjAkSZIkSVLh2cCQJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeDYwJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MCRJkiRJUuHZwJAkSZIkSYU3r9EBSI0QEUuABZn5o2ruK0mau4j4aeDezLxvBscsAvZk5v7aRSZJzS0iAjgJ6AEWT/jvLZl54xzP/VLgV4APAqPAIuBjmXnGnIKWJnAEhtrVbwD/owb7SpIOERFHRMRTIuJV5Rvcw/krYNkML/NR4A9mHp0ktZUu4PPAe4DlwDOA9wM/VWnniFgWEddExLcj4ocRcV1E/EVEHFdh9zcC3wLOA15Xm/DV7mxgqG2Ub6B/pvxXumcCgxFxYkQ8ay77SpImFxGvB24GPgy8HfjkNA57AXDLhHN8IiJOnOIazwdeClw6t2glqbVl5sOZuSwzfw34O+A1wK9n5jWH7hsRv1Pe568oNYk/A7wS+G/guvJ98vi+HcAZwPoK5zkhIt4SEafUIie1Fx8hUTvpAd4LPInSzfFpwDDwQ0qjLGa7ryRpEpm5FlgLEBFvBe4qf78AOHb8MZGI+GVKTYinA0uAgYi4HfiflBogn4uI0yd5RORi4N2ZOVLrfCSpFUTEc4HLKI2W2BYRLwaeCnwnM78VEcuA3wVenpkPl+v3f2TmKPCJiOgF3g28q3zKXwceLe8L8LTy188DXwDup/SoygfrlqRaUmRmo2OQ6iYiOjJzLCK2AU+b6lnpmewrSTq8iFgHXJKZN0XEM4GPZOYvlLetAI4EXgTsB/4M+Hvgh5m5OiI+A3w3My8+5JwvAz4GPDczx+qYjiQ1lYh4AaVG8SnAucAPKDUVjgO+BtwBXJuZt0XEVcBnMvML5WNvBM7NzO+Vl58H/N/MfH5ELAS2AI8C/wi8E7gH+DLwgsw8IyJ+GzghM21gaE4cgaG2ERHHABsj4ijgicAd5Q7xx4ABYF5m3jbTfSVJkyvX07cCL6Y0vHh3RDwHuA/YPr5fZq4v7/8nwHmZORoR1wP95V3+F3B7RPxTZt414RJ/SWmExsaIuCMz/7DWOUlSk1pCaXLNf6E0cu1MSiONz8/MPYfs+3zgDwEi4mTguPHmRdlCYEH5+7cCnwNeC9wEXAV8l9I988dqkYjal3NgqG1k5kOZuYzSELcvZeZPl78+DBwFfGg2+0qSKouIZwPXU2pWPAysojRPxVJKz1M/KSJOmrD/8ymNDv1WedWxwIMAmTlE6WZ6ZML+rwBeDuwE3gCcXr7m+PYja5acJDWZzPzXzLw0M78OBKV5id5RoXkBpUc+xifqfAtw5SHbXw2M/zFvPXBR+Ro3ALsm7hgRx1K6f3bov+bMBoba0TGU5rOY6BZKnea57CtJOtjfAm+m1MR4CfDRzLwxM98DDAK3ArdFxNPL+/8xpck+xz2Z8pwZAJl5VWbeP2H7u4FLM3NVZt4L3E5pDiPKk8V9rSZZSVLzeyZwV04+n8A64B3l+vxWJoykKM9ZdD6wGiAz/3uSOYiS0oiPVZTeC24vT/YpzZqPkKgd3cfjPyrqWEqd5rnsK0k62BMo/ZXvM8AFmbkPHpvvoiMzz4+ID2bmjoj4WUoTJp8XEb9IaejxaykNcX6c8oRzzyjvM2478LyI2ELpr4U+ay1JlX0TeHZE/D6lR0q2HzKP0P8B1gDXAedQevxvJaVHAZ8KvD4z75jqApn5Y0qfcgJARHya0hxH11UxD7UZO2BqR3cC3RFxTkQcHRFPAj4CfHyO+0qSDvZ+SkOLPzdhjotfpTTK4q0AmbmjvO/PU5q4cxT4I0oTwP1HZv5gknP/MvDeQ/7q9yng9ymNxPhEpY8FlCSVHpcGXgUsA24AHp74cdWZ+WhmrsjMJ2bmlzLzUUqjkj8IPD0zpxrhNgY8uTy5JxHRUT73CymNvpNmzU8hUVsqNyL+AngesJvSxEMfqzSD/Uz2lSRNLiLmUZpD6B+maExIkppQRHw7M59dnovobylNhL+QUmN6GPhyZr63kTGq+dnAkCRJkiRJhecjJJIkSZIkqfBsYEiSJEmSpMKzgSFJkiRJkgqvaT5Gtbu7O0855ZRGh1E3e/fu5aijjmp0GHVhrq3JXKfntttuuy8zF1c5pJprp5rsa7k1mWtrsh63Nl/Lraud8jXX6ZmsJjdNA+PEE0/k1ltvbXQYdTMwMEB/f3+jw6gLc21N5jo9EfHj6kZTH+1Uk30ttyZzbU3W49bma7l1tVO+5jo9k9VkHyGRJEmSJEmFZwNDkiRJkiQVng0MSZIkSZJUeDYwJEmSJElS4dnAkCRJkiRJhWcDQ5IkSZIkFZ4NDEmSJEmSVHg2MCRJkiRJUuHNq8VJI+LXgLOBF2fm0grbzwbOBzqBgcx8Vy3imKl1m4dYvWGQu4dHWNLdxcrlfZx5aq/X8lotea1WzEmVNWtNbrTx1+3Q8Ai9N230dStpzpq1HrfqPYP3J1LzqUkDA7gX+D3g24duiIiTgYuAFwG7gasj4qzMXFOjWKZl3eYhVq3dwsj+UQCGhkdYtXYLQNULmdfyWo2+VivmpCk1XU1uNF+3kmqk6epxq94zWOel5lSTBkZmfh0gIiptPgNYk5kPlve5DDgHaGhxXr1h8LECNm5k/ygXfO4OrrplW1WvtXnbMI+Ojk15reHhES4dvLEu16qW2V5rNrk2Q16VHC7XeuVVj+uM5zrZtVZvGPQGoU6asSY32mTvCb5uJc1FM9bjIt0jV+v+eKprWeelYqvVCIypHA/smLC8HeiptGNEnAecB7B48WIGBgZqFtTQ8EjF9Y+OjnH//cNVvdahxbLytUarct3pXas6Zn+tmefaHHlVMnWu9cqrPtcp5TrZtYaGR2r6O11Pe/bsaeZcClmTG22y94RWet1W0uSv5Rkx19bU5LkWsh4X6x65OvfHU12rKHW+yV/LM9ZO+Zrr3DSigXEP8JQJyyeV1z1OZl4OXA7Q19eX/f39NQuq96aNFQv0ooVd/NFzTq/qtd5760Ye2Df1tcbGBujo6K/LtapltteaTa7NkFclh8u1XnnV4zrjuU52rd7uLmr5O11PAwMDzZxLIWtyo032ntBKr9tKmvy1PCPm2pqaPNdC1uMi3SNX6/54qmsVpc43+Wt5xtopX3Odm0Z8CskXgddFxDHl5XOBaxsQx0FWLu+ja37nQevmd3SyYmlf1a+1Ymkf8zu8ltdq3LUanVPX/E5WLq/+tTQrhazJjVbpPcHXraQaK2Q9bqd7ZOu8VHx1G4EREVcDl2Tm7RFxMXBdRDwKXN/oyYngJ5P1jM84v2hhFyuW9rGsp/rPwI2fc/22QR7Y57W8Vv2v1YicrrzzDg7kGL3O8l0IRa/JjXboe4KvW0m1UvR63A73yMP7/BQSqVnUtIGRmSdN+P6NE76/AriilteejTNP7eXMU3tZv77211rW01uTYuy1vFbRrjN+rW/cs43jj4fPvv0ldbmmHq/ZanKjjb8ntNNQT0n10Wz1uNXvkVesqMvlJFVBIx4hkSRJkiRJmhEbGJIkSZIkqfBsYEiSJEmSpMKzgSFJkiRJkgrPBoYkSZIkSSo8GxiSJEmSJKnwbGBIkiRJkqTCs4EhSZIkSZIKzwaGJEmSJEkqPBsYkiRJkiSp8GxgSJIkSZKkwrOBIUmSJEmSCs8GhiRJkiRJKjwbGJIkSZIkqfBsYEiSJEmSpMKzgSFJkiRJkgrPBoYkSZIkSSo8GxiSJEmSJKnwbGBIkiRJkqTCs4EhSZIkSZIKzwaGJEmSJEkqPBsYkiRJkiSp8GxgSJIkSZKkwrOBIUmSJEmSCs8GhiRJkiRJKjwbGJIkSZIkqfBsYEiSJEmSpMKzgSFJkiRJkgrPBoYkSZL0/9u7/2BL6/o+4O/P8mu0IBqySMBKMNptajSagqnR6DXRQpJqYVDatBkzMrqOk9qxE6WNnUytWCZppI1JE8I6GmsGq6TrhFBpow25QqywBEljNTXTTBJGXDCOBV2Lit5v/7hnw2Vd7j279zzP+d5zXq+ZM9znB+d8Ppxz3vPMh+d5DgDdM8AAAAAAumeAAQAAAHTPAAMAAADongEGAAAA0D0DDAAAAKB7BhgAAABA9wYbYFTVZVV1oKrurKqrj7L9yqq6rao+UVWvGaoOYP7u+Pw9+fMv35/b/+yLed7P3ZzfuuueeZe0VOQxQD9kMsDxG2SAUVXnJrkyyUuSnJ/kSVV16YbtL07yrCQ/kOT5SfZW1VOHqAWYrzs+f0/+059+Mt9oa0mSe+5/MD/zwU8aYoxEHgP0QyYDbM9QZ2BclGR/a+2B1lpLcm2Sizdsf3aS322trbXW/l+S/5r1IAcWzI13fyYPrX3zEesefOib+YXf+cycKlo68higHzIZYBtOHOh5z0hy74blg0nO3LB8V5J/XlXXJHlskh9L8oEjn6Sq9ibZmyS7d+/O6urqQOU+0traKC+zhUNZW1uddxEj0etiWu/1/37twaNuvef+B0f7Tg/t0KFDPfcykzxO5pfJ89b5+ztTel1Meu2KY+RtGeY4qsePzA74LM/UMvWr1+0ZaoBxX5LzNiyfNVmXJGmt/feq+t4kvzdZfyDJ3Uc+SWttX5J9SbJnz562srIyULmPdOONo7zMptbWVrNr18q8yxiFXhfT4V6fcMrNRx1inPP4x2Ss7/TQVldXe+5lJnk82XcumTxvnb+/M6XXxaTXrjhG3oahjqN6/MjsgM/yTC1Tv3rdnqEuIbkpySVVddpk+fIkNxzeWFWPS3JDa+0Hkvxkkmdm/RQ5YMG89Ml7ctKuEx6x7jEnnZA3XbhnThUtHXkM0A+ZDLANg5yB0Vo7WFVXJbmlqr6e5NbW2v6qWk3yDye7/fuqekKSbyT5l621B4aoBZivC848J8n6vTDu/9qDOfvxj8mbLtyTi599zpwrWw7yGKAfMhlge4a6hCStteuSXHfEupUNiy8d6rWBvlxw5jm54Mxz8lLf+rmQxwD9kMkAx2+oS0gAAAAAZsYAAwAAAOieAQYAAADQPQMMAAAAoHsGGAAAAED3DDAAAACA7hlgAAAAAN0zwAAAAAC6Z4ABAAAAdM8AAwAAAOieAQYAAADQPQMMAAAAoHsGGAAAAED3DDAAAACA7hlgAAAAAN0zwAAAAAC6Z4ABAAAAdM8AAwAAAOieAQYAAADQPQMMAAAAoHsGGAAAAED3DDAAAACA7hlgAAAAAN0zwAAAAAC6Z4ABAAAAdM8AAwAAAOieAQYAAADQPQMMAAAAoHsGGAAAAED3DDAAAACA7hlgAAAAAN0zwAAAAAC6Z4ABAAAAdM8AAwAAAOjeidPsVFVnJ3lFktMPr2utvXWoogB4dDIZoA/yGGBc056BcUOS05Lcs+Gxqaq6rKoOVNWdVXX1EdtOqKp3VNVtk32uqaqTjrl6gOV0TJksjwEG4xgZYERTnYGR5EuttbdN+6RVdW6SK5M8J8mXkry/qi5tre2f7PKjSc5prf2dyf4fTHJxkt+cunKA5TV1JstjgEE5RgYY0bRnYKxW1cuq6uTDjy32vyjJ/tbaA621luTarIfvYZ9NcmJV7aqqXUkeSvLpY64eYDkdSybLY4DhOEYGGFGtZ+cWO1X9cZJTktRkVWutPWWT/d+c5FBr7Zcmy9+d5Bdbaxdu2OefJfmOyeKftdauOcrz7E2yN0l27979t6+//vqpmtquBx4Y5WW2cCjJqfMuYiR6XUzf2uvppx99z53u0KFDOfXU43tfX/SiF93ZWjv/WP6dY8nkWeXxZL+5ZPK8bef93Wn0upj0Op2h83iyv2PkbRnmOKrH45Nl+t4my9WvXqfzaJk81SUkrbXvPsbXuy/JeRuWz5qsS5JU1SuTnNxau2KyfEVVXd5aYMUvpAAAEldJREFUe/cRr7svyb4k2bNnT1tZWTnGMo7PjTeO8jKbWltbza5dK/MuYxR6XUxH63Wkr/DoVldXM1Y+JcecyTPJ48nrziWT523s93ee9LqY9Docx8jjGuo4qsevxzJ9b5Pl6lev2zPVJSRV9diquqqqPlxV/7aq/toW/8pNSS6pqtMmy5dn/SZHhz09jxyenJzkadMWDbDMjjGT5THAQBwjA4xr2ntg7EvyxSRvSHJvkndutnNr7WCSq5LcUlW3J7mvtba/qlar6qwkVyf5/qq6q6puS/J9Sd5+vE0ALJmpM1keAwzKMTLAiKb9FZJzWms/Mfn701V181b/QmvtuiTXHbFuZcPiy6Z8bQAe6ZgyWR4DDMYxMsCIpj0D4+SqekKSVNXjsn46GwDzIZMB+iCPAUY07RkYb01y++ROy3uyfpocAPMhkwH6II8BRjTtr5D8TlWdn/WbCP2f1trcf0QJYFnJZIA+yGOAcW06wKiqldbaalX93Q2rz6iqtNY+PHBtAGwgkwH6II8B5mOrMzB+OMlqkh8/Yn1LIpwBxiWTAfogjwHmYNMBRmvtZyf/fNU45QDwaGQyQB/kMcB8TPUrJFX1L6rqSVX1oqo6UFV+jxpgTmQyQB/kMcC4pv0Z1Qtba59N8vIk35/kucOVBMAWZDJAH+QxwIimHWCcUlWvS/InSU5J8tBwJQGwBZkM0Ad5DDCiaQcYr0lSSX4lyRlJ/tVgFQGwFZkM0Ad5DDCirX6FJEnSWvtUkk9NFu+ZPACYA5kM0Ad5DDCuTQcYVfXfWmsXVdXBrP8sVLI+ZW6ttbMHrw6AvyKTAfogjwHmY6szMC5Nktbad4xQCwCbk8kAfZDHAHOw6T0wWmtfSZKqempVvXDy949U1bPGKA6Ah8lkgD7IY4D5mPYmnu9K8peTv/9nkl8ephwApiCTAfogjwFGNO0Ao1prn06S1trnBqwHgK3JZIA+yGOAEU07wHigqv5RVT2hql6Z5P4hiwJgUzIZoA/yGGBE0w4wLk9yUZJbkvzwZBmA+ZDJAH2QxwAj2upXSJIkrbW/rKpXJXlSks+11h4atiwAHo1MBuiDPAYY11RnYFTVhUkOJPkvSb6vqn5+0KoAeFQyGaAP8hhgXNNeQvKzSV6Q5AuttduTPHe4kgDYgkwG6IM8BhjRtAOMEye/d90OLw9UDwBbk8kAfZDHACOadoDxoar6SJInVdV/TPKhAWsCYHMyGaAP8hhgRNPexPPKqvqhJM9O8kettY8MWxYAj0YmA/RBHgOMa6oBRlW9s7X2miQ3D1wPAFuQyQB9kMcA45r2EpI7quonqurUqjq5qk4etCoANiOTAfogjwFGNO2Nhl4/2ffKyXJL8pRBKgJgKzIZoA/yGGBE094D4xlDFwLAdGQyQB/kMcC4Nr2EpKr+VlXdUFXvraozxyoKgG8lkwH6II8B5mOre2D8UpL/kOTjSX5++HIA2IRMBuiDPAaYg60GGCe01j7SWrsmyXeNURAAj0omA/RBHgPMwVYDjLbh728MWQgAW5LJAH2QxwBzsNVNPJ9fVZ9LUkm+bcPfrbV29uDVAbCRTAbogzwGmINNBxitNb9lDdAJmQzQB3kMMB9bXUICAAAAMHeDDTCq6rKqOlBVd1bV1Uds+wdVtbrhcU9VvWGoWgCWmTwG6IdMBjh+gwwwqurcJFcmeUmS85M8qaouPby9tfaB1tpKa20lyY8l+WySdw5RC8Ayk8cA/ZDJANsz1BkYFyXZ31p7oLXWklyb5OJH2feKJL/aWvvKQLUALDN5DNAPmQywDVv9CsnxOiPJvRuWDyY588idquoJSV6W5IKjPUlV7U2yN0l2796d1dXVmRd6NGtro7zMFg5lbW113kWMRK+L6Vt7HekrPLpDhw6Nlk/HYSZ5PNlnLpk8b52/vzOl18Wk1644Rt6WYY6jevzI7IDP8kwtU7963Z6hBhj3JTlvw/JZk3VHem2S97XWjvr72a21fUn2JcmePXvaysrKjMs8uhtvHOVlNrW2tppdu1bmXcYo9LqYjtbrSF/h0a2urmasfDoOM8njZH6ZPG+dv78zpdfFpNeuOEbehqGOo3r8yOyAz/JMLVO/et2eoS4huSnJJVV12mT58iQ3HGW/Vyf5jYFqAEAeA/REJgNswyADjNbawSRXJbmlqm5Pcl9rbf/kbspnJUlVnZ/k/tbavZs9FwDHTx4D9EMmA2zPUJeQpLV2XZLrjli3suHvP8j63ZcBGJA8BuiHTAY4fkNdQgIAAAAwMwYYAAAAQPcMMAAAAIDuGWAAAAAA3TPAAAAAALpngAEAAAB0zwADAAAA6J4BBgAAANA9AwwAAACgewYYAAAAQPcMMAAAAIDuGWAAAAAA3TPAAAAAALpngAEAAAB0zwADAAAA6J4BBgAAANA9AwwAAACgewYYAAAAQPcMMAAAAIDuGWAAAAAA3TPAAAAAALpngAEAAAB0zwADAAAA6J4BBgAAANA9AwwAAACgewYYAAAAQPcMMAAAAIDuGWAAAAAA3TPAAAAAALpngAEAAAB0zwADAAAA6J4BBgAAANA9AwwAAACgewYYAAAAQPcMMAAAAIDuDTbAqKrLqupAVd1ZVVcfZfszqurDVXVzVX2oqs4bqhaAZSaPAfohkwGO34lDPGlVnZvkyiTPSfKlJO+vqktba/sn209Icm2SV7TW7qmqs5McGqIWgGUmjwH6IZMBtmeoMzAuSrK/tfZAa61lPYgv3rD9giR3J3lbVd2a5KeSfGWgWgCWmTwG6IdMBtiGQc7ASHJGkns3LB9McuaG5ScneV6S52c9pN+Z5DVJfm3jk1TV3iR7k2T37t1ZXV0dqNxHWlsb5WW2cChra6vzLmIkel1M39rrSF/h0R06dGi0fDoOM8njZH6ZPG+dv78zpdfFpNeuOEbelmGOo3r8yOyAz/JMLVO/et2eoQYY9yXZeL3eWZN1h92f5NbW2l8kSVXtT/LyHBHOrbV9SfYlyZ49e9rKyspA5T7SjTeO8jKbWltbza5dK/MuYxR6XUxH63Wkr/DoVldXM1Y+HYeZ5HEyv0yet87f35nS62LSa1ccI2/DUMdRPX5kdsBneaaWqV+9bs9Ql5DclOSSqjptsnx5khs2bP94kmdW1RMnyy9OctdAtQAsM3kM0A+ZDLANgwwwWmsHk1yV5Jaquj3Jfa21/VW1WlVntda+nOT1SfZX1ceSPD7r1wACMEPyGKAfMhlge4a6hCStteuSXHfEupUNf/9e1q/vA2BA8higHzIZ4PgNdQkJAAAAwMwYYAAAAADdM8AAAAAAumeAAQAAAHTPAAMAAADongEGAAAA0D0DDAAAAKB7BhgAAABA9wwwAAAAgO4ZYAAAAADdM8AAAAAAumeAAQAAAHTPAAMAAADongEGAAAA0D0DDAAAAKB7BhgAAABA9wwwAAAAgO4ZYAAAAADdM8AAAAAAumeAAQAAAHTPAAMAAADongEGAAAA0D0DDAAAAKB7BhgAAABA9wwwAAAAgO4ZYAAAAADdM8AAAAAAumeAAQAAAHTPAAMAAADongEGAAAA0D0DDAAAAKB7BhgAAABA9wwwAAAAgO4ZYAAAAADdM8AAAAAAunfiUE9cVZcleWOSE5KsttZ++ojtq0f8K1e01g4MVQ/AspLHAP2QyQDHb5ABRlWdm+TKJM9J8qUk76+qS1tr+zfsdkpr7blDvD4A6+QxQD9kMsD2DHUJyUVJ9rfWHmittSTXJrn48MaqOjHJ6VV1fVXdUlVXVtUJA9UCsMzkMUA/ZDLANgx1CckZSe7dsHwwyZkblk9N8tEkP5P16fO+JK/Oeoj/laram2TvZPFrVfW/Bqq3R9+e5AvzLmIkel1Mep3OubMs5ChmksfJUmeyz/Ji0uti6jmPE8fI2+WzvLiWqV+9TueomTzUAOO+JOdtWD5rsi5J0lq7P8nrDi9X1QeTXJojwrm1ti/rwZ2q+oPW2vkD1dudZepXr4tJr92YSR5P9l3KTNbrYtLrYtoBvTpG3ga9Lq5l6lev2zPUJSQ3Jbmkqk6bLF+e5IbDG6vqrKp6c1XVZNVFST4xUC0Ay0weA/RDJgNswyADjNbawSRXJbmlqm5Pcl9rbX9VrVbV4UnzqUk+UVW3JqlMpsgAzI48BuiHTAbYnsF+RrW1dl2S645Yt7Jh8c2Tx7SWLbyXqV+9Lia9dmKAPE4673nG9LqY9LqYuu/VMfK26HVxLVO/et2GWr8BMgAAAEC/hroHBgAAAMDMGGAAAAAA3etugFFVl1XVgaq6s6quPsr2f1pVd1TVH1bVG+dR4yxN0e/rq+q2qvp4Vf1qVXX3nk1rq1437PeuqnrPiKXN3BTv6zOq6sNVdXNVfaiqzjva8+wEm/VaVSdU1Tsmn+EDVXVNVZ00r1q3q6peXlXXV9Xdj7J9qs/4TrJMmSyPj7qfPN5B5PEjtsvjHUweH3U/ebzDLEsmj57HrbVuHknOTfKZJKdn/a7LH0hy6Ybtz0vy8SQnTx6/n+T8edc9YL9PT/LhJCdMln8zycvmXfcQvW7Y7+8n+Y0k75l3zQO+ryck+R9Jzpksn53kcfOue6BeX5rkP29Y/mCSV8y77m30+8Ik357k3mP9b7ETH8uUyfJYHk+W5fEOechjeSyPd9ZjmfJ4yn4XJpPHzuPeppUXJdnfWnugrXd8bZKLN2z/e0l+vbX29dba15O8O+tf6J1q035ba5/KeiB/c7LqxCQPjl/mTGz13qaqnpjkTUn+zRzqm6Wter0gyd1J3lbrP5H2U0m+Mn6ZM7FVr59NcmJV7Zr835GHknx6DnXORGvto621LzzK5i0/4zvQMmWyPN5AHu9I8vhh8lge7xTy+GGLlMfJEmXy2Hnc2wDjjCT3blg+mOTMY9i+02zZT2vtq1X1+Kp6X5I/bK19ZMwCZ2ia9+7XkrwxyVfHKmogW/X65Kz/n5K3JHlBkicmec1Yxc3Ypr221u5K8tEkPzd5rE4OPBbRouVTslyZLI8fSR7vPPL4YYuUTYfJ4w3k8Y60THmcyOTDZp5NvQ0w7ssjGzprsm7a7TvNlv1U1fdk/VSbd7TW/vWItc3apr1W1WuT/HFr7baxCxvAVu/r/Uluba39xWQSuT/rU+edaKv39ZVJTm6tXdFauyLJaVV1+cg1jmXR8ilZrkyWxxPyWB4vgEXKpsPk8QbyeEdapjxOZPJhM8+m3gYYNyW5pKpOmyxfnuSGDdtvSPLKqjqpqk5I8pNJfnvkGmdp036raneSX0xyWWvt9jnUN0tbvbcXJvneqvqtJPuS/FBVvX3kGmdlq14/nuSZk1MCk+TFSe4asb5Z2qrXp2f91M7DTk7ytJFqG9tW/y12omXKZHn8MHm8M8njh8ljebxTyOOHLVIeJzL5sNnn8XZuoDHEI8k/zvqH9fYkb5+sW01y1uTvNyb5RJI7kvz0vOsdst8k/yTJn06WDz/2zrvmod7bDft9Z3bwTYqm6TXJi7J+g62PJXlXkpPmXfMQvWZ94vrbk+23Zf0GRWfMu+YZ9Hzvhr/fn+RZj/bfYqc/limT5bE8lsc77yGP5bE83jmPZcrjrfpdxEweK49r8qQAAAAA3ertEhIAAACAb2GAAQAAAHTPAAMAAADongEGAAAA0D0DDAAAAKB7BhgstKr6elWtVtVHq+p3q+qpM3zueyf/XKmq98/qeQEWlUwG6IM8Zqc6cd4FwMC+2FpbSZKqekmSX0hyyVwrAlheMhmgD/KYHckZGCyTpyS5O0mqak9V3TyZOr+vqk6ZrH9VVd0xebxlsu5pVXVrVX1ssq/vDcD2yWSAPshjdgwfMhbdt02C9fNJ/nqSN03WvyvJW1prL0xye5LXVdXfSPKGJC9orV2Q5GBVnZzk9CSvb609L8lXkzx79C4AFoNMBuiDPGZHcgkJi+6LrbUfrKq/meT6JL+c5L4k35PkrVWVJCcnOZDkmUluaa09mCSttWuTpKoel+RNVfWYJN+Z5L1jNwGwIGQyQB/kMTuSMzBYCq21/53kmiRvm6z6ZJLXTq79+9Ek70nyR0l+sKoemyRV9eOTYP53Sd482ff3k9SoxQMsGJkM0Ad5zE5jgMEyeXeSF1fVdyV5dZJfqaqPJvlgkrTW/iTJO5LcWlW3JXlOki8nuTbJe6tqf5I/z/ppdgBsj0wG6IM8Zseo1tq8awAAAADYlDMwAAAAgO4ZYAAAAADdM8AAAAAAumeAAQAAAHTPAAMAAADongEGAAAA0D0DDAAAAKB7/x/Tju4Iw35WnQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x576 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# プロット\n",
    "plt.figure(figsize=(15, 8))\n",
    "\n",
    "for i, _cls in enumerate(df_dict.keys()):\n",
    "    df = df_dict[_cls]\n",
    "    \n",
    "    ax = plt.subplot(2, 3, i + 1)\n",
    "    ax.plot(df['rec'], df['pre'], '-o')\n",
    "    ax.fill_between(df['rec'], df['pre'], facecolor='b', alpha=0.3)\n",
    "    ax.set_xlim([0, 1.05])\n",
    "    ax.set_ylim([0.5, 1.03])\n",
    "    ax.grid(True)\n",
    "    ax.set_title(CAT_NAME_JP[_cls])\n",
    "    ax.set_xlabel('Recall')\n",
    "    ax.set_ylabel('Precision')\n",
    "    \n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "シャミ子がすごいです（笑）。次いで、モモ、ごせん像という感じです。これは、データ数の順番ですよねぇ・・・\n",
    "\n",
    "> ```\n",
    "[04/13 23:04:09 d2.data.build]: Removed 0 images with no usable annotations. 92 images left.\n",
    "[04/13 23:04:09 d2.data.build]: Distribution of instances among all 6 categories:\n",
    "|  category  | #instances   |  category  | #instances   |  category  | #instances   |\n",
    "|:----------:|:-------------|:----------:|:-------------|:----------:|:-------------|\n",
    "|  Shamiko   | 70           |  Gosenzo   | 28           |   Lilith   | 12           |\n",
    "|    Momo    | 41           |   Mikan    | 15           |    Mob     | 12           |\n",
    "|            |              |            |              |            |              |\n",
    "|   total    | 178          |            |              |            |              |\n",
    "```\n",
    "\n",
    "### AP, mAP の計算\n",
    "\n",
    "* mAP は期待できませんが、計算してみます。\n",
    "* AP は、上記チャートのハッチ部分の面積となります。方法は色々ありますが、今回はデータ数が少ないので普通に計算しました。\n",
    "* mAP は、全クラスの平均となります。\n",
    "* 今回はしきい値 0.5 です。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_AP(df_dict, cat_names):\n",
    "    \"\"\"\n",
    "    AP を計算する\n",
    "    \"\"\"\n",
    "    columns=['AP', 'iou']\n",
    "    tmp_dict = {key: [] for key in columns}\n",
    "    \n",
    "    for _cls in df_dict.keys():\n",
    "        df = df_dict[_cls]\n",
    "        rec, pre = df['rec'].values, df['pre'].values\n",
    "\n",
    "        # 1つしかない場合は 0\n",
    "        if len(rec) < 2:\n",
    "            tmp_dict['AP'].append(0)\n",
    "            tmp_dict['iou'].append(0)\n",
    "            continue\n",
    "\n",
    "        # 台形として面積を求めます\n",
    "        S = 0\n",
    "        for i in range(len(rec) - 1):\n",
    "            h = rec[i + 1] - rec[i]\n",
    "            p1, p2 = pre[i], pre[i + 1]\n",
    "\n",
    "            S += (p1 + p2) * h / 2\n",
    "\n",
    "        tmp_dict['AP'].append(S)\n",
    "        tmp_dict['iou'].append(df['iou'].mean())\n",
    "    \n",
    "    return pd.DataFrame(tmp_dict, columns=columns, index=(cat_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AP</th>\n",
       "      <th>iou</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>シャミ子</td>\n",
       "      <td>0.930458</td>\n",
       "      <td>0.565528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ごせん像</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.554993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>リリス</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>もも</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.768603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>みかん</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>その他</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.815177</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            AP       iou\n",
       "シャミ子  0.930458  0.565528\n",
       "ごせん像  0.500000  0.554993\n",
       "リリス   0.000000  0.000000\n",
       "もも    0.727273  0.768603\n",
       "みかん   0.000000  0.000000\n",
       "その他   0.181818  0.815177"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = calc_AP(df_dict, cat_names=CAT_NAME_JP)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AP     0.389925\n",
       "iou    0.450717\n",
       "dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "というか、上位以外が足をひっぱているのがよくわかりますね。\n",
    "\n",
    "データ数は正義です・・・"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}