scene-classification/scene-classification-quickstart.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '/home/yons/data/AID/AID'\n",
    "import glob\n",
    "image_path_list = glob.glob(os.path.join(data_dir, '*', '*'))\n",
    "image_path_list.sort()\n",
    "print(len(image_path_list))\n",
    "\n",
    "categories = [d.name for d in os.scandir(data_dir) if d.is_dir()]\n",
    "categories.sort()\n",
    "print(len(categories))\n",
    "class_to_idx = {categories[i]: i for i in range(len(categories))}\n",
    "idx_to_class = {idx: class_ for class_, idx in class_to_idx.items()}\n",
    "print(class_to_idx)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## view numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_categories = []\n",
    "for category in categories:\n",
    "    n_categories.append(len(os.listdir(os.path.join(data_dir, category))))\n",
    "print(n_categories)\n",
    "\n",
    "import pandas as pd\n",
    "cat_df = pd.DataFrame({'categories':categories, 'number':n_categories})\n",
    "cat_df.sort_values('number', ascending=False, inplace=True)\n",
    "cat_df.head()\n",
    "cat_df.tail()\n",
    "\n",
    "_ = plt.figure()\n",
    "cat_df.set_index('categories')['number'].plot.bar(color='r', figsize=(20, 6))\n",
    "_ = plt.xticks(rotation=80)\n",
    "_ = plt.ylabel('Count')\n",
    "_ = plt.title('Images by Category')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## view height and width"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "def multiprocess_run(process, obj_list, notebook=False):\n",
    "    if notebook is True:\n",
    "        from tqdm import tqdm_notebook as tqdm\n",
    "    else:\n",
    "        from tqdm import tqdm\n",
    "    from multiprocessing import Pool\n",
    "    import multiprocessing\n",
    "    import time\n",
    "    import sys\n",
    "    cpus = multiprocessing.cpu_count()\n",
    "    pool = Pool(processes=cpus)\n",
    "    result_list = []\n",
    "    tic = time.time()\n",
    "    for result in tqdm(pool.imap(process, obj_list)):\n",
    "        result_list.append(result)\n",
    "        sys.stdout.flush()\n",
    "    toc = time.time()\n",
    "    print('time waste', toc - tic)\n",
    "    return result_list\n",
    "\n",
    "def process(image_path):\n",
    "    category = os.path.dirname(image_path).split('/')[-1]\n",
    "    try:\n",
    "        h, w, _ = cv2.imread(image_path).shape\n",
    "    except Exception as e:\n",
    "        print(image_path)\n",
    "        return None\n",
    "    return {'category':category, 'height':h, 'width': w}\n",
    "\n",
    "result_list = multiprocess_run(process, image_path_list, notebook=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_category_list = []\n",
    "img_height_list = []\n",
    "img_width_list = []\n",
    "for d in result_list:\n",
    "    if d is not None:\n",
    "        img_category_list.append(d['category'])\n",
    "        img_height_list.append(d['height'])\n",
    "        img_width_list.append(d['width'])\n",
    "    \n",
    "image_df = pd.DataFrame({\n",
    "    'category': img_category_list,\n",
    "    'height': img_height_list,\n",
    "    'width': img_width_list\n",
    "})\n",
    "\n",
    "import seaborn as sns\n",
    "img_dsc = image_df.groupby('category').describe()\n",
    "img_dsc\n",
    "\n",
    "## all of the dataset are 600*600\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## split datasets\n",
    "\n",
    "the aid official baseline: 20% - 86.86 | 50% - 89.53\n",
    "\n",
    "we just conduct experiments using 50% as trainset, 50% as the valset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.shuffle(image_path_list)\n",
    "image_path_list_train = image_path_list[:5000]\n",
    "image_path_list_val = image_path_list[5000:]\n",
    "print('train:', len(image_path_list_train))\n",
    "print('val:', len(image_path_list_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## define datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "from PIL import Image\n",
    "\n",
    "class SceneDataset(Dataset):\n",
    "\n",
    "    def __init__(self, samples, transform, notebook=False):\n",
    "        self.transform = transform\n",
    "        self.samples = samples\n",
    "        self.get_item = self.get_item_from_path\n",
    "        print(len(self.samples))\n",
    "\n",
    "    def get_item_from_path(self, index):\n",
    "        path, target = self.samples[index]\n",
    "        with open(path, 'rb') as f:\n",
    "            img = Image.open(f).convert('RGB')\n",
    "        return img, target\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        sample, target = self.get_item(index)\n",
    "        sample = self.transform(sample)\n",
    "        return sample, target\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.samples)\n",
    "    \n",
    "    \n",
    "# test\n",
    "\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "transform = transforms.Compose([transforms.Resize(224),\n",
    "                             transforms.ToTensor()])\n",
    "batch_size = 128\n",
    "data = {\n",
    "    'train':\n",
    "    SceneDataset([(x, class_to_idx[x.split('/')[-2]]) for x in image_path_list_train], transform=transform),\n",
    "    'val':\n",
    "    SceneDataset([(x, class_to_idx[x.split('/')[-2]]) for x in image_path_list_val], transform=transform),\n",
    "}\n",
    "dataloaders = {\n",
    "    'train': DataLoader(data['train'], batch_size=batch_size, shuffle=False),\n",
    "    'val': DataLoader(data['val'], batch_size=batch_size, shuffle=False),\n",
    "}\n",
    "\n",
    "test_iter = iter(dataloaders['train'])\n",
    "inputs, targets = next(test_iter)\n",
    "inputs.shape, targets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(16, 16))\n",
    "plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)\n",
    "for i in range(64):\n",
    "    ax = plt.subplot(8, 8, i + 1)\n",
    "    image = inputs[i]\n",
    "    image = image.numpy().transpose((1, 2, 0))\n",
    "    image = np.clip(image, 0, 1)\n",
    "    label = idx_to_class[targets[i].item()]\n",
    "    _ = ax.imshow(image)\n",
    "    _ = ax.text(50, 25, str(label), bbox=dict(facecolor='red', alpha=0.5))\n",
    "    _ = ax.axis('off')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cat mean and std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "transform = transforms.Compose([transforms.Resize(224),\n",
    "                             transforms.ToTensor()])\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "dataset = SceneDataset(samples=[(x, class_to_idx[x.split('/')[-2]]) for x in image_path_list_train], transform=transform)\n",
    "\n",
    "loader = DataLoader(\n",
    "    dataset,\n",
    "    batch_size=128,\n",
    "    num_workers=32,\n",
    "    shuffle=False\n",
    ")\n",
    "\n",
    "print(len(loader))                                \n",
    "\n",
    "mean = 0.\n",
    "std = 0.\n",
    "nb_samples = 0.\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "for step, data in tqdm(enumerate(loader)):\n",
    "    inputs, targets = data\n",
    "    batch_samples = inputs.size(0)\n",
    "    inputs = inputs.view(batch_samples, inputs.size(1), -1)\n",
    "    mean += inputs.mean(2).sum(0)\n",
    "    std += inputs.std(2).sum(0)\n",
    "    nb_samples += batch_samples\n",
    "\n",
    "mean /= nb_samples\n",
    "std /= nb_samples\n",
    "mean = mean.numpy()\n",
    "std = std.numpy()\n",
    "\n",
    "print(mean)\n",
    "print(std)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = [0.3989299, 0.41072574, 0.36983183]\n",
    "std = [0.14689957, 0.13350782, 0.12862574]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## define transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "# Image transformations\n",
    "image_transforms = {\n",
    "    # Train uses data augmentation\n",
    "    'train':\n",
    "    transforms.Compose([\n",
    "        transforms.RandomResizedCrop(size=224, scale=(0.8, 1.0)),\n",
    "        transforms.RandomRotation(degrees=15),\n",
    "        transforms.ColorJitter(),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean, std)  # Imagenet standards\n",
    "    ]),\n",
    "    # Validation does not use augmentation\n",
    "    'val':\n",
    "    transforms.Compose([\n",
    "        transforms.Resize(size=256),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean, std)\n",
    "    ])\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import datasets\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "data = {\n",
    "    'train':\n",
    "    SceneDataset([(x, class_to_idx[x.split('/')[-2]]) for x in image_path_list_train], transform=image_transforms['train']),\n",
    "    'val':\n",
    "    SceneDataset([(x, class_to_idx[x.split('/')[-2]]) for x in image_path_list_val], transform=image_transforms['val']),\n",
    "}\n",
    "dataloaders = {\n",
    "    'train': DataLoader(data['train'], batch_size=batch_size, shuffle=False),\n",
    "    'val': DataLoader(data['val'], batch_size=batch_size, shuffle=False),\n",
    "}\n",
    "\n",
    "test_iter = iter(dataloaders['train'])\n",
    "inputs, targets = next(test_iter)\n",
    "inputs.shape, targets.shape\n",
    "\n",
    "plt.figure(figsize=(16, 16))\n",
    "plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)\n",
    "for i in range(64):\n",
    "    ax = plt.subplot(8, 8, i + 1)\n",
    "    image = inputs[i]\n",
    "    image = image.numpy().transpose((1, 2, 0))\n",
    "    image = np.array(std) * image + np.array(mean)\n",
    "    \n",
    "    image = np.clip(image, 0, 1)\n",
    "    label = idx_to_class[targets[i].item()]\n",
    "    _ = ax.imshow(image)\n",
    "    _ = ax.text(50, 25, str(label), bbox=dict(facecolor='red', alpha=0.5))\n",
    "    _ = ax.axis('off')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## define model \n",
    "here we use resnext50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torch.nn as nn\n",
    "model = torchvision.models.resnext50_32x4d(pretrained=True)\n",
    "for i, layer in enumerate(model.children()):\n",
    "    if i < 6:\n",
    "        for param in layer.parameters():\n",
    "            param.requires_grad = False\n",
    "\n",
    "n_inputs = model.fc.in_features\n",
    "model.fc = nn.Sequential(\n",
    "    nn.Linear(n_inputs, 256), nn.ReLU(), nn.Dropout(0.3),\n",
    "    nn.Linear(256, 30))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchsummary import summary\n",
    "summary(model, input_size=(3, 224, 224), batch_size=1, device='cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_val(net, criterion, optimizer, train_loader, val_loader):\n",
    "    from tqdm.autonotebook import tqdm\n",
    "    _ = net.train()\n",
    "    train_loss = 0\n",
    "    train_acc = 0\n",
    "    for step, (inputs, targets) in tqdm(enumerate(train_loader)):\n",
    "        inputs, targets = inputs.cuda(), targets.cuda()\n",
    "        outputs = net(inputs)\n",
    "        optimizer.zero_grad()\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        train_loss += loss.item()\n",
    "        _, pred = torch.max(outputs, dim=1)\n",
    "        correct_tensor = pred.eq(targets.data.view_as(pred))\n",
    "        accuracy = torch.mean(correct_tensor.type(torch.FloatTensor))\n",
    "        train_acc += accuracy.item()\n",
    "    train_loss, train_acc = train_loss / len(train_loader), train_acc / len(train_loader)\n",
    "\n",
    "    _ = net.eval()\n",
    "    with torch.no_grad():\n",
    "        val_loss = 0\n",
    "        val_acc = 0\n",
    "        for step, (inputs, targets) in enumerate(val_loader):\n",
    "            inputs, targets = inputs.cuda(), targets.cuda()\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            val_loss += loss.item()\n",
    "            _, pred = torch.max(outputs, dim=1)\n",
    "            correct_tensor = pred.eq(targets.data.view_as(pred))\n",
    "            accuracy = torch.mean(correct_tensor.type(torch.FloatTensor))\n",
    "            val_acc += accuracy.item()\n",
    "    val_loss, val_acc = val_loss / len(val_loader), val_acc / len(val_loader)\n",
    "    return train_loss, train_acc, val_loss, val_acc\n",
    "\n",
    "def get_lr(optimizer):\n",
    "    for param_group in optimizer.param_groups:\n",
    "        return param_group['lr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "dataloaders = {\n",
    "    'train': DataLoader(data['train'], batch_size=batch_size, shuffle=True),\n",
    "    'val': DataLoader(data['val'], batch_size=batch_size, shuffle=False),\n",
    "}\n",
    "print('train batch:%s' % len(dataloaders['train']))\n",
    "print('val batch:%s' % len(dataloaders['val']))\n",
    "# loss\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "import torch\n",
    "net = model.cuda()\n",
    "net = nn.DataParallel(net)\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=0.0005)\n",
    "scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[2, 8, 15, 20], gamma=0.1)\n",
    "version = '0.1'\n",
    "log_dir = 'AID-quickstart.log'\n",
    "save_dir = os.path.join(log_dir, version)\n",
    "import shutil\n",
    "import os\n",
    "if not os.path.exists(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "else:\n",
    "    shutil.rmtree(save_dir)\n",
    "    os.makedirs(save_dir)\n",
    "best_acc = 0\n",
    "for epoch in range(25):\n",
    "    scheduler.step()\n",
    "    print('\\n Version: %s Epoch: %d | learning rate:%f' % (version, epoch, get_lr(optimizer)))\n",
    "    train_loss, train_acc, val_loss, val_acc = train_val(net, criterion, optimizer, dataloaders['train'], dataloaders['val'])\n",
    "    print(epoch, train_loss, train_acc, val_loss, val_acc)\n",
    "\n",
    "    if val_acc > best_acc:\n",
    "        best_acc = val_acc\n",
    "        save_path = os.path.join(save_dir, 'best_acc.pth')\n",
    "        state = {\n",
    "            'net': net.state_dict(),\n",
    "            'epoch': epoch,\n",
    "            'loss': val_loss,\n",
    "            'acc': val_acc\n",
    "        }\n",
    "        torch.save(state, save_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_path = os.path.join(save_dir, 'best_acc.pth')\n",
    "checkpoint = torch.load(save_path)\n",
    "net.load_state_dict(checkpoint['net'])\n",
    "print(checkpoint['epoch'], checkpoint['loss'], checkpoint['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "_ = net.eval()\n",
    "predict_prob_list_val = []\n",
    "target_list_val = []\n",
    "with torch.no_grad():\n",
    "    val_loss = 0\n",
    "    val_acc = 0\n",
    "    for step, (inputs, targets) in tqdm(enumerate(dataloaders['val'])):\n",
    "        inputs, targets = inputs.cuda(), targets.cuda()\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, targets)\n",
    "        val_loss += loss.item()\n",
    "        _, pred = torch.max(outputs, dim=1)\n",
    "        correct_tensor = pred.eq(targets.data.view_as(pred))\n",
    "        accuracy = torch.mean(correct_tensor.type(torch.FloatTensor))\n",
    "        val_acc += accuracy.item()\n",
    "        # 计算probs备用\n",
    "        probs = torch.exp(F.log_softmax(outputs, dim=1)).cpu().numpy()\n",
    "        predict_prob_list_val.append(probs)\n",
    "        targets = targets.cpu().numpy()\n",
    "        target_list_val.append(targets)\n",
    "val_loss, val_acc = val_loss / len(dataloaders['val']), val_acc / len(dataloaders['val'])\n",
    "print(val_loss, val_acc)\n",
    "\n",
    "predict_prob_npy = np.array(predict_prob_list_val)\n",
    "predict_prob_npy = np.concatenate(predict_prob_npy, 0)\n",
    "target_npy_val = np.array(target_list_val)\n",
    "target_npy_val = np.concatenate(target_npy_val, 0)\n",
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(target_npy_val, np.argmax(predict_prob_npy, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## view top1 and top5 acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "topk = 2\n",
    "tmp = np.equal(np.argsort(predict_prob_npy, axis=1)[:,-topk:][:, ::-1], np.tile(target_npy_val.reshape((-1,1)), (1,topk)))\n",
    "a = np.sum(tmp[:,(0,)], axis=1).reshape((-1,1))\n",
    "b = np.sum(tmp, axis=1).reshape((-1,1))\n",
    "acc_results = np.hstack((a,b)) * 100\n",
    "print('top1 : ', np.sum(a) / predict_prob_npy.shape[0])\n",
    "print('top5 : ', np.sum(b) / predict_prob_npy.shape[0])\n",
    "\n",
    "classes = [idx_to_class[x] for x in target_npy_val]\n",
    "results = pd.DataFrame(acc_results, columns=['top1', 'top5'])\n",
    "results['class'] = classes\n",
    "results = results.groupby(classes).mean().reset_index().rename(columns={'index': 'class'})\n",
    "\n",
    "import seaborn as sns\n",
    "results = results.merge(cat_df, left_on='class', right_on='categories').drop(columns=['categories'])\n",
    "fig, axes = plt.subplots(1,2, figsize=(16,8))\n",
    "ax = axes[0]\n",
    "_ = sns.regplot(\n",
    "    y='top1', x='number', data=results, ax=ax)\n",
    "_ = ax.set_xlabel('images')\n",
    "_ = ax.set_ylabel('Accuracy (%)')\n",
    "_ = ax.set_title('Top 1 Accuracy vs Number of Training Images')\n",
    "_ = ax.set_ylim(-5, 105)\n",
    "\n",
    "ax = axes[1]\n",
    "_ = sns.regplot(\n",
    "    y='top5', x='number', data=results, ax=ax)\n",
    "_ = ax.set_xlabel('images')\n",
    "_ = ax.set_ylabel('Accuracy (%)')\n",
    "_ = ax.set_title('Top 5 Accuracy vs Number of Training Images')\n",
    "_ = ax.set_ylim(-5, 105)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = plt.subplot(211)\n",
    "results.set_index('class')['number'].plot.bar(\n",
    "    color='r', figsize=(20, 6))\n",
    "_ = plt.xticks(rotation=80)\n",
    "_ = plt.ylabel('Count')\n",
    "_ = plt.title('Number Images by Category')\n",
    "_ = plt.subplot(212)\n",
    "\n",
    "results.set_index('class')['top1'].plot.bar(\n",
    "    color='r', figsize=(20, 6))\n",
    "_ = plt.xticks(rotation=80)\n",
    "_ = plt.ylabel('acc')\n",
    "_ = plt.title('Acc by Category')\n",
    "_ = plt.hlines(85, 0, 100, colors = \"b\", linestyles = \"dashed\")\n",
    "_ = plt.hlines(90, 0, 100, colors = \"g\", linestyles = \"dashed\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## plot confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_true, y_pred, classes,\n",
    "                          normalize=False,\n",
    "                          title=None,\n",
    "                          cmap=None, figsize=(20, 20)):\n",
    "    import matplotlib.pyplot as plt\n",
    "    cmap = plt.cm.Blues\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    from sklearn.utils.multiclass import unique_labels\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if not title:\n",
    "        if normalize:\n",
    "            title = 'Normalized confusion matrix'\n",
    "        else:\n",
    "            title = 'Confusion matrix, without normalization'\n",
    "\n",
    "    # Compute confusion matrix\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    # Only use the labels that appear in the data\n",
    "    classes = classes[unique_labels(y_true, y_pred)]\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    fig = plt.figure(figsize=figsize)\n",
    "    ax = fig.add_subplot(111)\n",
    "    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    ax.figure.colorbar(im, ax=ax, fraction=0.046, pad=0.04)\n",
    "\n",
    "    # We want to show all ticks...\n",
    "    ax.set(xticks=np.arange(cm.shape[1]),\n",
    "           yticks=np.arange(cm.shape[0]),\n",
    "           # ... and label them with the respective list entries\n",
    "           xticklabels=classes, yticklabels=classes,\n",
    "           title=title,\n",
    "           ylabel='True label',\n",
    "           xlabel='Predicted label')\n",
    "\n",
    "    # Rotate the tick labels and set their alignment.\n",
    "    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n",
    "             rotation_mode=\"anchor\")\n",
    "\n",
    "    # Loop over data dimensions and create text annotations.\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i in range(cm.shape[0]):\n",
    "        for j in range(cm.shape[1]):\n",
    "            ax.text(j, i, format(cm[i, j], fmt),\n",
    "                    ha=\"center\", va=\"center\",\n",
    "                    color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "    fig.tight_layout()\n",
    "    return ax\n",
    "classes = [d.name for d in os.scandir(data_dir) if d.is_dir()]\n",
    "classes.sort()\n",
    "classes = np.array(classes)\n",
    "plot_confusion_matrix(target_npy_val, np.argmax(predict_prob_npy, 1), classes, normalize=True,\n",
    "                      title='confusion matrix', cmap=plt.cm.Blues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}