Lec4/Lab6_result_report.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[제가 미리 만들어놓은 이 링크](https://colab.research.google.com/github/heartcored98/Standalone-DeepLearning/blob/master/Lec4/Lab6_result_report.ipynb)를 통해 Colab에서 바로 작업하실 수 있습니다!  \n",
    "런타임 유형은 python3, GPU 가속 확인하기!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PD4cIKKvKFCC"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import argparse\n",
    "import numpy as np\n",
    "import time\n",
    "from copy import deepcopy # Add Deepcopy for args"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "29UainWPco7Y"
   },
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 73
    },
    "colab_type": "code",
    "id": "Cu753dPPKGkV",
    "outputId": "43b9be12-27a2-48ac-d9d4-08ea5dab0658"
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainset, valset = torch.utils.data.random_split(trainset, [40000, 10000])\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "partition = {'train': trainset, 'val':valset, 'test':testset}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RxnfFJwBcsAv"
   },
   "source": [
    "## Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_G6bZbbkMWWt"
   },
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, in_dim, out_dim, hid_dim, n_layer, act, dropout, use_bn, use_xavier):\n",
    "        super(MLP, self).__init__()\n",
    "        self.in_dim = in_dim\n",
    "        self.out_dim = out_dim\n",
    "        self.hid_dim = hid_dim\n",
    "        self.n_layer = n_layer\n",
    "        self.act = act\n",
    "        self.dropout = dropout\n",
    "        self.use_bn = use_bn\n",
    "        self.use_xavier = use_xavier\n",
    "        \n",
    "        # ====== Create Linear Layers ====== #\n",
    "        self.fc1 = nn.Linear(self.in_dim, self.hid_dim)\n",
    "        \n",
    "        self.linears = nn.ModuleList()\n",
    "        self.bns = nn.ModuleList()\n",
    "        for i in range(self.n_layer-1):\n",
    "            self.linears.append(nn.Linear(self.hid_dim, self.hid_dim))\n",
    "            if self.use_bn:\n",
    "                self.bns.append(nn.BatchNorm1d(self.hid_dim))\n",
    "                \n",
    "        self.fc2 = nn.Linear(self.hid_dim, self.out_dim)\n",
    "        \n",
    "        # ====== Create Activation Function ====== #\n",
    "        if self.act == 'relu':\n",
    "            self.act = nn.ReLU()\n",
    "        elif self.act == 'tanh':\n",
    "            self.act == nn.Tanh()\n",
    "        elif self.act == 'sigmoid':\n",
    "            self.act = nn.Sigmoid()\n",
    "        else:\n",
    "            raise ValueError('no valid activation function selected!')\n",
    "        \n",
    "        # ====== Create Regularization Layer ======= #\n",
    "        self.dropout = nn.Dropout(self.dropout)\n",
    "        if self.use_xavier:\n",
    "            self.xavier_init()\n",
    "          \n",
    "    def forward(self, x):\n",
    "        x = self.act(self.fc1(x))\n",
    "        for i in range(len(self.linears)):\n",
    "            x = self.act(self.linears[i](x))\n",
    "            x = self.bns[i](x)\n",
    "            x = self.dropout(x)\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "    \n",
    "    def xavier_init(self):\n",
    "        for linear in self.linears:\n",
    "            nn.init.xavier_normal_(linear.weight)\n",
    "            linear.bias.data.fill_(0.01)\n",
    "            \n",
    "net = MLP(3072, 10, 100, 4, 'relu', 0.1, True, True) # Testing Model Construction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "itGsp6jDWs_a"
   },
   "source": [
    "## Train, Validate, Test and Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(net, partition, optimizer, criterion, args):\n",
    "    trainloader = torch.utils.data.DataLoader(partition['train'], \n",
    "                                              batch_size=args.train_batch_size, \n",
    "                                              shuffle=True, num_workers=2)\n",
    "    net.train()\n",
    "\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    train_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        optimizer.zero_grad() # [21.01.05 오류 수정] 매 Epoch 마다 .zero_grad()가 실행되는 것을 매 iteration 마다 실행되도록 수정했습니다. \n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = data\n",
    "        inputs = inputs.view(-1, 3072)\n",
    "        inputs = inputs.cuda()\n",
    "        labels = labels.cuda()\n",
    "        outputs = net(inputs)\n",
    "\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "    train_loss = train_loss / len(trainloader)\n",
    "    train_acc = 100 * correct / total\n",
    "    return net, train_loss, train_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(net, partition, criterion, args):\n",
    "    valloader = torch.utils.data.DataLoader(partition['val'], \n",
    "                                            batch_size=args.test_batch_size, \n",
    "                                            shuffle=False, num_workers=2)\n",
    "    net.eval()\n",
    "\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    val_loss = 0 \n",
    "    with torch.no_grad():\n",
    "        for data in valloader:\n",
    "            images, labels = data\n",
    "            images = images.view(-1, 3072)\n",
    "            images = images.cuda()\n",
    "            labels = labels.cuda()\n",
    "            outputs = net(images)\n",
    "\n",
    "            loss = criterion(outputs, labels)\n",
    "            \n",
    "            val_loss += loss.item()\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "        val_loss = val_loss / len(valloader)\n",
    "        val_acc = 100 * correct / total\n",
    "    return val_loss, val_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(net, partition, args):\n",
    "    testloader = torch.utils.data.DataLoader(partition['test'], \n",
    "                                             batch_size=args.test_batch_size, \n",
    "                                             shuffle=False, num_workers=2)\n",
    "    net.eval()\n",
    "    \n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            images, labels = data\n",
    "            images = images.view(-1, 3072)\n",
    "            images = images.cuda()\n",
    "            labels = labels.cuda()\n",
    "\n",
    "            outputs = net(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "        test_acc = 100 * correct / total\n",
    "    return test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LiOCP6TqWw2V"
   },
   "outputs": [],
   "source": [
    "def experiment(partition, args):\n",
    "  \n",
    "    net = MLP(args.in_dim, args.out_dim, args.hid_dim, args.n_layer, args.act, args.dropout, args.use_bn, args.use_xavier)\n",
    "    net.cuda()\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    if args.optim == 'SGD':\n",
    "        optimizer = optim.RMSprop(net.parameters(), lr=args.lr, weight_decay=args.l2)\n",
    "    elif args.optim == 'RMSprop':\n",
    "        optimizer = optim.RMSprop(net.parameters(), lr=args.lr, weight_decay=args.l2)\n",
    "    elif args.optim == 'Adam':\n",
    "        optimizer = optim.Adam(net.parameters(), lr=args.lr, weight_decay=args.l2)\n",
    "    else:\n",
    "        raise ValueError('In-valid optimizer choice')\n",
    "    \n",
    "    # ===== List for epoch-wise data ====== #\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "    train_accs = []\n",
    "    val_accs = []\n",
    "    # ===================================== #\n",
    "        \n",
    "    for epoch in range(args.epoch):  # loop over the dataset multiple times\n",
    "        ts = time.time()\n",
    "        net, train_loss, train_acc = train(net, partition, optimizer, criterion, args)\n",
    "        val_loss, val_acc = validate(net, partition, criterion, args)\n",
    "        te = time.time()\n",
    "        \n",
    "        # ====== Add Epoch Data ====== #\n",
    "        train_losses.append(train_loss)\n",
    "        val_losses.append(val_loss)\n",
    "        train_accs.append(train_acc)\n",
    "        val_accs.append(val_acc)\n",
    "        # ============================ #\n",
    "        \n",
    "        print('Epoch {}, Acc(train/val): {:2.2f}/{:2.2f}, Loss(train/val) {:2.2f}/{:2.2f}. Took {:2.2f} sec'.format(epoch, train_acc, val_acc, train_loss, val_loss, te-ts))\n",
    "        \n",
    "    test_acc = test(net, partition, args)    \n",
    "    \n",
    "    # ======= Add Result to Dictionary ======= #\n",
    "    result = {}\n",
    "    result['train_losses'] = train_losses\n",
    "    result['val_losses'] = val_losses\n",
    "    result['train_accs'] = train_accs\n",
    "    result['val_accs'] = val_accs\n",
    "    result['train_acc'] = train_acc\n",
    "    result['val_acc'] = val_acc\n",
    "    result['test_acc'] = test_acc\n",
    "    return vars(args), result\n",
    "    # ===================================== #"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Manage Experiment Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hashlib\n",
    "import json\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import pandas as pd\n",
    "\n",
    "def save_exp_result(setting, result):\n",
    "    exp_name = setting['exp_name']\n",
    "    del setting['epoch']\n",
    "    del setting['test_batch_size']\n",
    "\n",
    "    hash_key = hashlib.sha1(str(setting).encode()).hexdigest()[:6]\n",
    "    filename = './results/{}-{}.json'.format(exp_name, hash_key)\n",
    "    result.update(setting)\n",
    "    with open(filename, 'w') as f:\n",
    "        json.dump(result, f)\n",
    "\n",
    "    \n",
    "def load_exp_result(exp_name):\n",
    "    dir_path = './results'\n",
    "    filenames = [f for f in listdir(dir_path) if isfile(join(dir_path, f)) if '.json' in f]\n",
    "    list_result = []\n",
    "    for filename in filenames:\n",
    "        if exp_name in filename:\n",
    "            with open(join(dir_path, filename), 'r') as infile:\n",
    "                results = json.load(infile)\n",
    "                list_result.append(results)\n",
    "    df = pd.DataFrame(list_result) # .drop(columns=[])\n",
    "    return df\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "omgExzmQgU1J"
   },
   "source": [
    "## Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DRoOy_B3Wu7B"
   },
   "outputs": [],
   "source": [
    "# ====== Random Seed Initialization ====== #\n",
    "seed = 123\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "args = parser.parse_args(\"\")\n",
    "args.exp_name = \"exp1_n_layer_hid_dim\"\n",
    "\n",
    "# ====== Model Capacity ====== #\n",
    "args.in_dim = 3072\n",
    "args.out_dim = 10\n",
    "args.hid_dim = 100\n",
    "args.act = 'relu'\n",
    "\n",
    "# ====== Regularization ======= #\n",
    "args.dropout = 0.2\n",
    "args.use_bn = True\n",
    "args.l2 = 0.00001\n",
    "args.use_xavier = True\n",
    "\n",
    "# ====== Optimizer & Training ====== #\n",
    "args.optim = 'RMSprop' #'RMSprop' #SGD, RMSprop, ADAM...\n",
    "args.lr = 0.0015\n",
    "args.epoch = 10\n",
    "\n",
    "args.train_batch_size = 256\n",
    "args.test_batch_size = 1024\n",
    "\n",
    "# ====== Experiment Variable ====== #\n",
    "name_var1 = 'n_layer'\n",
    "name_var2 = 'hid_dim'\n",
    "list_var1 = [1, 2, 3]\n",
    "list_var2 = [500, 300]\n",
    "\n",
    "\n",
    "for var1 in list_var1:\n",
    "    for var2 in list_var2:\n",
    "        setattr(args, name_var1, var1)\n",
    "        setattr(args, name_var2, var2)\n",
    "        print(args)\n",
    "                \n",
    "        setting, result = experiment(partition, deepcopy(args))\n",
    "        save_exp_result(setting, result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns \n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df = load_exp_result('exp1')\n",
    "\n",
    "fig, ax = plt.subplots(1, 3)\n",
    "fig.set_size_inches(15, 6)\n",
    "sns.set_style(\"darkgrid\", {\"axes.facecolor\": \".9\"})\n",
    "\n",
    "sns.barplot(x='n_layer', y='train_acc', hue='hid_dim', data=df, ax=ax[0])\n",
    "sns.barplot(x='n_layer', y='val_acc', hue='hid_dim', data=df, ax=ax[1])\n",
    "sns.barplot(x='n_layer', y='test_acc', hue='hid_dim', data=df, ax=ax[2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "var1 = 'n_layer'\n",
    "var2 = 'hid_dim'\n",
    "\n",
    "df = load_exp_result('exp1')\n",
    "list_v1 = df[var1].unique()\n",
    "list_v2 = df[var2].unique()\n",
    "list_data = []\n",
    "\n",
    "for value1 in list_v1:\n",
    "    for value2 in list_v2:\n",
    "        row = df.loc[df[var1]==value1]\n",
    "        row = row.loc[df[var2]==value2]\n",
    "        \n",
    "        train_losses = list(row.train_losses)[0]\n",
    "        val_losses = list(row.val_losses)[0]\n",
    "        \n",
    "        for epoch, train_loss in enumerate(train_losses):\n",
    "            list_data.append({'type':'train', 'loss':train_loss, 'epoch':epoch, var1:value1, var2:value2})\n",
    "        for epoch, val_loss in enumerate(val_losses):\n",
    "            list_data.append({'type':'val', 'loss':val_loss, 'epoch':epoch, var1:value1, var2:value2})\n",
    "        \n",
    "df = pd.DataFrame(list_data)\n",
    "g = sns.FacetGrid(df, row=var2, col=var1, hue='type', margin_titles=True, sharey=False)\n",
    "g = g.map(plt.plot, 'epoch', 'loss', marker='.')\n",
    "g.add_legend()\n",
    "g.fig.suptitle('Train loss vs Val loss')\n",
    "plt.subplots_adjust(top=0.89)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "var1 = 'n_layer'\n",
    "var2 = 'hid_dim'\n",
    "\n",
    "df = load_exp_result('exp1')\n",
    "list_v1 = df[var1].unique()\n",
    "list_v2 = df[var2].unique()\n",
    "list_data = []\n",
    "\n",
    "for value1 in list_v1:\n",
    "    for value2 in list_v2:\n",
    "        row = df.loc[df[var1]==value1]\n",
    "        row = row.loc[df[var2]==value2]\n",
    "        \n",
    "        train_accs = list(row.train_accs)[0]\n",
    "        val_accs = list(row.val_accs)[0]\n",
    "        test_acc = list(row.test_acc)[0]\n",
    "        \n",
    "        for epoch, train_acc in enumerate(train_accs):\n",
    "            list_data.append({'type':'train', 'Acc':train_acc, 'test_acc':test_acc, 'epoch':epoch, var1:value1, var2:value2})\n",
    "        for epoch, val_acc in enumerate(val_accs):\n",
    "            list_data.append({'type':'val', 'Acc':val_acc, 'test_acc':test_acc, 'epoch':epoch, var1:value1, var2:value2})\n",
    "        \n",
    "df = pd.DataFrame(list_data)\n",
    "g = sns.FacetGrid(df, row=var2, col=var1, hue='type', margin_titles=True, sharey=False)\n",
    "g = g.map(plt.plot, 'epoch', 'Acc', marker='.')\n",
    "\n",
    "def show_acc(x, y, metric, **kwargs):\n",
    "    plt.scatter(x, y, alpha=0.3, s=1)\n",
    "    metric = \"Test Acc: {:1.3f}\".format(list(metric.values)[0])\n",
    "    plt.text(0.05, 0.95, metric,  horizontalalignment='left', verticalalignment='center', transform=plt.gca().transAxes, bbox=dict(facecolor='yellow', alpha=0.5, boxstyle=\"round,pad=0.1\"))\n",
    "g = g.map(show_acc, 'epoch', 'Acc', 'test_acc')\n",
    "\n",
    "g.add_legend()\n",
    "g.fig.suptitle('Train Accuracy vs Val Accuracy')\n",
    "\n",
    "\n",
    "\n",
    "plt.subplots_adjust(top=0.89)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "Untitled4.ipynb의 사본",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python (comet2)",
   "language": "python",
   "name": "comet2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}