src/ML_w16_hw.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HW\n",
    "\n",
    "1. Take 10 pictures of objects within the classifications of CIFAR-10 and change the image such that the format is the same (32x32 pixel resolution). Use your trained CNN model to classify the images. \n",
    "2. Train two more modified versions of the CNN model to improve the results of the first model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.optimizers import SGD, Adam\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, Conv3D, MaxPooling2D, MaxPooling3D, BatchNormalization\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib import gridspec as gridspec\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix, classification_report, ConfusionMatrixDisplay\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, cross_validate\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn import tree, datasets\n",
    "from scipy.io import arff\n",
    "from os.path import join\n",
    "from timeit import default_timer as timer\n",
    "from numba import cuda\n",
    "from GPUtil import showUtilization as gpu_usage\n",
    "import keras.datasets.cifar10 as cifar10\n",
    "import keras.datasets.cifar100 as cifar100\n",
    "import tensorflow as tf\n",
    "import numba\n",
    "import torch\n",
    "import datetime\n",
    "import time\n",
    "import os\n",
    "import math\n",
    "import json\n",
    "import inspect\n",
    "import concurrent.futures as cf  # doesn't work with sklearn\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import copy\n",
    "import statistics as stt\n",
    "import seaborn as sns\n",
    "import pickle\n",
    "import sys\n",
    "sns.set_theme()\n",
    "%matplotlib inline\n",
    "tf.compat.v1.Session(\n",
    "    config=tf.compat.v1.ConfigProto(log_device_placement=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ImgPrep():\n",
    "    '''\n",
    "    This class is used to prepare the images for the model.\n",
    "    Methods below are used to increase training set size.\n",
    "    Becareful when using the methods, they increase the dataset dramatically and can easily exhaust your ram.\n",
    "    For example, if using all methods, the dataset will increase by 16 times. (for cifar10, 50000 -> 800000, require 36.6 GiB of memory)\n",
    "    If during training, you encounter the error '_EagerConst: Dst tensor is not initialized', try to reduce the batch size.\n",
    "    '''\n",
    "    def __init__(self, data='cifar10') -> None:\n",
    "        if data == 'cifar10':\n",
    "            (X_train, y_train), (X_test, y_test) = cifar10.load_data()\n",
    "            labels = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', 5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'}\n",
    "            y_train = np_utils.to_categorical(y_train, 10)\n",
    "            y_test = np_utils.to_categorical(y_test, 10)\n",
    "        elif data == 'cifar100':\n",
    "            (X_train, y_train), (X_test, y_test) = cifar100.load_data()\n",
    "            # label is still unavailable\n",
    "            y_train = np_utils.to_categorical(y_train, 100)\n",
    "            y_test = np_utils.to_categorical(y_test, 100)\n",
    "        elif data == 'Fashion MINST':\n",
    "            (X_train, y_train), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "            labels = {0: 'T-shirt/top', 1: 'Trouser', 2: 'Pullover', 3: 'Dress', 4: 'Coat', 5: 'Sandal', 6: 'Shirt', 7: 'Sneaker', 8: 'Bag', 9: 'Ankle boot'}\n",
    "            y_train = np_utils.to_categorical(y_train, 10)\n",
    "            y_test = np_utils.to_categorical(y_test, 10)\n",
    "        self.X_train = X_train\n",
    "        self.y_train = y_train\n",
    "        self.X_test = X_test\n",
    "        self.y_test = y_test\n",
    "        self.labels = labels\n",
    "\n",
    "    def flip_img(self) -> None:\n",
    "        X_train1 = [np.flip(i) for i in self.X_train]\n",
    "        X_train2 = [np.fliplr(i) for i in self.X_train]\n",
    "        X_train3 = [np.flipud(i) for i in self.X_train]\n",
    "        self.X_train = np.concatenate((self.X_train, X_train1, X_train2, X_train3))\n",
    "        self.y_train = np.concatenate((self.y_train, self.y_train, self.y_train, self.y_train))\n",
    "\n",
    "    def rotate_img(self) -> None:\n",
    "        X_train1 = [np.rot90(i, k=1) for i in self.X_train]\n",
    "        X_train2 = [np.rot90(i, k=2) for i in self.X_train]\n",
    "        X_train3 = [np.rot90(i, k=3) for i in self.X_train]\n",
    "        self.X_train = np.concatenate((self.X_train, X_train1, X_train2, X_train3))\n",
    "        self.y_train = np.concatenate((self.y_train, self.y_train, self.y_train, self.y_train))\n",
    "\n",
    "    def noise_img(self) -> None:\n",
    "        X_train1 = [np.random.normal(i, 0.1) for i in self.X_train]\n",
    "        self.X_train = np.concatenate((self.X_train, X_train1))\n",
    "        self.y_train = np.concatenate((self.y_train, self.y_train))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN():\n",
    "    '''\n",
    "    This class is used to build, train, test, and evaluate the model.\n",
    "    Currently only support CIFAR10\n",
    "    '''\n",
    "    def __init__(self, X_train, y_train, X_test, y_test):\n",
    "        # data variables\n",
    "        self.X_train = X_train\n",
    "        self.y_train = y_train\n",
    "        self.X_test = X_test\n",
    "        self.y_test = y_test\n",
    "        # stats variables\n",
    "        self.stats_dict = {\n",
    "            'model_name': None,\n",
    "            'model_file_path': None,\n",
    "            'epochs': None,\n",
    "            'batch_size': None,\n",
    "            'building_time': None,\n",
    "            'training_time': None,\n",
    "            'testing_time': None,\n",
    "            'loss': None,\n",
    "            'accuracy': None,\n",
    "            'evaluating_proba_time': None,\n",
    "            }\n",
    "        # labels\n",
    "        self.label = {0: \"airplane\", 1: \"automobile\", 2: \"bird\", 3: \"cat\",\n",
    "                 4: \"deer\", 5: \"dog\", 6: \"frog\", 7: \"horse\", 8: \"ship\", 9: \"truck\"}\n",
    "        current_time = str(time.strftime(\"%Y/%m/%d %H:%M:%S\", time.localtime()))\n",
    "        self.stats_dict['time'] = current_time\n",
    "        current_time = current_time.replace('/', '').replace(' ', '_').replace(':', '')\n",
    "        self.current_time = current_time\n",
    "\n",
    "    def build_model1(self):\n",
    "        start = timer()\n",
    "        activation1 = 'relu'\n",
    "        channel1 = 100\n",
    "        channel2 = 200\n",
    "        channel3 = 400\n",
    "        model = Sequential()\n",
    "        model.add(Conv2D(channel1, (9, 9),\n",
    "                         input_shape=X_train.shape[1:], activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(channel2, (5, 5), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(channel3, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(channel3, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(160, activation=activation1))\n",
    "        model.add(Dense(40, activation=activation1))\n",
    "        model.add(Dense(10, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer='adam', metrics=['accuracy'])\n",
    "        self.model = model\n",
    "        self.stats_dict['building_time'] = timer() - start\n",
    "\n",
    "    def build_model2(self):\n",
    "        # testing conv layer depth\n",
    "        # https://pyimagesearch.com/2018/12/31/keras-conv2d-and-convolutional-layers/\n",
    "        # it doesn't looks like 4th or more conv layers are possible (error: ValueError: Negative dimension size caused by subtracting 3 from 1 for 'conv2d_4/convolution' (op: 'Conv2D') with input shapes: [?,1,1,256], [3,3,256,256].)\n",
    "        start = timer()\n",
    "        activation1 = 'relu'\n",
    "        model = Sequential()\n",
    "        model.add(Conv2D(32, (3, 3),\n",
    "                         input_shape=X_train.shape[1:], activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(64, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(128, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        # model.add(Conv2D(256, (3, 3), activation=activation1))\n",
    "        # model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(128, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(160, activation=activation1))\n",
    "        model.add(Dense(40, activation=activation1))\n",
    "        model.add(Dense(10, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer='adam', metrics=['accuracy'])\n",
    "        self.model = model\n",
    "        self.stats_dict['building_time'] = timer() - start\n",
    "\n",
    "    def build_model3(self):\n",
    "        # testing dense layer depth\n",
    "        start = timer()\n",
    "        activation1 = 'relu'\n",
    "        model = Sequential()\n",
    "        model.add(Conv2D(32, (3, 3),\n",
    "                         input_shape=X_train.shape[1:], activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(64, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(128, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(128, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(256, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(128, activation=activation1))\n",
    "        model.add(Dense(64, activation=activation1))\n",
    "        model.add(Dense(32, activation=activation1))\n",
    "        model.add(Dense(16, activation=activation1))\n",
    "        model.add(Dense(10, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer='adam', metrics=['accuracy'])\n",
    "        self.model = model\n",
    "        self.stats_dict['building_time'] = timer() - start\n",
    "\n",
    "    def build_model4(self):\n",
    "        # testing even deeper dense layer\n",
    "        start = timer()\n",
    "        activation1 = 'relu'\n",
    "        model = Sequential()\n",
    "        model.add(Conv2D(32, (3, 3),\n",
    "                        input_shape=X_train.shape[1:], activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(64, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(128, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(128, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(256, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(512, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(1024, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(512, activation=activation1))\n",
    "        model.add(Dense(256, activation=activation1))\n",
    "        model.add(Dense(128, activation=activation1))\n",
    "        model.add(Dense(64, activation=activation1))\n",
    "        model.add(Dense(32, activation=activation1))\n",
    "        model.add(Dense(16, activation=activation1))\n",
    "        model.add(Dense(10, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                    optimizer='adam', metrics=['accuracy'])\n",
    "        self.model = model\n",
    "        self.stats_dict['building_time'] = timer() - start\n",
    "\n",
    "    def build_model5(self):\n",
    "        # testing conv layer features\n",
    "        # https://pyimagesearch.com/2018/12/31/keras-conv2d-and-convolutional-layers/\n",
    "        # it doesn't looks like 4th or more conv layers are possible (error: ValueError: Negative dimension size caused by subtracting 3 from 1 for 'conv2d_4/convolution' (op: 'Conv2D') with input shapes: [?,1,1,256], [3,3,256,256].)\n",
    "        start = timer()\n",
    "        activation1 = 'relu'\n",
    "        model = Sequential()\n",
    "        model.add(Conv2D(32, (5, 5),\n",
    "                         input_shape=X_train.shape[1:], activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(128, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Conv2D(512, (3, 3), activation=activation1))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(512, activation=activation1))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(160, activation=activation1))\n",
    "        model.add(Dense(40, activation=activation1))\n",
    "        model.add(Dense(10, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer='adam', metrics=['accuracy'])\n",
    "        self.model = model\n",
    "        self.stats_dict['building_time'] = timer() - start\n",
    "\n",
    "    def build_model6(self):\n",
    "        # testing deep conv layer\n",
    "        start = timer()\n",
    "        img_width = self.X_train.shape[1:][0]\n",
    "        filter_size = 3\n",
    "        layer_depth = img_width / (filter_size - 1)\n",
    "        model = Sequential()\n",
    "        model.add(Conv2D(32, (3, 3),\n",
    "                                input_shape=self.X_train.shape[1:], activation='relu'))\n",
    "        for i in range(int(layer_depth)-2):\n",
    "            model.add(Conv2D(32, (3, 3), activation='relu'))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(512, activation='relu'))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(160, activation='relu'))\n",
    "        model.add(Dense(40, activation='relu'))\n",
    "        model.add(Dense(10, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                        optimizer='adam', metrics=['accuracy'])\n",
    "        self.model = model\n",
    "        self.stats_dict['building_time'] = timer() - start\n",
    "\n",
    "    def train_model(self, epochs=5, batch_size=500, verbose=1):\n",
    "        self.stats_dict['epochs'] = epochs\n",
    "        self.stats_dict['batch_size'] = batch_size\n",
    "        start = timer()\n",
    "        self.history = self.model.fit(self.X_train, self.y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "        self.stats_dict['training_time'] = timer() - start\n",
    "        self.model.summary()\n",
    "\n",
    "    def store_model(self, model_name, path=''):\n",
    "        path = os.path.join(path, f'{model_name}_{self.current_time}.h5')\n",
    "        print(path)\n",
    "        self.model.save(path)\n",
    "        self.stats_dict['model_name'] = model_name\n",
    "        self.stats_dict['model_file_path'] = path\n",
    "\n",
    "    def store_meta_data(self, model_name, path=''):\n",
    "        # check empty values\n",
    "        values = list(self.stats_dict.values())\n",
    "        if None in values:\n",
    "            arr = []\n",
    "            for element in self.stats_dict:\n",
    "                if self.stats_dict[element] == None:\n",
    "                    arr.append(element)\n",
    "            ans = input(\n",
    "                f'Not all stats are recorded...\\nThese stats are not recorded: {arr}\\nDo you still want to export? (y/[n]): ')\n",
    "            if ans != 'y':\n",
    "                return\n",
    "        \n",
    "        # read meta data, append, and write\n",
    "        path = os.path.join(path, 'model_meta_data.csv')\n",
    "        if os.path.exists(path):\n",
    "            meta_data = pd.read_csv(path, encoding='utf-8', index_col=0)\n",
    "            meta_data = pd.concat([meta_data, pd.DataFrame(self.stats_dict, index=[0])])\n",
    "            meta_data.reset_index(drop=True, inplace=True)\n",
    "        else:\n",
    "            meta_data = pd.DataFrame(self.stats_dict, index=[0])\n",
    "        meta_data.to_csv(path, encoding='utf-8')\n",
    "        print('Meta data exported to', path)\n",
    "\n",
    "    def load_model(self, path):\n",
    "        self.model = load_model(path)\n",
    "        self.model_name = os.path.basename(path).split('.')[0]\n",
    "\n",
    "    def evaluate(self):\n",
    "        # record loss and accuracy\n",
    "        start = timer()\n",
    "        result = self.model.evaluate(X_test, y_test)\n",
    "        self.stats_dict['loss'] = result[0]\n",
    "        self.stats_dict['accuracy'] = result[1]\n",
    "        self.stats_dict['testing_time'] = timer() - start\n",
    "\n",
    "    def evaluate_proba(self, img_row_num=2, img_col_num=5, display_bar_num=3, random=False, start_img_num=0, save=False, path='', show=True):\n",
    "        # top 1 accuracy\n",
    "        '''\n",
    "        display images and their predicted probabilities\n",
    "        -----------------------------------------------------------------------\n",
    "        start_img_num: start image number (necessary if random=False)\n",
    "        save_img_path: path to save images\n",
    "        img_row_num: number of rows of images\n",
    "        img_col_num: number of columns of images\n",
    "        display_bar_num: number of bars to display (probability of each class)\n",
    "        random: if True, select images randomly\n",
    "        save: if True, save images\n",
    "        show: if True, show images\n",
    "        -----------------------------------------------------------------------\n",
    "        src: https://stackoverflow.com/questions/34933905/matplotlib-adding-subplots-to-a-subplot\n",
    "        '''\n",
    "        start = timer()\n",
    "        model = self.model\n",
    "        X_test = self.X_test\n",
    "        y_test = self.y_test\n",
    "        label = self.label\n",
    "        # load test data (images)\n",
    "        image_count = img_row_num * img_col_num\n",
    "        test_image_count = X_test.shape[0]\n",
    "        if random:\n",
    "            selected_image_indexes = np.random.choice(a=test_image_count, size=image_count, replace=False)\n",
    "            selected_image = X_test[selected_image_indexes]\n",
    "        else:\n",
    "            selected_image_indexes = np.arange(\n",
    "                start=start_img_num, stop=start_img_num+image_count)\n",
    "            selected_image = X_test[selected_image_indexes]\n",
    "        # predict\n",
    "        probabilities = model.predict(selected_image)\n",
    "        # calculate prediction accuracy\n",
    "        correct_count = 0\n",
    "        for i in range(image_count):\n",
    "            prediction = np.argmax(probabilities[i])\n",
    "            answer = np.argmax(y_test[selected_image_indexes[i]])\n",
    "            if prediction == answer:\n",
    "                correct_count += 1\n",
    "        accuracy = correct_count / image_count * 100\n",
    "        # plot\n",
    "        fig = plt.figure(figsize=(img_col_num*2, img_row_num*2), dpi=300)\n",
    "        plt.title(\n",
    "            f'Top {display_bar_num} Accuracy of {image_count} Images ({img_col_num} x {img_row_num})\\nAccuracy of this batch: {accuracy:.2f}%', pad=30)\n",
    "        plt.axis('off')\n",
    "        outer = gridspec.GridSpec(img_row_num, img_col_num, wspace=0.1, hspace=0.1)\n",
    "        for i in range(image_count):\n",
    "            prediction = label[np.argmax(probabilities[i])]\n",
    "            answer = label[np.argmax(y_test[selected_image_indexes[i]])]\n",
    "            x = i//img_col_num\n",
    "            y = i%img_col_num\n",
    "            inner = gridspec.GridSpecFromSubplotSpec(\n",
    "                2, 1, subplot_spec=outer[i], wspace=0.1, hspace=0.1)\n",
    "            # plot image\n",
    "            ax = plt.Subplot(fig, inner[0])\n",
    "            ax.set_title(f'{prediction} ({answer})', fontsize=8)\n",
    "            ax.imshow(selected_image[i])\n",
    "            ax.axis('off')\n",
    "            fig.add_subplot(ax)\n",
    "            # get top x probabilities\n",
    "            temp_probabilities = (copy.copy(probabilities[i])).tolist()\n",
    "            temp_probabilities.sort(reverse=True)\n",
    "            temp_probabilities = temp_probabilities[:display_bar_num]\n",
    "            temp_label = []\n",
    "            for j in temp_probabilities:\n",
    "                prob_list = probabilities[i].tolist()\n",
    "                index = prob_list.index(j)\n",
    "                temp_label.append(label[index])\n",
    "            # plot bar\n",
    "            ax = plt.Subplot(fig, inner[1])\n",
    "            ax.barh(np.arange(display_bar_num),\n",
    "                    temp_probabilities, align='center', alpha=0.5)\n",
    "            for element in temp_label:\n",
    "                text = f'{element} ({temp_probabilities[temp_label.index(element)]*100:.2f}%)'\n",
    "                ax.text(0.01, temp_label.index(element), text, fontsize=6, va='center')\n",
    "            ax.invert_yaxis()\n",
    "            ax.axis('off')\n",
    "            fig.add_subplot(ax)\n",
    "        if save:\n",
    "            path = os.path.join(\n",
    "                path, '{}_eval_top{}_{}x{}_{}-{}.png'.format(self.model_name, display_bar_num, img_col_num, img_row_num, start_img_num, start_img_num+image_count))\n",
    "            plt.savefig(path)\n",
    "        if show:\n",
    "            plt.show()\n",
    "        plt.close()\n",
    "        print('Image exported to', path)\n",
    "        self.stats_dict['evaluating_proba_time'] = timer() - start\n",
    "\n",
    "    def clear_mem(self):\n",
    "        # https://www.kaggle.com/getting-started/140636\n",
    "        torch.cuda.empty_cache()\n",
    "        cuda.select_device(0)\n",
    "        cuda.close()\n",
    "        cuda.select_device(0)\n",
    "        print('Cleared GPU memory')\n",
    "\n",
    "    def plot_history(self, model_name, path):\n",
    "        history = self.history\n",
    "        plt.figure(figsize=(20, 20), dpi=300)\n",
    "        # plot loss\n",
    "        plt.subplot(211)\n",
    "        plt.title(f'{model_name}_{self.current_time} - Train Loss')\n",
    "        plt.plot(history.history['loss'], label='train_loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        # plt.legend(loc='upper right')\n",
    "        # plot accuracy\n",
    "        plt.subplot(212)\n",
    "        plt.title(f'{model_name}_{self.current_time} - Train Accuracy')\n",
    "        plt.plot(history.history['accuracy'], label='train_accuracy')\n",
    "        plt.xlabel('Epoch')\n",
    "        # plt.legend(loc='lower right')\n",
    "        # save\n",
    "        path = os.path.join(\n",
    "            path, f'{model_name}_{self.current_time}_train_hist.png')\n",
    "        plt.savefig(path)\n",
    "        plt.close()\n",
    "\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare/Preprocess Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params\n",
    "model_root_path = os.path.join(os.getcwd().rstrip('src'), 'data', 'ml_w14_hw')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datas = ImgPrep(data='cifar10')\n",
    "datas.flip_img()\n",
    "# datas.rotate_img()\n",
    "X_train = datas.X_train\n",
    "y_train = datas.y_train\n",
    "X_test = datas.X_test\n",
    "y_test = datas.y_test\n",
    "del datas\n",
    "print('Data loaded')\n",
    "print('training set shape:', X_train.shape)\n",
    "print('testing set shape:', X_test.shape)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model1\n",
    "cnn = CNN(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)\n",
    "cnn.build_model1()\n",
    "cnn.train_model(epochs=100, batch_size=500, verbose=1)\n",
    "cnn.plot_history(model_name='model1', path=model_root_path)\n",
    "cnn.store_model(model_name='model1', path=model_root_path)\n",
    "cnn.evaluate()\n",
    "cnn.store_meta_data(model_name='model1', path=model_root_path)\n",
    "# cnn.clear_mem()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model2\n",
    "cnn = CNN(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)\n",
    "cnn.build_model2()\n",
    "cnn.train_model(epochs=100, batch_size=500, verbose=1)\n",
    "cnn.plot_history(model_name='model2', path=model_root_path)\n",
    "cnn.store_model(model_name='model2', path=model_root_path)\n",
    "cnn.evaluate()\n",
    "cnn.store_meta_data(model_name='model2', path=model_root_path)\n",
    "# cnn.clear_mem()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model3\n",
    "cnn = CNN(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)\n",
    "cnn.build_model3()\n",
    "cnn.train_model(epochs=100, batch_size=500, verbose=1)\n",
    "cnn.plot_history(model_name='model3', path=model_root_path)\n",
    "cnn.store_model(model_name='model3', path=model_root_path)\n",
    "cnn.evaluate()\n",
    "cnn.store_meta_data(model_name='model3', path=model_root_path)\n",
    "# cnn.clear_mem()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model4\n",
    "cnn = CNN(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)\n",
    "cnn.build_model4()\n",
    "cnn.train_model(epochs=100, batch_size=500, verbose=1)\n",
    "cnn.plot_history(model_name='model4', path=model_root_path)\n",
    "cnn.store_model(model_name='model4', path=model_root_path)\n",
    "cnn.evaluate()\n",
    "cnn.store_meta_data(model_name='model4', path=model_root_path)\n",
    "# cnn.clear_mem()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model5\n",
    "cnn = CNN(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)\n",
    "cnn.build_model5()\n",
    "cnn.train_model(epochs=100, batch_size=1000, verbose=1)\n",
    "cnn.plot_history(model_name='model5', path=model_root_path)\n",
    "cnn.store_model(model_name='model5', path=model_root_path)\n",
    "cnn.evaluate()\n",
    "cnn.store_meta_data(model_name='model5', path=model_root_path)\n",
    "# cnn.clear_mem()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model6\n",
    "cnn = CNN(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)\n",
    "cnn.build_model6()\n",
    "cnn.train_model(epochs=1, batch_size=1, verbose=1)\n",
    "cnn.plot_history(model_name='model6', path=model_root_path)\n",
    "cnn.store_model(model_name='model6', path=model_root_path)\n",
    "cnn.evaluate()\n",
    "cnn.store_meta_data(model_name='model6', path=model_root_path)\n",
    "# cnn.clear_mem()\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Evaluating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate_proba\n",
    "cnn = CNN(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)\n",
    "cnn.load_model('D:/Note_Database/Subject/BD_ML Big Data and Machine Learning/BD_ML_Code/data/ml_w14_hw/model1_20221217_031354.h5')\n",
    "cnn.evaluate_proba(img_row_num=4, img_col_num=6,\n",
    "                   display_bar_num=3, random=False, start_img_num=10,\n",
    "                   save=True, path=model_root_path,show=True)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "26de051ba29f2982a8de78e945f0abaf191376122a1563185a90213a26c5da77"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}