3. Cross Validation for Multi-Class Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1cfd676d",
   "metadata": {},
   "source": [
    "# Notebook 3: Cross Validation for Multi-Class Classification\n",
    "__Author: Bibek Poudel__\n",
    "\n",
    "This Notebook performs 5 fold cross validation to find the best \"number of hidden units in 2 hidden layers\" in the MLP size specified by the question. For the dataset given, \n",
    "\n",
    "The number of hidden units in the first hidden layer (L1) is chosen from {50, 75, 100} and \n",
    "\n",
    "The number of hidden units in the second hidden layer (L2) is chosen from {10, 15,20}\n",
    "\n",
    "\n",
    "The results are as follows: \n",
    "\n",
    "| Dataset | No. of hidden units in L1 | No. of hidden units in L2 |Average Cross Validation Accuracy across 5 folds|\n",
    "| --- | --- | --- | ---|\n",
    "| Given Multiclass Dataset | 100 | 20 | 84.43% | \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "94b37545",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset\n",
    "import random\n",
    "from sklearn.model_selection import KFold\n",
    "from scipy.io import loadmat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "86cc13c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fe063251a10>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set seeds for reproducibility of results\n",
    "SEED = 101\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c895004",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset Loaded:\n",
      "\n",
      "\tdata: train_images , shape = (10000, 784)\n",
      "\n",
      "\tdata: test_labels , shape = (1, 1000)\n",
      "\n",
      "\tdata: train_labels , shape = (1, 10000)\n",
      "\n",
      "\tdata: test_images , shape = (1000, 784)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Load dataset paths\n",
    "datasets = glob.glob('./datasets/multi-class/*.mat')\n",
    "print(\"Dataset Loaded:\\n\")\n",
    "    \n",
    "# Data Exploration\n",
    "for dataset_path in datasets:\n",
    "    path_name_end = dataset_path.split('/')[-1]\n",
    "    #print(path_name_end)\n",
    "    \n",
    "    d_set = loadmat(dataset_path)\n",
    "    d_name = path_name_end.split('.')[0]\n",
    "    #print(d_set.keys())\n",
    "    data = d_set[d_name]\n",
    "    print(\"\\tdata: {} , shape = {}\\n\".format(d_name,data.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2e04c812",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a dataset class (Required by pytorch)\n",
    "class MultiDataset(Dataset):\n",
    "    def __init__(self, inputs, targets):\n",
    "        self.inputs_list = inputs\n",
    "        self.target_list = targets\n",
    "        assert (len(self.inputs_list) == len(self.target_list))\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.inputs_list)\n",
    "    \n",
    "    def __getitem__(self, key):\n",
    "        input_idx = self.inputs_list[key]\n",
    "        target_idx = self.target_list[key]\n",
    "        return [input_idx.astype(np.float32), target_idx.astype(np.float32) ]#.astype(np.long)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "751408f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a neural network with a single hidden layer\n",
    "class model(nn.Module):\n",
    "    def __init__(self, input_units, hidden_units_l1, hidden_units_l2, output_units):\n",
    "        super().__init__()\n",
    "        self.activation = nn.ReLU() # ReLU activation function\n",
    "        self.fc1 = nn.Linear(input_units, hidden_units_l1) # Hidden units can be specified arbitrarily\n",
    "        self.fc2 = nn.Linear(hidden_units_l1, hidden_units_l2) # Hidden units can be specified arbitrarily\n",
    "        self.fc3 = nn.Linear(hidden_units_l2, output_units) # Output units can be specified arbitrarily\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.activation(self.fc1(x))\n",
    "        x = self.activation(self.fc2(x)) \n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "40b9d89f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Performance metric\n",
    "def akkuracy(ground_truths, predictions):\n",
    "    num_correct = np.sum(ground_truths==predictions)\n",
    "    total = ground_truths.shape[0]\n",
    "    return num_correct*100.0 / total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f86e4705",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a method to train neural network model\n",
    "def train(model, dataloader, optimizer, learning_rate, loss_function, epochs, train_size):\n",
    "    \n",
    "    epoch_losses = np.zeros(epochs)\n",
    "    for epoch in range(epochs):\n",
    "        predictions =  np.zeros(train_size)\n",
    "        ground_truths_total= np.zeros(train_size)\n",
    "        count =0\n",
    "        \n",
    "        #print(\"Epoch:{}\".format(epoch), end = \"\\t\")\n",
    "        total_loss = 0.0\n",
    "\n",
    "        for bi, (inputs, targets) in enumerate(dataloader):\n",
    "            current_size=targets.shape[0]\n",
    "            ground_truths_total[count: count+current_size] = targets\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            outputs = model(inputs)\n",
    "\n",
    "            #print(outputs)\n",
    "            #print(targets)\n",
    "            \n",
    "            loss = loss_function(outputs.float(), targets.long()) # targets should be long [batch size], outputs are distributed across the classes?\n",
    "            \n",
    "            loss.backward()\n",
    "\n",
    "            optimizer.step()\n",
    "            \n",
    "            total_loss+=loss.item()\n",
    "            \n",
    "            _, predicted = torch.max(outputs.data, 1) \n",
    "            predicted_np = np.squeeze(predicted.cpu().detach().numpy())\n",
    "            predictions[count: count+current_size] = predicted_np\n",
    "            \n",
    "            count+=current_size\n",
    "        \n",
    "        avg_batch_loss = round(total_loss/len(dataloader),3)\n",
    "        #print(\"Average batch loss:{}\".format(avg_batch_loss))\n",
    "        epoch_losses[epoch] = avg_batch_loss\n",
    "        torch.save(model, \"./saved_models/cross_val_model_multi.pt\")\n",
    "        train_accuracy = akkuracy(ground_truths_total, predictions)\n",
    "    \n",
    "    return train_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "06ddefd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a method to perform model validation\n",
    "def validate(trained_model, dataloader, val_size):\n",
    "    predictions =  np.zeros(val_size)\n",
    "    ground_truths_total= np.zeros(val_size)\n",
    "    count =0\n",
    "    with torch.no_grad():\n",
    "        for bi, (inputs, ground_truths) in enumerate(dataloader):\n",
    "            \n",
    "            outputs = trained_model(inputs)\n",
    "            _, predicted = torch.max(outputs.data, 1) \n",
    "            predicted_np = np.squeeze(predicted.cpu().detach().numpy())\n",
    "            current_size=ground_truths.shape[0]\n",
    "            predictions[count: count+current_size] = predicted_np\n",
    "            ground_truths_total[count: count+current_size] = ground_truths\n",
    "            count+=current_size\n",
    "\n",
    "    validation_accuracy = akkuracy(ground_truths_total, predictions)\n",
    "    return validation_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cb22d8bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a method to perform corss five fold validation (80%, 20% split)\n",
    "def cross_validate(dataset_train_X, dataset_train_Y, folds, hidden_units_l1, hidden_units_l2, output_units):\n",
    "    #print(\"Training set received\", dataset_train_X.shape)\n",
    "    results=0.0\n",
    "    kfold = KFold(n_splits=folds, shuffle=True)\n",
    "    \n",
    "    for fold, (train_ids, val_ids) in enumerate(kfold.split(dataset_train_X)):\n",
    "        print(\"\\tCross validation fold\",fold+1)\n",
    "        #print(train_ids)\n",
    "        #print(val_ids)\n",
    "\n",
    "        # Training\n",
    "        net = model(dataset_train_X.shape[1], hidden_units_l1, hidden_units_l2, output_units).train()\n",
    "        \n",
    "        # Using some standard hyper-parameter values for cross validation\n",
    "        epochs =50\n",
    "        loss = nn.CrossEntropyLoss() # Loss function as Cross Entropy\n",
    "        lr = 0.0001 # Learning rate \n",
    "        optimizer = torch.optim.Adam(net.parameters(), lr = lr) # Optimizer as Adam\n",
    "        BATCH_SIZE = 32\n",
    "        \n",
    "        fold_train_features =  dataset_train_X[train_ids]\n",
    "        fold_train_targets = dataset_train_Y[train_ids]\n",
    "        \n",
    "        fold_train_dataset = MultiDataset(fold_train_features, fold_train_targets)\n",
    "\n",
    "        fold_train_dataloader = torch.utils.data.DataLoader(dataset=fold_train_dataset,\n",
    "                                                   batch_size=BATCH_SIZE,\n",
    "                                                   collate_fn=None,\n",
    "                                                   shuffle=False)\n",
    "        \n",
    "        train_acc = train(net, fold_train_dataloader, optimizer, lr, loss, epochs,fold_train_targets.shape[0])\n",
    "        print(\"\\tTraining Complete with accuracy = {}% ..saving a model\".format(round(train_acc,3)))\n",
    "        \n",
    "        # Validation\n",
    "        # Load the model that was trained this fold\n",
    "        \n",
    "        trained_model = torch.load(\"./saved_models/cross_val_model_multi.pt\")\n",
    "\n",
    "        fold_val_features = dataset_train_X[val_ids]\n",
    "        fold_val_targets = dataset_train_Y[val_ids]\n",
    "        \n",
    "        fold_val_dataset = MultiDataset(fold_val_features, fold_val_targets)\n",
    "        \n",
    "       \n",
    "        fold_val_dataloader = torch.utils.data.DataLoader(dataset=fold_val_dataset,\n",
    "                                                   batch_size=BATCH_SIZE,\n",
    "                                                   collate_fn=None,\n",
    "                                                   shuffle=False)\n",
    "    \n",
    "        valid_acc = validate(trained_model, fold_val_dataloader, fold_val_targets.shape[0])\n",
    "        results+=valid_acc\n",
    "        print(\"\\tValidation Accuracy = {}%\\n\".format(round(valid_acc,3)))\n",
    "        \n",
    "    final_acc = round(results/folds,3)\n",
    "    print(\"Mean validation accuracy across {} folds: {}%\".format(folds, final_acc))\n",
    "    return final_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f0939ff0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "__________________________________________________________\n",
      "\n",
      "Performing 5 fold cross validation in the multi class dataset\n",
      "Varying number of hidden units: L1 in [50, 75, 100] and L2 in [10, 15, 20]\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 50 and L2= 10\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 92.312% ..saving a model\n",
      "\tValidation Accuracy = 81.6%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 92.775% ..saving a model\n",
      "\tValidation Accuracy = 82.4%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 91.412% ..saving a model\n",
      "\tValidation Accuracy = 82.1%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 92.75% ..saving a model\n",
      "\tValidation Accuracy = 84.35%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 94.125% ..saving a model\n",
      "\tValidation Accuracy = 82.7%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 82.63%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 50 and L2= 15\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 95.512% ..saving a model\n",
      "\tValidation Accuracy = 83.1%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 94.438% ..saving a model\n",
      "\tValidation Accuracy = 82.3%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 94.288% ..saving a model\n",
      "\tValidation Accuracy = 80.75%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 93.062% ..saving a model\n",
      "\tValidation Accuracy = 82.0%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 93.775% ..saving a model\n",
      "\tValidation Accuracy = 82.1%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 82.05%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 50 and L2= 20\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 95.862% ..saving a model\n",
      "\tValidation Accuracy = 83.5%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 96.325% ..saving a model\n",
      "\tValidation Accuracy = 82.4%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 96.05% ..saving a model\n",
      "\tValidation Accuracy = 83.35%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 95.912% ..saving a model\n",
      "\tValidation Accuracy = 83.2%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 96.062% ..saving a model\n",
      "\tValidation Accuracy = 84.05%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 83.3%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 75 and L2= 10\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 91.262% ..saving a model\n",
      "\tValidation Accuracy = 82.1%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 94.05% ..saving a model\n",
      "\tValidation Accuracy = 81.75%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 93.825% ..saving a model\n",
      "\tValidation Accuracy = 83.0%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 95.375% ..saving a model\n",
      "\tValidation Accuracy = 83.5%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 94.075% ..saving a model\n",
      "\tValidation Accuracy = 83.55%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 82.78%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 75 and L2= 15\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 96.175% ..saving a model\n",
      "\tValidation Accuracy = 82.4%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 94.925% ..saving a model\n",
      "\tValidation Accuracy = 82.8%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 96.275% ..saving a model\n",
      "\tValidation Accuracy = 82.8%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 95.438% ..saving a model\n",
      "\tValidation Accuracy = 83.6%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 95.375% ..saving a model\n",
      "\tValidation Accuracy = 83.45%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 83.01%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 75 and L2= 20\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 95.762% ..saving a model\n",
      "\tValidation Accuracy = 82.65%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 96.325% ..saving a model\n",
      "\tValidation Accuracy = 82.8%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 95.712% ..saving a model\n",
      "\tValidation Accuracy = 84.05%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 96.625% ..saving a model\n",
      "\tValidation Accuracy = 82.15%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 96.425% ..saving a model\n",
      "\tValidation Accuracy = 80.85%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 82.5%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 100 and L2= 10\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 94.912% ..saving a model\n",
      "\tValidation Accuracy = 83.75%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 94.8% ..saving a model\n",
      "\tValidation Accuracy = 82.6%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 94.112% ..saving a model\n",
      "\tValidation Accuracy = 84.25%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 94.575% ..saving a model\n",
      "\tValidation Accuracy = 81.9%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 93.3% ..saving a model\n",
      "\tValidation Accuracy = 81.8%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 82.86%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 100 and L2= 15\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 95.275% ..saving a model\n",
      "\tValidation Accuracy = 84.4%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 94.788% ..saving a model\n",
      "\tValidation Accuracy = 80.35%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 95.688% ..saving a model\n",
      "\tValidation Accuracy = 84.4%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 94.375% ..saving a model\n",
      "\tValidation Accuracy = 83.65%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 93.6% ..saving a model\n",
      "\tValidation Accuracy = 84.1%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 83.38%\n",
      "__________________________________________________________\n",
      "\n",
      "\n",
      "Number of hidden units in L1= 100 and L2= 20\n",
      "\n",
      "\tCross validation fold 1\n",
      "\tTraining Complete with accuracy = 95.725% ..saving a model\n",
      "\tValidation Accuracy = 81.0%\n",
      "\n",
      "\tCross validation fold 2\n",
      "\tTraining Complete with accuracy = 97.388% ..saving a model\n",
      "\tValidation Accuracy = 86.1%\n",
      "\n",
      "\tCross validation fold 3\n",
      "\tTraining Complete with accuracy = 96.138% ..saving a model\n",
      "\tValidation Accuracy = 85.65%\n",
      "\n",
      "\tCross validation fold 4\n",
      "\tTraining Complete with accuracy = 96.638% ..saving a model\n",
      "\tValidation Accuracy = 84.65%\n",
      "\n",
      "\tCross validation fold 5\n",
      "\tTraining Complete with accuracy = 95.95% ..saving a model\n",
      "\tValidation Accuracy = 84.75%\n",
      "\n",
      "Mean validation accuracy across 5 folds: 84.43%\n",
      "__________________________________________________________\n",
      "\n",
      "********************************************************************\n",
      "\n",
      "Best number of hidden units in L1 and L2 found to be 100 and 20 respectively with validation accuracy of 84.43 %\n",
      "\n",
      "********************************************************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def return_data(dataset_path):\n",
    "    path_name_end = dataset_path.split('/')[-1]\n",
    "    d_set = loadmat(dataset_path)\n",
    "    d_name = path_name_end.split('.')[0]\n",
    "    data = d_set[d_name]\n",
    "    return data\n",
    "\n",
    "train_X = return_data(datasets[0])\n",
    "train_Y = np.squeeze(return_data(datasets[2]))\n",
    "\n",
    "folds = 5\n",
    "output_units = 10\n",
    "l1_values = [50, 75, 100]\n",
    "l2_values = [10, 15, 20]\n",
    "\n",
    "\n",
    "print(\"\\n__________________________________________________________\")\n",
    "print(\"\\nPerforming 5 fold cross validation in the multi class dataset\")\n",
    "print(\"Varying number of hidden units: L1 in [50, 75, 100] and L2 in [10, 15, 20]\")\n",
    "print(\"__________________________________________________________\\n\")\n",
    "# Vary hidden units in L1 and L2\n",
    "results=[]\n",
    "for l1 in l1_values:\n",
    "    for l2 in l2_values:\n",
    "        print(\"\\nNumber of hidden units in L1= {} and L2= {}\\n\".format(l1,l2))\n",
    "        results.append(cross_validate(train_X, train_Y, folds, l1, l2, output_units))\n",
    "        print(\"__________________________________________________________\\n\")\n",
    "print(\"********************************************************************\\n\")  \n",
    "results = np.array(results)\n",
    "best_acc = np.max(results)\n",
    "ind = np.where(results==np.max(results))[0]\n",
    "#print(ind)\n",
    "\n",
    "l1_best = l1_values[int(ind/3)] # /3 is correct\n",
    "l2_best = l2_values[int(ind%3)]\n",
    "\n",
    "print(\"Best number of hidden units in L1 and L2 found to be {} and {} respectively with validation accuracy of {} %\".format(l1_best, l2_best, best_acc))\n",
    "print(\"\\n********************************************************************\\n\")  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}