(for future reference) Spinetrack Data Cleaning and ML Model - Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data cleaning and reformating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets, transforms, models\n",
    "from torch.utils.data import DataLoader, Dataset\\\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "from random import shuffle, seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = ['TimeSec', 'Sensor', 'Quatx', 'Quaty', 'Quatz', 'Quat0', 'Heading',\n",
    "       'Pitch', 'Roll', 'LinAccx', 'LinAccy', 'LinAccz', 'Vbat', 'Accx',\n",
    "       'Accy', 'Accz', 'Gyrox', 'Gyroy', 'Gyroz']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Take a look at one dataset\n",
    "# file_path = \"./spinetrack data/Task_Drilling_Stoop.csv\"\n",
    "# df = pd.read_csv(file_path, error_bad_lines=False)\n",
    "# df.columns = header\n",
    "\n",
    "# df[\"Activity\"] = [1 for i in range(len(df))]\n",
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index_lst = []\n",
    "# index_dict = {}\n",
    "# # for i in sorted(os.listdir('./Spinetrack Data/data/Alex_data/Processed_redone')):\n",
    "# for i in sorted(os.listdir('./Spinetrack Data/data/Alex_data/Processed')):\n",
    "#     if i.endswith(\".csv\") and i != 'super_features.csv': \n",
    "#         # should we combinme same activities together?\n",
    "#         act_lst = i.split('.')[0].split('_')\n",
    "#         if len(act_lst) > 1 and act_lst[1] == 'OneH':\n",
    "#             act = act_lst[0] + '_' + act_lst[1]\n",
    "#         else:\n",
    "#             act = act_lst[0]\n",
    "#         #act = i.split('.')[0]\n",
    "#         index_lst.append(act)\n",
    "\n",
    "# # delete multiple items\n",
    "# index_set = set(index_lst)\n",
    "# index_set.remove('Static')\n",
    "# index_set.remove('L')\n",
    "# index_set.remove('L_OneH')\n",
    "# index_set.add('Static_Stoop')\n",
    "# index_set.add('Lifting')\n",
    "# index_set.add('Lifting_OneH')\n",
    "\n",
    "# index_dict = {act:i for i, act in enumerate(index_set)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(index_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_dict = {'Pulling_OneH': 0, 'Overhead': 1, 'Pulling': 2, 'Sitting': 3, \n",
    "              'Lifting': 4, 'Crawling': 5, 'Standing': 6, 'Carrying': 7, \n",
    "              'Walking': 8, 'Pushing': 9, 'Reaching': 10, 'Static_Stoop': 11, \n",
    "              'Kneeling': 12, 'Lifting_OneH': 13, 'Crouching': 14}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = \"./Spinetrack Data/data/\"\n",
    "file = []\n",
    "\n",
    "for d in sorted(os.listdir(directory)):\n",
    "    if d != '.DS_Store' and d != 'Niraj_data':\n",
    "        files = directory + d + \"/Processed_redone\"\n",
    "#         files = directory + d\n",
    "        name = []\n",
    "        try:\n",
    "            for f in sorted(os.listdir(files)):\n",
    "                if f.endswith(\".csv\") and f != 'super_features.csv' and not f.startswith('._'): \n",
    "                    name.append(files + \"/\" + f)\n",
    "            file.append(name)\n",
    "            print(\"%s number of activity: %s\" %(d, len(name)))\n",
    "        except:\n",
    "            pass\n",
    "#file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame = []\n",
    "for person in file:\n",
    "    for file_path in person:\n",
    "        activity_name = ''\n",
    "        activity_lst = file_path.split(\"/\")[-1].split('.')[0].split('_')\n",
    "        \n",
    "        if activity_lst[0] == 'L' and activity_lst[1] == 'OneH':\n",
    "            activity_name = 'Lifting_OneH'\n",
    "        elif activity_lst[0] == 'L' and activity_lst[1] != 'OneH':\n",
    "            activity_name = 'Lifting'\n",
    "        elif activity_lst[0] == 'Static':\n",
    "            activity_name = 'Static_Stoop'\n",
    "        elif activity_lst[0] == 'Pulling' and activity_lst[1] == 'OneH':\n",
    "            activity_name = 'Pulling_OneH'\n",
    "        else:\n",
    "            activity_name = activity_lst[0]\n",
    "#         print(activity_name)\n",
    "        print(\"processing: \", file_path)\n",
    "        if activity_name in index_dict.keys():\n",
    "            df = pd.read_csv(file_path, error_bad_lines=False, skiprows=5)\n",
    "            df.columns = header\n",
    "            df[\"activity\"] = [index_dict[activity_name] for i in range(len(df))]\n",
    "            frame.append(df)\n",
    "            \n",
    "            # mkdir if not exist. Save to local csv file\n",
    "#             if not (os.path.exists('./Spinetrack Data/Yibin_Processed/' + category + folder_name)):\n",
    "#                 os.makedirs('./Spinetrack Data/Yibin_Processed/' + category + folder_name)\n",
    "#             csv_name = '/' + activity_name + '.csv'\n",
    "#             folder_name = file_path.split(\"/\")[-3] # person's name \n",
    "#             category = 'data/' # data or task\n",
    "#             file_name = './Spinetrack Data/Yibin_Processed/' + category + folder_name + csv_name\n",
    "#             df.to_csv(file_name) # save csv processed file to local\n",
    "            \n",
    "            #print(file_path)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the correct dataset should have 986091 rows and 20 columns\n",
    "data_df = pd.concat(frame).dropna()\n",
    "data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result_df = alex_data_df\n",
    "# result_df\n",
    "result_df = data_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # the sensor label index range from 0 to 7 \n",
    "# sensor_data = []\n",
    "# for i in range(0, 8):\n",
    "#     df = alex_data_df.where(alex_data_df['Sensor'] == i).dropna()\n",
    "#     sensor_data.append(df)\n",
    "# result_df = pd.concat(sensor_data).reset_index(drop=True)\n",
    "# result_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Data processing and deep learning model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter\n",
    "window_size: The number of timesteps in one window (e.g. how many rows in one window).\n",
    "\n",
    "channel: The number of features in one window. Similar to image channels (RGB).\n",
    "\n",
    "batch_size: The numebr of windows in one batch.\n",
    "\n",
    "learning_rate: How fast the model learns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 45\n",
    "channel = 1\n",
    "batch_size = 32\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process original dataset, create windows (window_size samples(rows), about 1 second)\n",
    "data = []\n",
    "window = 1\n",
    "while window*window_size < len(result_df):\n",
    "    data_window = result_df[(window - 1)*window_size:window*window_size]\n",
    "    data.append(data_window.values)\n",
    "    window += 1\n",
    "#data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete window if multiple activities and sensors presents or if the time is not increasing\n",
    "cleaned_data = []\n",
    "for i in data:\n",
    "    previous_activity = -1\n",
    "    previous_sensor = -1\n",
    "    previous_time = -1\n",
    "    for j in i:\n",
    "        current_activity = j[19]\n",
    "        current_sensor = j[1]\n",
    "        current_time = j[0]\n",
    "        if (previous_activity != -1) and (current_activity != previous_activity):\n",
    "            print(\"data contains different activities! Window droped\")\n",
    "            break\n",
    "#         elif (previous_sensor != -1) and (current_sensor != previous_sensor):\n",
    "#             print(\"data contains different sensors! Window droped\")\n",
    "#             break\n",
    "        elif (previous_time != -1) and (current_time < previous_time):\n",
    "            print(\"data contains different inconsistent time! Window droped\")\n",
    "            break\n",
    "        else:\n",
    "            previous_activity = current_activity\n",
    "            previous_sensor = current_sensor\n",
    "            previous_time = current_time\n",
    "    else:\n",
    "        cleaned_data.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cleaned_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_data[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# shuffle the data\n",
    "seed(101)\n",
    "shuffle(cleaned_data)\n",
    "#cleaned_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract label from each window\n",
    "labels = []\n",
    "for i in cleaned_data:\n",
    "    label = i[0][19]\n",
    "    labels.append(label)\n",
    "labels = np.array(labels)\n",
    "#labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract features from each window\n",
    "features = []\n",
    "for i in cleaned_data:\n",
    "    new = np.delete(i, 19, 1)\n",
    "    features.append(new)\n",
    "features = np.array(features)\n",
    "#features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine the features and labels\n",
    "k = list(zip(features, labels))\n",
    "activity_data = pd.DataFrame(k)\n",
    "activity_data.columns = ['features', 'labels']\n",
    "activity_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check the size of activity. The final output of neural net \n",
    "# has to have max_index + 1 output\n",
    "max_index = activity_data['labels'].max()\n",
    "label_size = int(max_index + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if CUDA is available\n",
    "train_on_gpu = torch.cuda.is_available()\n",
    "if train_on_gpu:\n",
    "    print(\"CUDA is available! Training on GPU.\")\n",
    "else:\n",
    "    print(\"CUDA is not available. Training on CPU...\")\n",
    "torch.cuda.current_device()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data to test, validation, and train\n",
    "valid_size = 0.1\n",
    "test_size = 0.1\n",
    "activity_data.columns = [\"features\", \"labels\"]\n",
    "activity_data_train = activity_data[:int(len(activity_data)*(1-valid_size-test_size))]\n",
    "activity_data_valid = activity_data[int(len(activity_data)*(1-valid_size-test_size)):int(len(activity_data)*(1-test_size))]\n",
    "activity_data_test = activity_data[int(len(activity_data)*(1-test_size)):]\n",
    "# activity_data_train.to_csv(\"./activity_data_train.csv\", encoding='utf-8-sig')\n",
    "# activity_data_valid.to_csv(\"./activity_data_valid.csv\", encoding='utf-8-sig')\n",
    "# activity_data_train.to_csv(\"./activity_data_test.csv\", encoding='utf-8-sig')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define our dataset in pytorch\n",
    "class DatasetSpineTrack(Dataset):\n",
    "    \n",
    "    def __init__(self, file, transform=None):\n",
    "        #self.data = pd.read_csv(file_path)\n",
    "        self.data = file\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        # load image as ndarray type (Height * Width * Channels)\n",
    "        # be carefull for converting dtype to np.uint8 [Unsigned integer (0 to 255)]\n",
    "        # in this example, i don't use ToTensor() method of torchvision.transforms\n",
    "        # so you can convert numpy ndarray shape to tensor in PyTorch (H, W, C) --> (C, H, W)\n",
    "        \n",
    "        features = torch.tensor(self.data[\"features\"].iloc[index])\n",
    "        features = features.view(channel, window_size, 19) \n",
    "        labels = torch.tensor(self.data[\"labels\"].iloc[index], dtype=torch.long)\n",
    "        #print(labels.type())\n",
    "        \n",
    "#         if self.transform is not None:\n",
    "#             image = self.transform(image)\n",
    "            \n",
    "        return features, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# construct training and testing dataset in csv\n",
    "# train_dataset = DatasetSpineTrack(\"./activity_data_train.csv\")\n",
    "# valid_dataset = DatasetSpineTrack(\"./activity_data_valid.csv\")\n",
    "# test_dataset = DatasetSpineTrack(\"./activity_data_test.csv\")\n",
    "train_dataset = DatasetSpineTrack(activity_data_train)\n",
    "valid_dataset = DatasetSpineTrack(activity_data_valid)\n",
    "test_dataset = DatasetSpineTrack(activity_data_test)\n",
    "feature, label = train_dataset.__getitem__(0)\n",
    "#feature\n",
    "#label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)\n",
    "validloader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=True, drop_last=True)\n",
    "testloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True, drop_last=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Network Architechture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_on_gpu = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.resnet18(pretrained=False)\n",
    "# window_size channels\n",
    "# model.conv1 = torch.nn.Conv2d(window_size, batch_size, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
    "model.conv1 = torch.nn.Conv2d(channel, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
    "model.fc = torch.nn.Linear(512, label_size, bias=True)\n",
    "model.add_module(\"dropout\", torch.nn.Dropout(p=0.5))\n",
    "model = model.double()\n",
    "\n",
    "# move tensors to GPU is CUDA is available\n",
    "if train_on_gpu:\n",
    "    model.cuda()\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Training the Network\n",
    "\n",
    "Remember to look at how the training and validation loss decreases over time; if the validation loss ever increases it indicates possible overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "#criterion = nn.NLLLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# number of epochs to train the model\n",
    "n_epochs = 30\n",
    "\n",
    "valid_loss_min = np.Inf # track change in validation loss\n",
    "\n",
    "for epoch in range(1, n_epochs+1):\n",
    "\n",
    "    # keep track of training and validation loss\n",
    "    train_loss = 0.0\n",
    "    valid_loss = 0.0\n",
    "    \n",
    "    ###################\n",
    "    # train the model #\n",
    "    ###################\n",
    "    model.train()\n",
    "    for features, labels in trainloader:\n",
    "        # move tensors to GPU if CUDA is available\n",
    "        if train_on_gpu:\n",
    "            features, labels = features.cuda(), labels.cuda()\n",
    "        # clear the gradients of all optimized variables\n",
    "        optimizer.zero_grad()\n",
    "        # forward pass: compute predicted outputs by passing inputs to the model\n",
    "        output = model(features)\n",
    "        # calculate the batch loss\n",
    "        loss = criterion(output, labels)\n",
    "        # backward pass: compute gradient of the loss with respect to model parameters\n",
    "        loss.backward()\n",
    "        # perform a single optimization step (parameter update)\n",
    "        optimizer.step()\n",
    "        # update training loss\n",
    "        train_loss += loss.item()*features.size(0)\n",
    "        \n",
    "    ######################    \n",
    "    # validate the model #\n",
    "    ######################\n",
    "    model.eval()\n",
    "    for features, labels in validloader:\n",
    "        # move tensors to GPU if CUDA is available\n",
    "        if train_on_gpu:\n",
    "            features, labels = features.cuda(), labels.cuda()\n",
    "        # forward pass: compute predicted outputs by passing inputs to the model\n",
    "        output = model(features)\n",
    "        # calculate the batch loss\n",
    "        loss = criterion(output, labels)\n",
    "        # update average validation loss \n",
    "        valid_loss += loss.item()*features.size(0)\n",
    "    \n",
    "    # calculate average losses\n",
    "    train_loss = train_loss/len(trainloader.sampler)\n",
    "    valid_loss = valid_loss/len(validloader.sampler)\n",
    "        \n",
    "    # print training/validation statistics \n",
    "    print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(\n",
    "        epoch, train_loss, valid_loss))\n",
    "    \n",
    "    # save model if validation loss has decreased\n",
    "    if valid_loss <= valid_loss_min:\n",
    "        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n",
    "        valid_loss_min,\n",
    "        valid_loss))\n",
    "        torch.save(model.state_dict(), 'model_Spinetrack_data.pt')\n",
    "        valid_loss_min = valid_loss\n",
    "    \n",
    "    if epoch == 1:\n",
    "        epoch_running_time = time.time() - start_time\n",
    "        epoch_sec = epoch_running_time % 60\n",
    "        epoch_miniute = epoch_running_time / 60\n",
    "        print(\"Training time for first epoch: {} min {} sec\".format(int(epoch_miniute), int(epoch_sec)))\n",
    "        \n",
    "# output running time\n",
    "running_time = time.time() - start_time\n",
    "sec = running_time % 60\n",
    "miniute = running_time / 60\n",
    "print(\"Training time: {} min {} sec\".format(int(miniute), int(sec)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Load the Model with the Lowest Validation Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('model_Spinetrack_data.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Test the Trained Network\n",
    "\n",
    "Test your trained model on previously unseen data! A \"good\" result will be a result that gets more than 70% accuracy on these test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Release all the GPU memory cache that can be freed\n",
    "torch.cuda.empty_cache() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall accuracy and each class accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# track test loss\n",
    "test_loss = 0.0\n",
    "class_correct = list(0. for i in range(len(index_dict)))\n",
    "class_total = list(0. for i in range(len(index_dict)))\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "#criterion = nn.NLLLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.003)\n",
    "\n",
    "model.eval()\n",
    "torch.no_grad()\n",
    "# iterate over test data\n",
    "for features, labels in testloader:\n",
    "    # move tensors to GPU if CUDA is available\n",
    "    if train_on_gpu:\n",
    "        features, labels = features.cuda(), labels.cuda()\n",
    "    # forward pass: compute predicted outputs by passing inputs to the model\n",
    "    output = model(features)\n",
    "    # calculate the batch loss\n",
    "    loss = criterion(output, labels)\n",
    "    # update test loss \n",
    "    test_loss += loss.item()*features.size(0)\n",
    "    # convert output probabilities to predicted class\n",
    "    _, pred = torch.max(output, 1)    \n",
    "    # compare predictions to true label\n",
    "    correct_tensor = pred.eq(labels.data.view_as(pred))\n",
    "    correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())\n",
    "    # calculate test accuracy for each object class\n",
    "    for i in range(batch_size):\n",
    "        try:\n",
    "            label = labels.data[i]\n",
    "            class_correct[label] += correct[i].item()\n",
    "            class_total[label] += 1\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "# average test loss\n",
    "test_loss = test_loss/len(testloader.dataset)\n",
    "print('Test Loss: {:.6f}\\n'.format(test_loss))\n",
    "\n",
    "for i in range(len(index_dict)):\n",
    "    if class_total[i] > 0:\n",
    "        print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (\n",
    "            list(index_dict.keys())[i], 100 * class_correct[i] / class_total[i],\n",
    "            np.sum(class_correct[i]), np.sum(class_total[i])))\n",
    "    else:\n",
    "        print('Test Accuracy of %5s: N/A (no training examples)' % (list(index_dict.keys())[i]))\n",
    "\n",
    "print('\\nTest Accuracy (Overall): %2d%% (%2d/%2d)' % (\n",
    "    100. * np.sum(class_correct) / np.sum(class_total),\n",
    "    np.sum(class_correct), np.sum(class_total)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall accuracy (different calculation method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Turn off gradients for validation, saves memory and computations\n",
    "torch.no_grad()\n",
    "accuracy = 0\n",
    "for features, labels in testloader:\n",
    "    # move tensors to GPU if CUDA is available\n",
    "    if train_on_gpu:\n",
    "        features, labels = features.cuda(), labels.cuda()\n",
    "    loss = model(features)\n",
    "    test_loss += criterion(loss, labels)\n",
    "\n",
    "#     ps = torch.exp(loss)\n",
    "    top_p, top_class = loss.topk(1, dim=1)\n",
    "    equals = top_class == labels.view(*top_class.shape)\n",
    "    accuracy += torch.mean(equals.type(torch.FloatTensor))\n",
    "print(\"Test Accuracy: {:.3f}\".format(accuracy/len(testloader)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Figure out pulling_OneH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}