Centaur/ConvAttn/Eval-PAMAP2-ConvAttn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "23c11f73-2263-44a8-9ab6-5c4e78b53e76",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/home/xyang18/miniconda3/envs/pytorch/bin/ python\n",
    "# -*- coding: utf-8 -*-\n",
    "# Python version: 3.6\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import copy\n",
    "import time\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset\n",
    "from torch.autograd import Variable\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import WeightedRandomSampler, TensorDataset\n",
    "from torch.nn import Linear, ReLU, CrossEntropyLoss, Sequential, Conv2d, MaxPool2d, Module, Softmax, BatchNorm2d, BatchNorm1d, Dropout, Flatten, BCELoss\n",
    "from torch.optim import Adam, SGD\n",
    "from torch import nn\n",
    "# from torchsummary import summary\n",
    "# from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a592452e-18a3-4ca4-b953-3bf3b114df38",
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_id=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "00555511-7293-4968-828b-561d95ba82f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda:0\n",
      "Current GPU ID: 0\n"
     ]
    }
   ],
   "source": [
    "if gpu_id>=0:\n",
    "    os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(gpu_id)\n",
    "    cuda_id = \"cuda:\" + str(0)  # cuda:2\n",
    "\n",
    "device = torch.device(cuda_id if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Device:\", device)\n",
    "if (torch.cuda.is_available()):\n",
    "    torch.cuda.set_device(cuda_id)\n",
    "    print(\"Current GPU ID:\", torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "032d5c46-5306-41bd-87e4-9bb5725d6a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_data_PAMAP2(root_path='../../PAMAP2_Dataset/Protocol/subject10'):\n",
    "    X=[]\n",
    "    user_labels=[]\n",
    "    act_labels=[]\n",
    "\n",
    "    window_len = 512\n",
    "    stride_len = 20\n",
    "    # columns for IMU data\n",
    "    imu_locs = [4,5,6, 10,11,12, 13,14,15, \n",
    "                21,22,23, 27,28,29, 30,31,32, \n",
    "                38,39,40, 44,45,46, 47,48,49\n",
    "            ] \n",
    "    \n",
    "    act_list = [1, 2, 3, 4, 5, 6, 7, 12, 13, 16, 17, 24]\n",
    "\n",
    "    scaler = MinMaxScaler()\n",
    "    # scaler = StandardScaler()\n",
    "\n",
    "    for uid in np.arange(1,10):\n",
    "        path = root_path + str(uid) + '.dat'\n",
    "        df = pd.read_table(path, sep=' ', header=None)\n",
    "        act_imu_filter = df.iloc[:, imu_locs] \n",
    "\n",
    "        for act_id in range(len(act_list)):\n",
    "            act_filter =  act_imu_filter[df.iloc[:, 1] == act_list[act_id]]\n",
    "            act_data = act_filter.to_numpy()\n",
    "                \n",
    "            act_data = np.transpose(act_data)\n",
    "            # sliding window segmentation\n",
    "            start_idx = 0\n",
    "            while start_idx + window_len < act_data.shape[1]:\n",
    "                window_data = act_data[:, start_idx:start_idx + window_len]\n",
    "                downsamp_data = window_data[:, ::3] # downsample from 100hz to 33.3hz\n",
    "                downsamp_data = np.nan_to_num(downsamp_data) # remove nan\n",
    "\n",
    "                X.append(downsamp_data)\n",
    "                user_labels.append(uid)\n",
    "                act_labels.append(act_id)\n",
    "                start_idx = start_idx + stride_len\n",
    "\n",
    "    X_n = np.array(X).astype('float32')\n",
    "\n",
    "    normalized_X = np.zeros_like(X_n) # allocate numpy array for normalized data\n",
    "    for ch_id in range(X_n.shape[1]): # loop the 27 sensor channels\n",
    "        ch_data = X_n[:, ch_id, :] # the data of channel id\n",
    "        scaler = MinMaxScaler() # maybe different scalers?\n",
    "        ch_data = scaler.fit_transform(ch_data) # scale the data in this channel to [0,1]\n",
    "        normalized_X[:, ch_id, :] = ch_data # assign normalized data to normalized_X\n",
    "    normalized_X = np.transpose(normalized_X, (0, 2, 1)) # overwrote X here, changed dimensions into: num_samples, sequence_length, feature_length\n",
    "        \n",
    "    # convert list to numpy array\n",
    "    # normalized_X= normalized_X.reshape(normalized_X.shape[0], 1, normalized_X.shape[1], normalized_X.shape[2]) \n",
    "    act_labels = np.array(act_labels).astype('float32')\n",
    "    act_labels = act_labels.reshape(act_labels.shape[0],1)\n",
    "    act_labels = to_categorical(act_labels, num_classes=len(act_list))\n",
    "\n",
    "    return normalized_X, act_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "733e11ee-95e2-4ac7-8f05-8d919813562a",
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_X, act_labels = prepare_data_PAMAP2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8d64a816-75f2-4e43-9b19-e9b5b0e6699e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(94895, 171, 27)\n"
     ]
    }
   ],
   "source": [
    "print(normalized_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ead0f4dc-2c91-4ffc-882c-f78348566762",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(normalized_X, act_labels, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67d445e5-7053-416d-b9ef-65025c3893e2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bbe559cd-c9e2-4ffc-8be3-a7d5536ccedb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sliding_window import sliding_window\n",
    "# import pickle as cp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "73a2cdc1-7025-4e3d-845c-d4b4c781dfda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Number of Sensor Channels used in the OPPORTUNITY dataset.\n",
    "# NB_SENSOR_CHANNELS = 113\n",
    "\n",
    "# # Number of classes in which data is classified (or to be classified).\n",
    "# NUM_CLASSES = 5\n",
    "\n",
    "# # Length of the sliding window used to segmenting the time-series-data.\n",
    "# SLIDING_WINDOW_LENGTH = 24\n",
    "\n",
    "# # Steps of the sliding window used in segmenting the data.\n",
    "# SLIDING_WINDOW_STEP = 12\n",
    "\n",
    "# act_labels_txt = ['std', 'wlk', 'sit', 'lie', 'null']\n",
    "\n",
    "# # Variable for Batch Size.\n",
    "# # BATCH_SIZE = 100\n",
    "\n",
    "# # Number filters used in convolutional layers.\n",
    "# # NUM_FILTERS = 64\n",
    "\n",
    "# # Size of filters used in convolutional layers.\n",
    "# # FILTER_SIZE = 5\n",
    "\n",
    "# # Units in the long short-term recurrent layers.\n",
    "# # NUM_UNITS_LSTM = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2fb453c5-a809-45c7-a23d-fd97db0b3ba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def load_dataset(filename):\n",
    "\n",
    "#     f = open(filename, 'rb')\n",
    "#     data = cp.load(f)\n",
    "#     f.close()\n",
    "\n",
    "#     X_train, y_train = data[0]\n",
    "#     X_test, y_test = data[1]\n",
    "\n",
    "#     print(\" ..from file {}\".format(filename))\n",
    "#     print(\" ..reading instances: train {0}, test {1}\".format(X_train.shape, X_test.shape))\n",
    "\n",
    "#     X_train = X_train.astype(np.float32)\n",
    "#     X_test = X_test.astype(np.float32)\n",
    "\n",
    "#     # The targets are casted to int8 for GPU compatibility.\n",
    "#     y_train = y_train.astype(np.uint8)\n",
    "#     y_test = y_test.astype(np.uint8)\n",
    "\n",
    "#     return X_train, y_train, X_test, y_test\n",
    "\n",
    "# print(\"Loading Data...\")\n",
    "# X_train, y_train, X_test, y_test = load_dataset('../../data/oppChallenge_gestures.data')\n",
    "\n",
    "# assert NB_SENSOR_CHANNELS == X_train.shape[1]\n",
    "# def opp_sliding_window(data_x, data_y, ws, ss):\n",
    "#     data_x = sliding_window(data_x,(ws,data_x.shape[1]),(ss,1))\n",
    "#     data_y = np.asarray([[i[-1]] for i in sliding_window(data_y,ws,ss)])\n",
    "#     return data_x.astype(np.float32), data_y.reshape(len(data_y)).astype(np.uint8)\n",
    "\n",
    "# # Sensor data is segmented using a sliding window mechanism\n",
    "# X_test, y_test = opp_sliding_window(X_test, y_test, SLIDING_WINDOW_LENGTH, SLIDING_WINDOW_STEP)\n",
    "# print(\" ..after sliding window (testing): inputs {0}, targets {1}\".format(X_test.shape, y_test.shape))\n",
    "\n",
    "# # Data is reshaped since the input of the network is a 4 dimension tensor\n",
    "# X_test = X_test.reshape((-1, SLIDING_WINDOW_LENGTH, NB_SENSOR_CHANNELS))\n",
    "# # X_test = X_test.reshape((-1, 1, SLIDING_WINDOW_LENGTH, NB_SENSOR_CHANNELS))\n",
    "\n",
    "# # X_test = np.transpose(X_test, (0, 2, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2c820654-1fd1-4d0e-a55e-130b31f599d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train, y_train = opp_sliding_window(X_train, y_train, SLIDING_WINDOW_LENGTH, SLIDING_WINDOW_STEP)\n",
    "# print(\" ..after sliding window (training): inputs {0}, targets {1}\".format(X_train.shape, y_train.shape))\n",
    "# X_train = X_train.reshape((-1,SLIDING_WINDOW_LENGTH, NB_SENSOR_CHANNELS))\n",
    "# # X_train = X_train.reshape((-1,1,SLIDING_WINDOW_LENGTH, NB_SENSOR_CHANNELS,1))\n",
    "\n",
    "# # X_train = np.transpose(X_train, (0, 2, 1))\n",
    "# X_train.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a00b4557-6625-4893-ba69-6d7520f87f7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_train = to_categorical(y_train, num_classes=NUM_CLASSES)\n",
    "# y_test = to_categorical(y_test, num_classes=NUM_CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9106183-640b-4f60-9a45-5187da4b6123",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b68832a1-cee5-4152-8e96-1a7fc3a84ad7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff90bb9d-5770-44cc-9838-5f3ec86bdaea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cf03f80-c7e8-439b-82d1-ccd22adf98dd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "932b91b8-a0b4-4c74-9f55-7a56f18cda94",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ccadf5e9-5d4e-4008-bd3a-7ce4f37606aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# normalized_X = np.zeros_like(X)\n",
    "# for ch_id in range(X.shape[1]):\n",
    "#     ch_data = X[:, ch_id, :]\n",
    "#     scaler = MinMaxScaler()\n",
    "#     ch_data = scaler.fit_transform(ch_data)\n",
    "#     normalized_X[:, ch_id, :] = ch_data\n",
    "# X = normalized_X\n",
    "# # X = np.transpose(normalized_X, (0, 2, 1))\n",
    "# print(X.shape)\n",
    "# # (94895, 27, 171)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e31856f2-67aa-4118-995c-85a825d9ba06",
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1], X_train.shape[2]) # convert list to numpy array\n",
    "# X_test = X_test.reshape(X_test.shape[0], 1, X_test.shape[1], X_test.shape[2]) # convert list to numpy array\n",
    "# #print(X.shape)\n",
    "# # #(94895, 1, 27, 171)\n",
    "\n",
    "# # act_labels = np.array(act_labels).astype('float32')\n",
    "# # act_labels = act_labels.reshape(act_labels.shape[0],1)\n",
    "# # act_labels = to_categorical(act_labels, num_classes=len(act_list))\n",
    "# y_train = to_categorical(y_train, num_classes=NUM_CLASSES)\n",
    "# y_test = to_categorical(y_test, num_classes=NUM_CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc5fcae5-5052-4916-9f46-d5fb3d001526",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4665d9a1-3e17-43b5-b2d9-eda196cf8926",
   "metadata": {},
   "outputs": [],
   "source": [
    "class HARModel(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_sensor_channels=113, len_seq=24, n_hidden=128, n_layers=1, n_filters=64, \n",
    "                 n_classes=5, filter_size=(1,5), drop_prob=0.5):\n",
    "        super(HARModel, self).__init__()\n",
    "        self.drop_prob = drop_prob\n",
    "        self.n_layers = n_layers\n",
    "        self.n_hidden = n_hidden\n",
    "        self.n_filters = n_filters\n",
    "        self.n_classes = n_classes\n",
    "        self.filter_size = filter_size\n",
    "        self.n_sensor_channels = n_sensor_channels\n",
    "        self.len_seq = len_seq\n",
    "\n",
    "             \n",
    "        self.conv1 = nn.Conv2d(1, n_filters, filter_size)\n",
    "        self.conv2 = nn.Conv2d(n_filters, n_filters, filter_size)\n",
    "        self.conv3 = nn.Conv2d(n_filters, n_filters, filter_size)\n",
    "        self.conv4 = nn.Conv2d(n_filters, n_filters, filter_size)\n",
    "        \n",
    "        # self.lstm1  = nn.LSTM(64, n_hidden, n_layers)\n",
    "        # self.lstm2  = nn.LSTM(n_hidden, n_hidden, n_layers)\n",
    "        self.multihead_attn = nn.MultiheadAttention(embed_dim=n_sensor_channels*n_filters, num_heads=1) # 7232=113*64\n",
    "        # self.fc0 = nn.Linear(57856, 128)\n",
    "        self.fc = nn.Linear(n_sensor_channels*n_filters*(len_seq-4*(filter_size[1]-1)), n_classes) #57856 = 8*113*64\n",
    "\n",
    "        self.dropout = nn.Dropout(drop_prob)\n",
    "        # self.softmax = nn.Softmax(dim=1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        # x = x.view(-1, NB_SENSOR_CHANNELS, SLIDING_WINDOW_LENGTH,1) # for direct channel_gate\n",
    "        # batch_size = x.shape[0]\n",
    "\n",
    "        # x = x.view(-1, NB_SENSOR_CHANNELS, SLIDING_WINDOW_LENGTH) # for deepconvlstm conv layers\n",
    "        # x = torch.permute(x,(1,0,2))\n",
    "        # print(x.shape)\n",
    "        # x, attn_output_weights = self.multihead_attn0(x,x,x)\n",
    "\n",
    "        # print(x.shape)\n",
    "        # x = torch.permute(x,(2,1,0))\n",
    "        # print(x.shape)\n",
    "        # x = x.view(-1, 1, NB_SENSOR_CHANNELS, SLIDING_WINDOW_LENGTH) # draft\n",
    "        x = torch.permute(x, (0,2,1))\n",
    "        x = torch.unsqueeze(x, dim=1)\n",
    "        # print(x.shape)\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.conv4(x)) # [64, 113, 8]\n",
    "        # x = x.view(-1, NB_SENSOR_CHANNELS, 8, 1)\n",
    "        # x = x.view(x.shape[0], x.shape[1], x.shape[2], 1)\n",
    "        # x = x.view(x.shape[0], -1, 8)\n",
    "        \n",
    "        # print(x.shape)\n",
    "        x = torch.permute(x, (3,0,1,2))\n",
    "        x = x.view(x.shape[0], x.shape[1],-1)\n",
    "        \n",
    "        # print(x.shape)\n",
    "        # x = x.view(8, x.shape[0], -1) # bak\n",
    "        \n",
    "    \n",
    "        x, attn_output_weights = self.multihead_attn(x,x,x)\n",
    "        x = self.dropout(x)\n",
    "        x = F.relu(x)\n",
    "#         x, attn_output_weights = self.multihead_attn1(x,x,x)\n",
    "#         # x = self.dropout(x)\n",
    "#         x = F.relu(x)    \n",
    "        \n",
    "        x = torch.permute(x, (1,0,2))\n",
    "        \n",
    "        # x, hidden = self.lstm1(x, hidden)\n",
    "        # # x = self.dropout(x)\n",
    "        # x, hidden = self.lstm2(x, hidden)\n",
    "        # x = self.dropout(x)\n",
    "        \n",
    "        # x = x.contiguous().view(-1, self.n_hidden)\n",
    "\n",
    "        x = torch.reshape(x, (x.shape[0],-1))\n",
    "        # x = F.relu(self.fc0(x))\n",
    "        # x = self.dropout(x)\n",
    "        x = self.fc(x)\n",
    "        \n",
    "        # out = x.view(batch_size, -1, self.n_classes)[:,-1,:]\n",
    "        return x\n",
    "    \n",
    "#     def init_hidden(self, batch_size):\n",
    "#         ''' Initializes hidden state '''\n",
    "#         # Create two new tensors with sizes n_layers x batch_size x n_hidden,\n",
    "#         # initialized to zero, for hidden state and cell state of LSTM\n",
    "#         weight = next(self.parameters()).data\n",
    "        \n",
    "#         if (train_on_gpu):\n",
    "#             hidden = (weight.new(self.n_layers, batch_size, self.n_hidden).zero_().cuda(),\n",
    "#                   weight.new(self.n_layers, batch_size, self.n_hidden).zero_().cuda())\n",
    "#         else:\n",
    "#             hidden = (weight.new(self.n_layers, batch_size, self.n_hidden).zero_(),\n",
    "#                       weight.new(self.n_layers, batch_size, self.n_hidden).zero_())\n",
    "        \n",
    "#         return hidden\n",
    "    \n",
    "net = HARModel(n_sensor_channels=X_train.shape[2], len_seq=X_train.shape[1], n_classes=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2677b12a-ecf1-4e48-aafa-2a5fab0a25f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def init_weights(m):\n",
    "#     if type(m) == nn.LSTM:\n",
    "#         for name, param in m.named_parameters():\n",
    "#             if 'weight_ih' in name:\n",
    "#                 torch.nn.init.orthogonal_(param.data)\n",
    "#             elif 'weight_hh' in name:\n",
    "#                 torch.nn.init.orthogonal_(param.data)\n",
    "#             elif 'bias' in name:\n",
    "#                 param.data.fill_(0)\n",
    "#     elif type(m) == nn.Conv1d or type(m) == nn.Linear:\n",
    "#         torch.nn.init.orthogonal_(m.weight)\n",
    "#         m.bias.data.fill_(0)\n",
    "# net.apply(init_weights)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5231c773-8a32-485f-8309-1d9d5a54e3f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def iterate_minibatches(inputs, targets, batchsize, shuffle=True):\n",
    "#     assert len(inputs) == len(targets)\n",
    "#     if shuffle:\n",
    "#         indices = np.arange(len(inputs))\n",
    "#         np.random.shuffle(indices)\n",
    "#     for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):\n",
    "#         if shuffle:\n",
    "#             excerpt = indices[start_idx:start_idx + batchsize]\n",
    "#         else:\n",
    "#             excerpt = slice(start_idx, start_idx + batchsize)\n",
    "#         yield inputs[excerpt], targets[excerpt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9fcf4b5b-2e10-43fc-8e4e-1cb59c708251",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on GPU!\n",
      "Epoch: 1/10... Train Loss: 0.7282... Val Loss: 0.2738... Val Acc: 0.9202... F1-Score: 0.9198...\n",
      "Epoch: 2/10... Train Loss: 0.1473... Val Loss: 0.1114... Val Acc: 0.9664... F1-Score: 0.9664...\n",
      "Epoch: 3/10... Train Loss: 0.0661... Val Loss: 0.0433... Val Acc: 0.9876... F1-Score: 0.9877...\n",
      "Epoch: 4/10... Train Loss: 0.0425... Val Loss: 0.0295... Val Acc: 0.9920... F1-Score: 0.9920...\n",
      "Epoch: 5/10... Train Loss: 0.0306... Val Loss: 0.0216... Val Acc: 0.9940... F1-Score: 0.9940...\n",
      "Epoch: 6/10... Train Loss: 0.0240... Val Loss: 0.0175... Val Acc: 0.9947... F1-Score: 0.9947...\n",
      "Epoch: 7/10... Train Loss: 0.0218... Val Loss: 0.0211... Val Acc: 0.9940... F1-Score: 0.9940...\n",
      "Epoch: 8/10... Train Loss: 0.0154... Val Loss: 0.0115... Val Acc: 0.9964... F1-Score: 0.9964...\n",
      "Epoch: 9/10... Train Loss: 0.0133... Val Loss: 0.0103... Val Acc: 0.9974... F1-Score: 0.9974...\n",
      "Epoch: 10/10... Train Loss: 0.0116... Val Loss: 0.0252... Val Acc: 0.9929... F1-Score: 0.9929...\n"
     ]
    }
   ],
   "source": [
    "def train(net, epochs=10, batch_size=64, lr=0.01):\n",
    "    # opt = torch.optim.Adam(net.parameters(), lr=lr)\n",
    "    opt = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9, weight_decay=1e-4)\n",
    "    # opt = torch.optim.RMSprop(net.parameters(), lr=lr, momentum=0.1)\n",
    "    # opt = torch.optim.SGD(net.parameters(), lr=lr)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    # criterion = nn.BCEWithLogitsLoss()\n",
    "    \n",
    "    train_dataset = TensorDataset(torch.from_numpy(X_train), torch.from_numpy(y_train))\n",
    "    train_loader = torch.utils.data.DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, drop_last = True)  \n",
    "\n",
    "    test_dataset = TensorDataset(torch.from_numpy(X_test), torch.from_numpy(y_test))\n",
    "    test_loader = torch.utils.data.DataLoader(test_dataset,\n",
    "        batch_size=batch_size, shuffle=False, drop_last = True) \n",
    "    \n",
    "    if(train_on_gpu):\n",
    "        net.cuda()\n",
    "     \n",
    "    for e in range(epochs):\n",
    "        \n",
    "        # initialize hidden state\n",
    "        # h = net.init_hidden(batch_size)         \n",
    "        train_losses = []    \n",
    "        net.train()\n",
    "        # for batch in iterate_minibatches(X_train, y_train, batch_size):\n",
    "        for batch in train_loader:\n",
    "            x, y = batch\n",
    "\n",
    "            # inputs, targets = torch.from_numpy(x), torch.from_numpy(y)\n",
    "            inputs, targets = x.to(device), y.to(device)  \n",
    "\n",
    "            # if(train_on_gpu):\n",
    "            #         inputs, targets = inputs.cuda(), targets.cuda()\n",
    "\n",
    "            # Creating new variables for the hidden state, otherwise\n",
    "            # we'd backprop through the entire training history\n",
    "            # h = tuple([each.data for each in h])\n",
    "            \n",
    "            # h = h[0].reshape((batch_size, -1)) # for GRU\n",
    "            \n",
    "            # zero accumulated gradients\n",
    "            opt.zero_grad()   \n",
    "            \n",
    "            # get the output from the model\n",
    "            output = net(inputs)\n",
    "            # loss = criterion(output, torch.from_numpy(to_categorical(y, num_classes=NUM_CLASSES)).to(device))\n",
    "            loss = criterion(output, torch.argmax(targets,dim=1))\n",
    "            # print(output.shape)\n",
    "            # print(targets.shape)\n",
    "            # loss = criterion(output, targets)\n",
    "            train_losses.append(loss.item())\n",
    "            loss.backward()\n",
    "            opt.step()\n",
    "            \n",
    "        # val_h = net.init_hidden(batch_size)\n",
    "        val_losses = []\n",
    "        accuracy=0\n",
    "        f1score=0\n",
    "        \n",
    "        correct = 0\n",
    "        total = 0\n",
    "        total_true = []\n",
    "        total_pred = []\n",
    "        \n",
    "        net.eval()\n",
    "        with torch.no_grad():\n",
    "            for batch in test_loader:\n",
    "                x, y = batch\n",
    "                inputs, targets = x.to(device), y.to(device)  \n",
    " \n",
    "                # print(images.shape)            \n",
    "            # for batch in iterate_minibatches(X_test, y_test, batch_size):\n",
    "            #     x, y = batch     \n",
    "\n",
    "                # inputs, targets = torch.from_numpy(x), torch.from_numpy(y)\n",
    "\n",
    "                # val_h = tuple([each.data for each in val_h])\n",
    "\n",
    "                if(train_on_gpu):\n",
    "                    inputs, targets = inputs.cuda(), targets.cuda()\n",
    "                    \n",
    "                output = net(inputs)\n",
    "\n",
    "                # val_loss = criterion(output, torch.from_numpy(to_categorical(y, num_classes=NUM_CLASSES)).to(device))\n",
    "                val_loss = criterion(output, torch.argmax(targets,dim=1))\n",
    "                # val_loss = criterion(output, targets)\n",
    "                val_losses.append(val_loss.item())\n",
    "                \n",
    "                predicted = torch.argmax(output.data, dim=1)\n",
    "                total += targets.size(0)\n",
    "                correct += (predicted == torch.argmax(targets, dim=1)).sum().item()\n",
    "\n",
    "                total_pred = total_pred + predicted.cpu().numpy().tolist()\n",
    "                total_true = total_true + (torch.argmax(targets, dim=1).cpu().numpy().tolist())\n",
    "\n",
    "\n",
    "#                 top_p, top_class = output.topk(1, dim=1)\n",
    "                \n",
    "#                 # equals = top_class == torch.argmax(targets, dim=1)\n",
    "#                 equals = top_class == targets.view(*top_class.shape).long()\n",
    "#                 accuracy += torch.mean(equals.type(torch.FloatTensor))\n",
    "#                 # f1score += metrics.f1_score(top_class.cpu(), torch.argmax(targets, dim=1).cpu(), average='micro')\n",
    "#                 f1score += metrics.f1_score(top_class.cpu(), targets.view(*top_class.shape).long().cpu(), average='micro')\n",
    "        net.train() # reset to train mode after iterationg through validation data\n",
    "    \n",
    "        # print(f'Test Accuracy: {100.0 * correct / total} %')\n",
    "        # print(\" | \".join(act_labels_txt))\n",
    "        # conf_mat = confusion_matrix(y_true = total_true, y_pred = total_pred)\n",
    "        # conf_mat = conf_mat.astype('float') / conf_mat.sum(axis=1)[:, np.newaxis]\n",
    "        # print(np.array(conf_mat).round(3) * 100)  \n",
    "        f1_score = metrics.f1_score(y_true = total_true, y_pred = total_pred, average='weighted')\n",
    "        # print('F1 score:', f1_score)\n",
    "        # print('')      \n",
    "\n",
    "        print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
    "        \"Train Loss: {:.4f}...\".format(np.mean(train_losses)),\n",
    "        \"Val Loss: {:.4f}...\".format(np.mean(val_losses)),\n",
    "        \"Val Acc: {:.4f}...\".format(correct / total),\n",
    "        \"F1-Score: {:.4f}...\".format(f1_score))\n",
    "        \n",
    "        PATH = 'pamap2_ConvAttn_ep'+str(e)+'.pt'\n",
    "        torch.save(net.state_dict(), PATH)\n",
    "        \n",
    "## check if GPU is available\n",
    "train_on_gpu = torch.cuda.is_available()\n",
    "if(train_on_gpu):\n",
    "    print('Training on GPU!')\n",
    "else: \n",
    "    print('No GPU available, training on CPU; consider making n_epochs very small.')\n",
    "\n",
    "train(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7eeccfaf-7e5f-49e4-881a-2bb2e68002eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# PATH = 'opportunity_ConvAttn.pt'\n",
    "# torch.save(net.state_dict(), PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e461cc9b-e4fc-457e-93e6-4b8c426337ad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a22b682f-33db-45d8-a57c-15d4dc0ed939",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}