GNN_Event_Based_withGraph.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GNN_Event_Based_withGraph.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anindya2001/GraphSemanticSegmentation/blob/main/GNN_Event_Based_withGraph.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lJHsLlx0C-4F",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2a34028a-9c10-450d-b42e-8765eff83dcc"
      },
      "source": [
        "pip install torch torchvision"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.8.0+cu101)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.9.0+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.0.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a_SydYQiC_03"
      },
      "source": [
        "import math\n",
        "\n",
        "import torch\n",
        "\n",
        "from torch.nn.parameter import Parameter\n",
        "from torch.nn.modules.module import Module\n",
        "\n",
        "class GraphConvolution(Module):\n",
        "\n",
        "    def __init__(self, in_features, out_features, bias=True):\n",
        "        super(GraphConvolution, self).__init__()\n",
        "        self.in_features = in_features\n",
        "        self.out_features = out_features\n",
        "        self.weight = Parameter(torch.FloatTensor(in_features, out_features))\n",
        "        if bias:\n",
        "            self.bias = Parameter(torch.FloatTensor(out_features))\n",
        "        else:\n",
        "            self.register_parameter('bias', None)\n",
        "        self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        stdv = 1. / math.sqrt(self.weight.size(1))\n",
        "        self.weight.data.uniform_(-stdv, stdv)\n",
        "        if self.bias is not None:\n",
        "            self.bias.data.uniform_(-stdv, stdv)\n",
        "\n",
        "    def forward(self, input, adj):\n",
        "        support = torch.mm(input, self.weight)\n",
        "        output = torch.spmm(adj, support)\n",
        "        if self.bias is not None:\n",
        "            return output + self.bias\n",
        "        else:\n",
        "            return output\n",
        "\n",
        "    def __repr__(self):\n",
        "        return self.__class__.__name__ + ' (' \\\n",
        "               + str(self.in_features) + ' -> ' \\\n",
        "               + str(self.out_features) + ')'\n",
        "\n",
        "\n",
        "class GraphPreConvolution(Module):\n",
        "\n",
        "    def __init__(self, output_dimension, number_nodes, number_classes, bias=True):\n",
        "        super(GraphPreConvolution, self).__init__()\n",
        "        self.output_dimension = output_dimension\n",
        "        self.number_nodes = number_nodes\n",
        "        self.weight = Parameter(torch.FloatTensor(output_dimension, number_nodes))\n",
        "        if bias:\n",
        "            self.bias = Parameter(torch.FloatTensor(number_classes))\n",
        "        else:\n",
        "            self.register_parameter('bias', None)\n",
        "        self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        stdv = 1. / math.sqrt(self.weight.size(1))\n",
        "        self.weight.data.uniform_(-stdv, stdv)\n",
        "        if self.bias is not None:\n",
        "            self.bias.data.uniform_(-stdv, stdv)\n",
        "\n",
        "    def forward(self, input):\n",
        "        output = torch.mm(self.weight, input)\n",
        "        if self.bias is not None:\n",
        "            return output + self.bias\n",
        "        else:\n",
        "            return output\n",
        "\n",
        "    def __repr__(self):\n",
        "        return self.__class__.__name__ + ' (' \\\n",
        "               + str(self.in_features) + ' -> ' \\\n",
        "               + str(self.out_features) + ')'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yr7jjaBMvy3A"
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "class GCN(nn.Module):\n",
        "    def __init__(self, nfeat, nhid, nclass, dropout, number_nodes, width_times_heigth):\n",
        "        super(GCN, self).__init__()\n",
        "\n",
        "        self.gc1 = GraphConvolution(nfeat, nhid)\n",
        "        self.gc2 = GraphConvolution(nhid, nclass)\n",
        "        self.gpc = GraphPreConvolution(width_times_heigth, number_nodes, nclass)\n",
        "        self.dropout = dropout\n",
        "\n",
        "    def forward(self, x, adj):\n",
        "        x = F.relu(self.gc1(x, adj))\n",
        "        x = F.dropout(x, self.dropout, training=self.training)\n",
        "        x = self.gc2(x, adj)\n",
        "        x = self.gpc(x)\n",
        "        #Maybe try removing the following comment\n",
        "        #x = F.dropout(x, self.dropout, training=self.training)\n",
        "        return F.log_softmax(x, dim=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mYQIQOGUxnRd"
      },
      "source": [
        "from __future__ import division\n",
        "from __future__ import print_function\n",
        "\n",
        "import time\n",
        "import argparse\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "from scipy.io import loadmat, savemat\n",
        "\n",
        "##### Training settings\n",
        "parser = argparse.ArgumentParser()\n",
        "parser.add_argument('--no-cuda', action='store_true', default=False,\n",
        "                    help='Disables CUDA training.')\n",
        "parser.add_argument('--fastmode', action='store_true', default=True,\n",
        "                    help='Validate during training pass.')\n",
        "parser.add_argument('--seed', type=int, default=42, help='Random seed.')\n",
        "##### Maybe you need to modify the number of epochs\n",
        "parser.add_argument('--epochs', type=int, default=200,\n",
        "                    help='Number of epochs to train.')\n",
        "##### Most likely, you will need to change the learning rate\n",
        "parser.add_argument('--lr', type=float, default=0.01,\n",
        "                    help='Initial learning rate.')\n",
        "parser.add_argument('--weight_decay', type=float, default=5e-4,\n",
        "                    help='Weight decay (L2 loss on parameters).')\n",
        "##### Maybe, we need also to change the number of hidden units\n",
        "parser.add_argument('--hidden', type=int, default=16,\n",
        "                    help='Number of hidden units.')\n",
        "parser.add_argument('--dropout', type=float, default=0.5,\n",
        "                    help='Dropout rate (1 - keep probability).')\n",
        "\n",
        "args = parser.parse_args()\n",
        "args.cuda = not args.no_cuda and torch.cuda.is_available()\n",
        "\n",
        "np.random.seed(args.seed)\n",
        "torch.manual_seed(args.seed)\n",
        "if args.cuda:\n",
        "  torch.cuda.manual_seed(args.seed)\n",
        "\n",
        "##### Here you need to configure the following variables:\n",
        "##### nfeat=1 this is the second dimension of the vector of polarity, i.e., 1\n",
        "##### nclass= _____ you need to put the number of semantic labels we are classifying\n",
        "##### number_nodes= _____ you need to put here the number of nodes of the adjacency matrices, I think you called it M in your code\n",
        "##### width_times_height= _____ this is the multplication between the width and the heigth of the event-based data, i.e., the spatial dimensions of the camera.\n",
        "\n",
        "model = GCN(nfeat=1,\n",
        "            nhid=args.hidden,\n",
        "            nclass= ____, #complete this\n",
        "            dropout=args.dropout,\n",
        "            number_nodes=_____, #complete this\n",
        "            width_times_heigth=_____) #complete this\n",
        "\n",
        "##### Optimizer, we are gonna use Adam. Notice we are using l2 regularization in all layers\n",
        "optimizer = optim.Adam(model.parameters(),\n",
        "                       lr=args.lr, weight_decay=args.weight_decay)\n",
        "\n",
        "if args.cuda:\n",
        "  model.cuda()\n",
        "\n",
        "#for i in set_of_tensors_in_training_set:\n",
        "  ###############\n",
        "  ##### In this block of code, you need to put your code for the construction of the graph for each tensor i in the TRAINING set\n",
        "  ###############\n",
        "  #adj_\n",
        "\n",
        "for epoch in range(args.epochs):\n",
        "  print('Epoch ', epoch + 1)\n",
        "  t = time.time()\n",
        "  ##### The following for loop iterates in the TRAINING set of our semantic segmentation dataset\n",
        "  ##### Please modify it accordingly\n",
        "  for i in set_of_tensors_in_training_set:\n",
        "    ###############\n",
        "    ##### In this block of code, you need to put your code for the construction of the graph for each tensor i in the TRAINING set\n",
        "    # adj_matrix = load(output of anindya and shashant algorithm)\n",
        "    ###############\n",
        "    A = torch.FloatTensor(adj_matrix) #here you have the adjacency matrix\n",
        "    tensor_ones = torch.ones(A.shape[0], 1)\n",
        "    A_tilde_1 = A + torch.eye(A.shape[0])\n",
        "    ##### The following is the filtering operation\n",
        "    diag_elements_1 = torch.mm(A_tilde_1, tensor_ones)\n",
        "    diag_elements_1 = torch.div(tensor_ones, torch.sqrt(diag_elements_1))\n",
        "    D_tilde_1 = torch.diag(diag_elements_1.view(-1))\n",
        "    A_tilde_1 = torch.spmm(D_tilde_1, torch.spmm(A_tilde_1, D_tilde_1))\n",
        "    adj_1 = A_tilde_1\n",
        "    ##### The vector 'features' contain the vector of polarities, the dimension should be N times 1,\n",
        "    ##### where N is the number of nodes, I think you called it M in your code.\n",
        "    #features = put here the vector of polarities\n",
        "    features = torch.FloatTensor(features)\n",
        "    ##### The variable 'labels' should contain the matrix (width*heigth times number_clases), i.e., you need to vectorize the ground truth of tensor i.\n",
        "    ##### Please modify the following line accordingly\n",
        "    ##### Remember this is the ground truth corresponding to the tensor i, the current iteration\n",
        "    #labels = torch.LongTensor(put here the matrix with the appropiate dimensions)\n",
        "    if args.cuda:\n",
        "      features = features.cuda()\n",
        "      adj_1 = adj_1.cuda()\n",
        "      labels = labels.cuda()\n",
        "    ##### The following lines, we have the actual training procedure\n",
        "    model.train() # We do this here, and not outside the FOR LOOP, because later we will include a VALIDATION set to see if we are overfitting\n",
        "    optimizer.zero_grad()\n",
        "    output = model(features, adj_1)\n",
        "    loss_train = F.nll_loss(output, labels)\n",
        "    loss_train.backward()\n",
        "    optimizer.step()\n",
        "    ##### In the following line we are gonna see the training loss, we need to do this to be sure our network is not diverging.\n",
        "    print(\"Training results:\",\n",
        "          \"loss= {:.4f}\".format(loss_train.item()))\n",
        "  ##### Here we are gonna see the time each epoch requires\n",
        "  print(\"Total time elapsed per epoch: {:.4f}s\".format(time.time() - t))\n",
        "\n",
        "##### Once the network is trained, we need to test our trained model in the tensors of the TEST set.\n",
        "##### To do that you need to put your network in evaluation mode with the following line of code.\n",
        "model.eval()\n",
        "##### You need to make a FOR LOOP in the TEST set and do a similar loop as before, but let's try first to get the training loop.\n",
        "##### Perhaps, we will need to use a part of the TRAINING set as VALIDATION to check overfitting. We can do that later."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nm4og3LBsWmd"
      },
      "source": [
        "from PIL import Image as im\n",
        "import os\n",
        "import cv2\n",
        "import math\n",
        "import scipy.io\n",
        "import random\n",
        "import networkx\n",
        "import h5py\n",
        "import numpy as np\n",
        "\n",
        "def label_data(events):\n",
        "  object_to_color = {\n",
        "    \"background\"  : (0, 0, 0),\n",
        "    \"person\"      : (192, 128, 128),\n",
        "    \"car\"         : (128, 128, 128)\n",
        "  }\n",
        "  color_to_object = { v : k for k, v in object_to_color.items()}\n",
        "  GT = []\n",
        "  for event in events:\n",
        "    y, x, t, p = event\n",
        "    if color_to_object[tuple(I[int(x)][int(y)])] == \"person\":\n",
        "      GT.append(list((0, 0, 1)))\n",
        "      continue\n",
        "    elif color_to_object[tuple(I[int(x)][int(y)])] == \"car\":\n",
        "      GT.append(list((0, 1, 0)))\n",
        "      continue\n",
        "    GT.append(list((1, 0, 0)))\n",
        "  return np.array(GT)\n",
        "\n",
        "def isWithin(event1, event2, radius, alpha=1, beta=0.000005):\n",
        "    y_i, x_i, t_i, p_i = event1\n",
        "    y_j, x_j, t_j, p_j = event2\n",
        "    t_i = t_i/10000\n",
        "    t_j = t_j/10000\n",
        "    spatial = (x_i - x_j)*(x_i - x_j) + (y_i - y_j)*(y_i - y_j)\n",
        "    temporal = (t_i - t_j)*(t_i - t_j)\n",
        "    if (alpha * spatial) + (beta * temporal * 100000000) >= 0:\n",
        "      #print(math.sqrt(alpha * spatial + beta * temporal))\n",
        "      return math.sqrt(alpha * spatial + beta * temporal) <= radius\n",
        "\n",
        "def visualize_graph(adMat):\n",
        "  M = len(adMat)\n",
        "  labeldict = {}\n",
        "  for i in range(M):\n",
        "    labeldict[i] = str(i) + \"+\" if events[i][3] == 1 else str(i) +\"-\"\n",
        "    \n",
        "  G = networkx.from_numpy_matrix(np.array(adMat))\n",
        "\n",
        "  color_map = []\n",
        "  for node in G:\n",
        "    if events[node][3] == -1:\n",
        "      color_map.append(\"red\")\n",
        "    else:\n",
        "      color_map.append(\"green\")\n",
        "  networkx.draw(G,node_color=color_map, labels = labeldict, with_labels=True)\n",
        "\n",
        "def construct_graph(events, R = 100):\n",
        "  M = len(events)\n",
        "  adMat = np.zeros([M, M], dtype=int)\n",
        "  for row in range(M):\n",
        "      for col in range(M):\n",
        "        if row == col:\n",
        "          continue \n",
        "        if isWithin(events[row], events[col], R):\n",
        "          adMat[row][col] = 1\n",
        "          adMat[col][row] = 1\n",
        "  return adMat\n",
        "\n",
        "def polar_events(events):\n",
        "  return events[:,-1]\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}