irm_feature_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EPIC(Dataset):\n",
    "    def __init__(self, data_path, p_id=None, df=None, train_batch_size=None):\n",
    "        super().__init__()\n",
    "        if p_id is not None:\n",
    "            idd = df.index[df['participant_id'] == p_id].tolist()\n",
    "            if len(idd)< train_batch_size:\n",
    "                self.hasData = False\n",
    "            else:\n",
    "                self.hasData = True\n",
    "            if 0 in idd:\n",
    "                idd.remove(0)\n",
    "            data = np.load(data_path)\n",
    "            self.verb = data[idd, -2]\n",
    "            self.noun = data[idd, -1]\n",
    "            self.data = data[idd, :-2]\n",
    "        else:\n",
    "            data = np.load(data_path)\n",
    "            self.verb = data[1:, -2]\n",
    "            self.noun = data[1:, -1]\n",
    "            self.data = data[1:, :-2]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return np.shape(self.data)[0]\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        verb = np.squeeze(self.verb[idx])\n",
    "        noun = np.squeeze(self.noun[idx])\n",
    "\n",
    "        return {\n",
    "                \"data\":     torch.tensor(np.expand_dims(np.squeeze(self.data[idx,:]), 0)).float(), \n",
    "                \"verb_id\":  torch.tensor(np.expand_dims(verb, 1)).long(), \n",
    "                \"noun_id\":  torch.tensor(np.expand_dims(noun, 1)).long()\n",
    "                }\n",
    "\n",
    "class IRM(torch.nn.Module):\n",
    "    def __init__(self, num_class_verb, num_class_noun):\n",
    "        super().__init__()\n",
    "        self.verb_fc = torch.nn.Linear(2048, num_class_verb)\n",
    "        self.noun_fc = torch.nn.Linear(2048, num_class_noun)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return {\"verb_out\": self.verb_fc(x), \"noun_out\": self.noun_fc(x)}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": [
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "al Accuracy/Verb(Top 1) 0.28365384615384615\nVal Accuracy/Noun(Top 1) 0.11365384615384615\nVal Accuracy/Action(Top 1) 0.026153846153846153\nEpoch : 40\n0.1473712130290706\n0.07448281260525308\nTrain Accuracy/Verb(Top 1) 0.38073913043478264\nTrain Accuracy/Noun(Top 1) 0.12578260869565216\nTrain Accuracy/Action(Top 1) 0.0388695652173913\nVal Accuracy/Verb(Top 1) 0.28442307692307695\nVal Accuracy/Noun(Top 1) 0.11384615384615385\nVal Accuracy/Action(Top 1) 0.02576923076923077\nEpoch : 41\n0.1466542488191186\n0.07384980379081355\nTrain Accuracy/Verb(Top 1) 0.38130434782608696\nTrain Accuracy/Noun(Top 1) 0.12565217391304348\nTrain Accuracy/Action(Top 1) 0.0388695652173913\nVal Accuracy/Verb(Top 1) 0.28365384615384615\nVal Accuracy/Noun(Top 1) 0.11403846153846153\nVal Accuracy/Action(Top 1) 0.025961538461538463\nEpoch : 42\n0.1462756841647916\n0.07375104492757378\nTrain Accuracy/Verb(Top 1) 0.382\nTrain Accuracy/Noun(Top 1) 0.1254782608695652\nTrain Accuracy/Action(Top 1) 0.03904347826086957\nVal Accuracy/Verb(Top 1) 0.2840384615384615\nVal Accuracy/Noun(Top 1) 0.11365384615384615\nVal Accuracy/Action(Top 1) 0.025961538461538463\nEpoch : 43\n0.14569466811854664\n0.07339004837885135\nTrain Accuracy/Verb(Top 1) 0.3824782608695652\nTrain Accuracy/Noun(Top 1) 0.12543478260869564\nTrain Accuracy/Action(Top 1) 0.03891304347826087\nVal Accuracy/Verb(Top 1) 0.2851923076923077\nVal Accuracy/Noun(Top 1) 0.11403846153846153\nVal Accuracy/Action(Top 1) 0.025961538461538463\nEpoch : 44\n0.14577749298840034\n0.07321242304836831\nTrain Accuracy/Verb(Top 1) 0.3835217391304348\nTrain Accuracy/Noun(Top 1) 0.1254782608695652\nTrain Accuracy/Action(Top 1) 0.03904347826086957\nVal Accuracy/Verb(Top 1) 0.2857692307692308\nVal Accuracy/Noun(Top 1) 0.11365384615384615\nVal Accuracy/Action(Top 1) 0.026153846153846153\nEpoch : 45\n0.14536395291002785\n0.0730594013522311\nTrain Accuracy/Verb(Top 1) 0.38369565217391305\nTrain Accuracy/Noun(Top 1) 0.12560869565217392\nTrain Accuracy/Action(Top 1) 0.03908695652173913\nVal Accuracy/Verb(Top 1) 0.2857692307692308\nVal Accuracy/Noun(Top 1) 0.11423076923076923\nVal Accuracy/Action(Top 1) 0.026346153846153846\nEpoch : 46\n0.14500608153459502\n0.07267761753826607\nTrain Accuracy/Verb(Top 1) 0.3836086956521739\nTrain Accuracy/Noun(Top 1) 0.1255217391304348\nTrain Accuracy/Action(Top 1) 0.0391304347826087\nVal Accuracy/Verb(Top 1) 0.2878846153846154\nVal Accuracy/Noun(Top 1) 0.11384615384615385\nVal Accuracy/Action(Top 1) 0.02673076923076923\nEpoch : 47\n0.14521439060932254\n0.07258874694021737\nTrain Accuracy/Verb(Top 1) 0.3849130434782609\nTrain Accuracy/Noun(Top 1) 0.1257391304347826\nTrain Accuracy/Action(Top 1) 0.03939130434782609\nVal Accuracy/Verb(Top 1) 0.28615384615384615\nVal Accuracy/Noun(Top 1) 0.11346153846153846\nVal Accuracy/Action(Top 1) 0.026346153846153846\nEpoch : 48\n0.14468826622497744\n0.0721967282818585\nTrain Accuracy/Verb(Top 1) 0.38417391304347825\nTrain Accuracy/Noun(Top 1) 0.12569565217391304\nTrain Accuracy/Action(Top 1) 0.03930434782608696\nVal Accuracy/Verb(Top 1) 0.28423076923076923\nVal Accuracy/Noun(Top 1) 0.11288461538461539\nVal Accuracy/Action(Top 1) 0.026153846153846153\nEpoch : 49\n0.14445839509731387\n0.07227570705297516\nTrain Accuracy/Verb(Top 1) 0.38465217391304346\nTrain Accuracy/Noun(Top 1) 0.12569565217391304\nTrain Accuracy/Action(Top 1) 0.03917391304347826\nVal Accuracy/Verb(Top 1) 0.2871153846153846\nVal Accuracy/Noun(Top 1) 0.11288461538461539\nVal Accuracy/Action(Top 1) 0.025961538461538463\nEpoch : 50\n0.1444624265519584\n0.07189857487271471\nTrain Accuracy/Verb(Top 1) 0.3853913043478261\nTrain Accuracy/Noun(Top 1) 0.1257391304347826\nTrain Accuracy/Action(Top 1) 0.03943478260869565\nVal Accuracy/Verb(Top 1) 0.28692307692307695\nVal Accuracy/Noun(Top 1) 0.11307692307692307\nVal Accuracy/Action(Top 1) 0.02653846153846154\nEpoch : 51\n0.14385938877012672\n0.07166035611455034\nTrain Accuracy/Verb(Top 1) 0.38604347826086954\nTrain Accuracy/Noun(Top 1) 0.1257391304347826\nTrain Accuracy/Action(Top 1) 0.03939130434782609\nVal Accuracy/Verb(Top 1) 0.28673076923076923\nVal Accuracy/Noun(Top 1) 0.11288461538461539\nVal Accuracy/Action(Top 1) 0.02653846153846154\nEpoch : 52\n0.1439784214264009\n0.07166261723855646\nTrain Accuracy/Verb(Top 1) 0.3492173913043478\nTrain Accuracy/Noun(Top 1) 0.1255217391304348\nTrain Accuracy/Action(Top 1) 0.03256521739130435\nVal Accuracy/Verb(Top 1) 0.2569230769230769\nVal Accuracy/Noun(Top 1) 0.11326923076923077\nVal Accuracy/Action(Top 1) 0.02\nEpoch : 53\n0.14639001401459298\n0.07163263217705051\nTrain Accuracy/Verb(Top 1) 0.3446521739130435\nTrain Accuracy/Noun(Top 1) 0.1255217391304348\nTrain Accuracy/Action(Top 1) 0.03108695652173913\nVal Accuracy/Verb(Top 1) 0.255\nVal Accuracy/Noun(Top 1) 0.11346153846153846\nVal Accuracy/Action(Top 1) 0.019807692307692307\nEpoch : 54\n0.1458144833401936\n0.07145597207836989\nTrain Accuracy/Verb(Top 1) 0.3467826086956522\nTrain Accuracy/Noun(Top 1) 0.1254782608695652\nTrain Accuracy/Action(Top 1) 0.03160869565217391\nVal Accuracy/Verb(Top 1) 0.25403846153846155\nVal Accuracy/Noun(Top 1) 0.11346153846153846\nVal Accuracy/Action(Top 1) 0.019615384615384614\nEpoch : 55\n0.145825172197528\n0.07151030883556458\nTrain Accuracy/Verb(Top 1) 0.34995652173913044\nTrain Accuracy/Noun(Top 1) 0.12534782608695652\nTrain Accuracy/Action(Top 1) 0.03208695652173913\nVal Accuracy/Verb(Top 1) 0.25384615384615383\nVal Accuracy/Noun(Top 1) 0.11326923076923077\nVal Accuracy/Action(Top 1) 0.01942307692307692\nEpoch : 56\n0.14523079816888018\n0.07146269315626563\nTrain Accuracy/Verb(Top 1) 0.3527826086956522\nTrain Accuracy/Noun(Top 1) 0.12517391304347827\nTrain Accuracy/Action(Top 1) 0.03308695652173913\nVal Accuracy/Verb(Top 1) 0.25653846153846155\nVal Accuracy/Noun(Top 1) 0.11326923076923077\nVal Accuracy/Action(Top 1) 0.019807692307692307\nEpoch : 57\n0.14492593131414275\n0.07130256367892754\nTrain Accuracy/Verb(Top 1) 0.3538695652173913\nTrain Accuracy/Noun(Top 1) 0.12495652173913044\nTrain Accuracy/Action(Top 1) 0.03334782608695652\nVal Accuracy/Verb(Top 1) 0.25903846153846155\nVal Accuracy/Noun(Top 1) 0.11307692307692307\nVal Accuracy/Action(Top 1) 0.02\nEpoch : 58\n0.14468884104635657\n0.07120212134791584\nTrain Accuracy/Verb(Top 1) 0.3551739130434783\nTrain Accuracy/Noun(Top 1) 0.12491304347826086\nTrain Accuracy/Action(Top 1) 0.03373913043478261\nVal Accuracy/Verb(Top 1) 0.25884615384615384\nVal Accuracy/Noun(Top 1) 0.11288461538461539\nVal Accuracy/Action(Top 1) 0.02\nEpoch : 59\n0.14391813190971933\n0.07115227034906062\nTrain Accuracy/Verb(Top 1) 0.3567391304347826\nTrain Accuracy/Noun(Top 1) 0.12465217391304348\nTrain Accuracy/Action(Top 1) 0.03373913043478261\nVal Accuracy/Verb(Top 1) 0.26076923076923075\nVal Accuracy/Noun(Top 1) 0.11307692307692307\nVal Accuracy/Action(Top 1) 0.02076923076923077\nEpoch : 60\n0.14363430011563186\n0.07130804323568576\nTrain Accuracy/Verb(Top 1) 0.35843478260869566\nTrain Accuracy/Noun(Top 1) 0.12434782608695652\nTrain Accuracy/Action(Top 1) 0.033869565217391304\nVal Accuracy/Verb(Top 1) 0.26211538461538464\nVal Accuracy/Noun(Top 1) 0.11326923076923077\nVal Accuracy/Action(Top 1) 0.02076923076923077\nEpoch : 61\n0.143330195764216\n0.07094598267136551\nTrain Accuracy/Verb(Top 1) 0.35995652173913045\nTrain Accuracy/Noun(Top 1) 0.12430434782608696\nTrain Accuracy/Action(Top 1) 0.03426086956521739\nVal Accuracy/Verb(Top 1) 0.26269230769230767\nVal Accuracy/Noun(Top 1) 0.11365384615384615\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 62\n0.14310220261899437\n0.07103616449891068\nTrain Accuracy/Verb(Top 1) 0.3601304347826087\nTrain Accuracy/Noun(Top 1) 0.1242608695652174\nTrain Accuracy/Action(Top 1) 0.03434782608695652\nVal Accuracy/Verb(Top 1) 0.2625\nVal Accuracy/Noun(Top 1) 0.11346153846153846\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 63\n0.14294713822806754\n0.07103949298218983\nTrain Accuracy/Verb(Top 1) 0.3610434782608696\nTrain Accuracy/Noun(Top 1) 0.12417391304347826\nTrain Accuracy/Action(Top 1) 0.034782608695652174\nVal Accuracy/Verb(Top 1) 0.2628846153846154\nVal Accuracy/Noun(Top 1) 0.11346153846153846\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 64\n0.14258522711149077\n0.07090466814797099\nTrain Accuracy/Verb(Top 1) 0.36334782608695654\nTrain Accuracy/Noun(Top 1) 0.12408695652173914\nTrain Accuracy/Action(Top 1) 0.035130434782608695\nVal Accuracy/Verb(Top 1) 0.2648076923076923\nVal Accuracy/Noun(Top 1) 0.11326923076923077\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 65\n0.1425174777100726\n0.07110214320624747\nTrain Accuracy/Verb(Top 1) 0.3640434782608696\nTrain Accuracy/Noun(Top 1) 0.12408695652173914\nTrain Accuracy/Action(Top 1) 0.035260869565217394\nVal Accuracy/Verb(Top 1) 0.2655769230769231\nVal Accuracy/Noun(Top 1) 0.11288461538461539\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 66\n0.14230212990830585\n0.0707089430675274\nTrain Accuracy/Verb(Top 1) 0.3642173913043478\nTrain Accuracy/Noun(Top 1) 0.12391304347826088\nTrain Accuracy/Action(Top 1) 0.035173913043478264\nVal Accuracy/Verb(Top 1) 0.265\nVal Accuracy/Noun(Top 1) 0.11269230769230769\nVal Accuracy/Action(Top 1) 0.02\nEpoch : 67\n0.14221138154588095\n0.07081596836811159\nTrain Accuracy/Verb(Top 1) 0.3644782608695652\nTrain Accuracy/Noun(Top 1) 0.1238695652173913\nTrain Accuracy/Action(Top 1) 0.035304347826086956\nVal Accuracy/Verb(Top 1) 0.26519230769230767\nVal Accuracy/Noun(Top 1) 0.11269230769230769\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 68\n0.14182772113055717\n0.07085270961610282\nTrain Accuracy/Verb(Top 1) 0.36517391304347824\nTrain Accuracy/Noun(Top 1) 0.12391304347826088\nTrain Accuracy/Action(Top 1) 0.035391304347826086\nVal Accuracy/Verb(Top 1) 0.26653846153846156\nVal Accuracy/Noun(Top 1) 0.1123076923076923\nVal Accuracy/Action(Top 1) 0.02\nEpoch : 69\n0.14179950661775542\n0.0706992861701221\nTrain Accuracy/Verb(Top 1) 0.36517391304347824\nTrain Accuracy/Noun(Top 1) 0.12395652173913044\nTrain Accuracy/Action(Top 1) 0.035434782608695654\nVal Accuracy/Verb(Top 1) 0.26634615384615384\nVal Accuracy/Noun(Top 1) 0.11211538461538462\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 70\n0.14198177965675912\n0.0705480975348775\nTrain Accuracy/Verb(Top 1) 0.36543478260869566\nTrain Accuracy/Noun(Top 1) 0.12378260869565218\nTrain Accuracy/Action(Top 1) 0.035565217391304346\nVal Accuracy/Verb(Top 1) 0.26576923076923076\nVal Accuracy/Noun(Top 1) 0.11192307692307692\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 71\n0.14188937777426186\n0.07058840844689346\nTrain Accuracy/Verb(Top 1) 0.3658695652173913\nTrain Accuracy/Noun(Top 1) 0.12373913043478262\nTrain Accuracy/Action(Top 1) 0.035608695652173915\nVal Accuracy/Verb(Top 1) 0.26653846153846156\nVal Accuracy/Noun(Top 1) 0.11173076923076923\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 72\n0.14160893152399762\n0.07054088166574153\nTrain Accuracy/Verb(Top 1) 0.3660434782608696\nTrain Accuracy/Noun(Top 1) 0.12369565217391304\nTrain Accuracy/Action(Top 1) 0.035565217391304346\nVal Accuracy/Verb(Top 1) 0.2667307692307692\nVal Accuracy/Noun(Top 1) 0.11173076923076923\nVal Accuracy/Action(Top 1) 0.020576923076923076\nEpoch : 73\n0.14129307546266695\n0.0703973741066165\nTrain Accuracy/Verb(Top 1) 0.36643478260869566\nTrain Accuracy/Noun(Top 1) 0.12373913043478262\nTrain Accuracy/Action(Top 1) 0.035608695652173915\nVal Accuracy/Verb(Top 1) 0.26692307692307693\nVal Accuracy/Noun(Top 1) 0.11134615384615384\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 74\n0.14137224293336637\n0.07026538289174801\nTrain Accuracy/Verb(Top 1) 0.3665217391304348\nTrain Accuracy/Noun(Top 1) 0.12365217391304348\nTrain Accuracy/Action(Top 1) 0.035391304347826086\nVal Accuracy/Verb(Top 1) 0.2671153846153846\nVal Accuracy/Noun(Top 1) 0.11096153846153846\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 75\n0.1412898632084451\n0.07042848802194362\nTrain Accuracy/Verb(Top 1) 0.3665652173913043\nTrain Accuracy/Noun(Top 1) 0.12352173913043478\nTrain Accuracy/Action(Top 1) 0.035434782608695654\nVal Accuracy/Verb(Top 1) 0.26596153846153847\nVal Accuracy/Noun(Top 1) 0.11096153846153846\nVal Accuracy/Action(Top 1) 0.02\nEpoch : 76\n0.14135872387304538\n0.07039179147743597\nTrain Accuracy/Verb(Top 1) 0.36695652173913046\nTrain Accuracy/Noun(Top 1) 0.12343478260869566\nTrain Accuracy/Action(Top 1) 0.035478260869565216\nVal Accuracy/Verb(Top 1) 0.2667307692307692\nVal Accuracy/Noun(Top 1) 0.11115384615384616\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 77\n0.14128208291239855\n0.07030274519106237\nTrain Accuracy/Verb(Top 1) 0.3666086956521739\nTrain Accuracy/Noun(Top 1) 0.12334782608695652\nTrain Accuracy/Action(Top 1) 0.035347826086956524\nVal Accuracy/Verb(Top 1) 0.2678846153846154\nVal Accuracy/Noun(Top 1) 0.11115384615384616\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 78\n0.14087370357862333\n0.07028258130317781\nTrain Accuracy/Verb(Top 1) 0.36678260869565216\nTrain Accuracy/Noun(Top 1) 0.12321739130434782\nTrain Accuracy/Action(Top 1) 0.035304347826086956\nVal Accuracy/Verb(Top 1) 0.26865384615384613\nVal Accuracy/Noun(Top 1) 0.11115384615384616\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 79\n0.1410826738287763\n0.07048355079278713\nTrain Accuracy/Verb(Top 1) 0.3666086956521739\nTrain Accuracy/Noun(Top 1) 0.12317391304347826\nTrain Accuracy/Action(Top 1) 0.035173913043478264\nVal Accuracy/Verb(Top 1) 0.26884615384615385\nVal Accuracy/Noun(Top 1) 0.11115384615384616\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 80\n0.1410992087387457\n0.0703244529119352\nTrain Accuracy/Verb(Top 1) 0.36717391304347824\nTrain Accuracy/Noun(Top 1) 0.12308695652173914\nTrain Accuracy/Action(Top 1) 0.035173913043478264\nVal Accuracy/Verb(Top 1) 0.26865384615384613\nVal Accuracy/Noun(Top 1) 0.11115384615384616\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 81\n0.14101224381749222\n0.07014711281148399\nTrain Accuracy/Verb(Top 1) 0.3672608695652174\nTrain Accuracy/Noun(Top 1) 0.12308695652173914\nTrain Accuracy/Action(Top 1) 0.035217391304347825\nVal Accuracy/Verb(Top 1) 0.26942307692307693\nVal Accuracy/Noun(Top 1) 0.11096153846153846\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 82\n0.14115359041748976\n0.07029068499076657\nTrain Accuracy/Verb(Top 1) 0.3676521739130435\nTrain Accuracy/Noun(Top 1) 0.12304347826086956\nTrain Accuracy/Action(Top 1) 0.035260869565217394\nVal Accuracy/Verb(Top 1) 0.2684615384615385\nVal Accuracy/Noun(Top 1) 0.11076923076923077\nVal Accuracy/Action(Top 1) 0.02\nEpoch : 83\n0.1407395090998673\n0.07020730012800636\nTrain Accuracy/Verb(Top 1) 0.36717391304347824\nTrain Accuracy/Noun(Top 1) 0.12295652173913044\nTrain Accuracy/Action(Top 1) 0.035304347826086956\nVal Accuracy/Verb(Top 1) 0.2673076923076923\nVal Accuracy/Noun(Top 1) 0.11076923076923077\nVal Accuracy/Action(Top 1) 0.019807692307692307\nEpoch : 84\n0.14111913457149414\n0.07001095792142356\nTrain Accuracy/Verb(Top 1) 0.367\nTrain Accuracy/Noun(Top 1) 0.12291304347826087\nTrain Accuracy/Action(Top 1) 0.035304347826086956\nVal Accuracy/Verb(Top 1) 0.26884615384615385\nVal Accuracy/Noun(Top 1) 0.11076923076923077\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 85\n0.14104688283873767\n0.06999389383851028\nTrain Accuracy/Verb(Top 1) 0.3674782608695652\nTrain Accuracy/Noun(Top 1) 0.1228695652173913\nTrain Accuracy/Action(Top 1) 0.035173913043478264\nVal Accuracy/Verb(Top 1) 0.2680769230769231\nVal Accuracy/Noun(Top 1) 0.11076923076923077\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 86\n0.14076203064220708\n0.07010111910540884\nTrain Accuracy/Verb(Top 1) 0.3681304347826087\nTrain Accuracy/Noun(Top 1) 0.12282608695652174\nTrain Accuracy/Action(Top 1) 0.035086956521739134\nVal Accuracy/Verb(Top 1) 0.26903846153846156\nVal Accuracy/Noun(Top 1) 0.11096153846153846\nVal Accuracy/Action(Top 1) 0.020576923076923076\nEpoch : 87\n0.14072194564633253\n0.07000562156118997\nTrain Accuracy/Verb(Top 1) 0.3681304347826087\nTrain Accuracy/Noun(Top 1) 0.12282608695652174\nTrain Accuracy/Action(Top 1) 0.035217391304347825\nVal Accuracy/Verb(Top 1) 0.2701923076923077\nVal Accuracy/Noun(Top 1) 0.11076923076923077\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 88\n0.140849935717699\n0.0698006957042508\nTrain Accuracy/Verb(Top 1) 0.3676521739130435\nTrain Accuracy/Noun(Top 1) 0.12273913043478261\nTrain Accuracy/Action(Top 1) 0.035130434782608695\nVal Accuracy/Verb(Top 1) 0.2696153846153846\nVal Accuracy/Noun(Top 1) 0.11076923076923077\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 89\n0.1410679654377263\n0.06992035384585217\nTrain Accuracy/Verb(Top 1) 0.36769565217391303\nTrain Accuracy/Noun(Top 1) 0.12260869565217392\nTrain Accuracy/Action(Top 1) 0.035086956521739134\nVal Accuracy/Verb(Top 1) 0.2696153846153846\nVal Accuracy/Noun(Top 1) 0.11038461538461539\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 90\n0.14076753636685813\n0.06986173965582033\nTrain Accuracy/Verb(Top 1) 0.36769565217391303\nTrain Accuracy/Noun(Top 1) 0.12256521739130435\nTrain Accuracy/Action(Top 1) 0.035\nVal Accuracy/Verb(Top 1) 0.26884615384615385\nVal Accuracy/Noun(Top 1) 0.11038461538461539\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 91\n0.14107367091062592\n0.06976449671314984\nTrain Accuracy/Verb(Top 1) 0.368\nTrain Accuracy/Noun(Top 1) 0.12234782608695652\nTrain Accuracy/Action(Top 1) 0.034739130434782606\nVal Accuracy/Verb(Top 1) 0.2684615384615385\nVal Accuracy/Noun(Top 1) 0.11038461538461539\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 92\n0.1408019682256187\n0.06980125358918818\nTrain Accuracy/Verb(Top 1) 0.36795652173913046\nTrain Accuracy/Noun(Top 1) 0.12234782608695652\nTrain Accuracy/Action(Top 1) 0.034782608695652174\nVal Accuracy/Verb(Top 1) 0.2684615384615385\nVal Accuracy/Noun(Top 1) 0.11019230769230769\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 93\n0.14016157141545924\n0.0698203823915342\nTrain Accuracy/Verb(Top 1) 0.368\nTrain Accuracy/Noun(Top 1) 0.12234782608695652\nTrain Accuracy/Action(Top 1) 0.034869565217391305\nVal Accuracy/Verb(Top 1) 0.2701923076923077\nVal Accuracy/Noun(Top 1) 0.11019230769230769\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 94\n0.14047126421114295\n0.06971065947195379\nTrain Accuracy/Verb(Top 1) 0.36882608695652175\nTrain Accuracy/Noun(Top 1) 0.12230434782608696\nTrain Accuracy/Action(Top 1) 0.034869565217391305\nVal Accuracy/Verb(Top 1) 0.2703846153846154\nVal Accuracy/Noun(Top 1) 0.11019230769230769\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 95\n0.14084774342978873\n0.06969839356294492\nTrain Accuracy/Verb(Top 1) 0.3686521739130435\nTrain Accuracy/Noun(Top 1) 0.1222608695652174\nTrain Accuracy/Action(Top 1) 0.034826086956521736\nVal Accuracy/Verb(Top 1) 0.27076923076923076\nVal Accuracy/Noun(Top 1) 0.11\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 96\n0.1403147950405028\n0.06985834175493659\nTrain Accuracy/Verb(Top 1) 0.368\nTrain Accuracy/Noun(Top 1) 0.12230434782608696\nTrain Accuracy/Action(Top 1) 0.034652173913043476\nVal Accuracy/Verb(Top 1) 0.2684615384615385\nVal Accuracy/Noun(Top 1) 0.11\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 97\n0.14063847137660515\n0.06971757673635716\nTrain Accuracy/Verb(Top 1) 0.36778260869565216\nTrain Accuracy/Noun(Top 1) 0.12221739130434782\nTrain Accuracy/Action(Top 1) 0.034826086956521736\nVal Accuracy/Verb(Top 1) 0.2678846153846154\nVal Accuracy/Noun(Top 1) 0.11\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 98\n0.14062446922790714\n0.06981993230377756\nTrain Accuracy/Verb(Top 1) 0.3680869565217391\nTrain Accuracy/Noun(Top 1) 0.1221304347826087\nTrain Accuracy/Action(Top 1) 0.034826086956521736\nVal Accuracy/Verb(Top 1) 0.2698076923076923\nVal Accuracy/Noun(Top 1) 0.11019230769230769\nVal Accuracy/Action(Top 1) 0.020384615384615383\nEpoch : 99\n0.14028036143721603\n0.06969866316492965\nTrain Accuracy/Verb(Top 1) 0.368\nTrain Accuracy/Noun(Top 1) 0.1221304347826087\nTrain Accuracy/Action(Top 1) 0.034782608695652174\nVal Accuracy/Verb(Top 1) 0.2692307692307692\nVal Accuracy/Noun(Top 1) 0.11\nVal Accuracy/Action(Top 1) 0.020192307692307693\nEpoch : 100\n0.1402540304311892\n0.0696401677480558\nTrain Accuracy/Verb(Top 1) 0.3683913043478261\nTrain Accuracy/Noun(Top 1) 0.12195652173913044\nTrain Accuracy/Action(Top 1) 0.034782608695652174\nVal Accuracy/Verb(Top 1) 0.26942307692307693\nVal Accuracy/Noun(Top 1) 0.10980769230769231\nVal Accuracy/Action(Top 1) 0.020192307692307693\n"
    }
   ],
   "source": [
    "torch.manual_seed(786)\n",
    "\n",
    "df = pd.read_csv(\"EPIC_train_action_labels.csv\", nrows=23192)\n",
    "p_ids = list(set(df[\"participant_id\"].tolist()))\n",
    "\n",
    "d = np.load(\"train-unseen.npy\")\n",
    "weight_verb = [0]*125\n",
    "weight_noun = [0]*352\n",
    "val_verb, weight_verb_ = np.unique(d[:,-2], return_counts=True)\n",
    "val_noun, weight_noun_ = np.unique(d[:,-1], return_counts=True)\n",
    "\n",
    "for i in range(len(val_verb)):\n",
    "    weight_verb[int(val_verb[i])] = weight_verb_[i]\n",
    "\n",
    "for i in range(len(val_noun)):\n",
    "    weight_noun[int(val_noun[i])] = weight_noun_[i]\n",
    "\n",
    "weight_verb = [weight_verb[i]/np.shape(d)[0] for i in range(len(weight_verb))]\n",
    "weight_noun = [weight_noun[i]/np.shape(d)[0] for i in range(len(weight_noun))]\n",
    "\n",
    "train_batch_size = 100\n",
    "train_loaders = []\n",
    "for i in range(len(p_ids)):\n",
    "    dataset_train = EPIC(\"train-unseen.npy\", p_ids[i], df, train_batch_size)\n",
    "    if dataset_train.hasData:\n",
    "        train_loaders.append(DataLoader(dataset_train, batch_size=100, shuffle=True, drop_last=True))\n",
    "\n",
    "dataset_ver= EPIC(\"train-unseen.npy\")\n",
    "ver_loader = DataLoader(dataset_ver, batch_size=200, shuffle=False, drop_last=True)\n",
    "\n",
    "dataset_val = EPIC(\"val-unseen.npy\")\n",
    "val_loader = DataLoader(dataset_val, batch_size=200, shuffle=False, drop_last=True)\n",
    "\n",
    "model = IRM(125, 352)\n",
    "# model = IRM(2, 2)\n",
    "\n",
    "dummy_w = torch.nn.Parameter(torch.tensor([1.0])) \n",
    "opt = torch.optim.SGD(model.parameters(), lr=1e-2)\n",
    "\n",
    "def compute_penalty(losses, dummy):\n",
    "  g1 = torch.autograd.grad(losses[0::2].mean(), dummy, create_graph=True)[0]\n",
    "  g2 = torch.autograd.grad(losses[1::2].mean(), dummy, create_graph=True)[0]\n",
    "  return (g1 * g2).sum()\n",
    "\n",
    "ce_verb = torch.nn.CrossEntropyLoss(weight=torch.tensor(weight_verb).float(), reduction=\"none\")\n",
    "ce_noun = torch.nn.CrossEntropyLoss(weight=torch.tensor(weight_noun).float(), reduction=\"none\")\n",
    "\n",
    "for epoch in range(100):\n",
    "    print(\"Epoch : \"+str(epoch+1))\n",
    "    model.train()\n",
    "    _loaders = [iter(x) for x in train_loaders]\n",
    "\n",
    "    err_verb = 0\n",
    "    err_noun = 0\n",
    "    counter = 0\n",
    "    while True:\n",
    "        verb_loss = 0\n",
    "        noun_loss = 0\n",
    "        verb_penalty = 0\n",
    "        noun_penalty = 0\n",
    "        weight_norms = 0\n",
    "        penalty_factor = 0.15\n",
    "        envs = 0\n",
    "        for i in range(len(_loaders)):\n",
    "            try:\n",
    "                sample_batch = next(_loaders[i])\n",
    "            except:\n",
    "                continue\n",
    "            envs+=1\n",
    "            data = sample_batch[\"data\"]\n",
    "            verb = sample_batch[\"verb_id\"]\n",
    "            noun = sample_batch[\"noun_id\"]\n",
    "\n",
    "            # if torch.isnan(data).any():\n",
    "            #     print(\"Nan Here\")\n",
    "\n",
    "            _out = model(data)\n",
    "            verb_out = _out[\"verb_out\"]\n",
    "            noun_out = _out[\"noun_out\"]\n",
    "            \n",
    "            verb_loss_ = ce_verb(verb_out.squeeze()*dummy_w, verb.view(-1))\n",
    "            noun_loss_= ce_noun(noun_out.squeeze()*dummy_w, noun.view(-1))\n",
    "\n",
    "            weight_norm = torch.as_tensor(0.0)\n",
    "\n",
    "            for w in model.parameters():\n",
    "                weight_norm += w.norm().pow(2)\n",
    "            weight_norms += weight_norm\n",
    "\n",
    "            if epoch > 50:\n",
    "                verb_penalty += compute_penalty(verb_loss_, dummy_w)\n",
    "                noun_penalty += compute_penalty(noun_loss_, dummy_w)\n",
    "\n",
    "            verb_loss += verb_loss_.mean()\n",
    "            noun_loss += noun_loss_.mean()\n",
    "        \n",
    "        if envs == 1:\n",
    "            break\n",
    "        else:\n",
    "            counter+=envs\n",
    "        # err_verb = verb_loss.item()+verb_penalty.item()\n",
    "        # err_noun = noun_loss.item()+noun_penalty.item()\n",
    "        err_verb += verb_loss.item()\n",
    "        err_noun += noun_loss.item()\n",
    "\n",
    "        opt.zero_grad()\n",
    "        (penalty_factor*(verb_loss+noun_loss)+(verb_penalty+noun_penalty)+0.005*weight_norms).backward()\n",
    "        torch.nn.utils.clip_grad_norm(model.parameters(), 20)\n",
    "        opt.step()\n",
    "\n",
    "    print(err_verb/counter)\n",
    "    print(err_noun/counter)\n",
    "    \n",
    "    model.eval()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        correct_verb = 0\n",
    "        correct_noun = 0\n",
    "        correct_action = 0\n",
    "        count = 0\n",
    "        for idx, sample_batch in enumerate(ver_loader):\n",
    "            data = sample_batch[\"data\"]\n",
    "            verb = sample_batch[\"verb_id\"]\n",
    "            noun = sample_batch[\"noun_id\"]\n",
    "\n",
    "            _out = model(data)\n",
    "            verb_out = _out[\"verb_out\"]\n",
    "            noun_out = _out[\"noun_out\"]\n",
    "\n",
    "            pred_class_verb = torch.argmax(verb_out, dim=2)\n",
    "\n",
    "            correct_verb += (\n",
    "                (verb.view(-1) == pred_class_verb.view(-1))\n",
    "                .float()\n",
    "                .sum()\n",
    "                .item()\n",
    "            )\n",
    "\n",
    "            pred_class_noun = torch.argmax(noun_out, dim=2)\n",
    "\n",
    "            correct_noun += (\n",
    "                (noun.view(-1) == pred_class_noun.view(-1))\n",
    "                .float()\n",
    "                .sum()\n",
    "                .item()\n",
    "            )\n",
    "\n",
    "            # Accuracy Action\n",
    "            correct_action += (\n",
    "                (\n",
    "                    (verb.view(-1) == pred_class_verb.view(-1)).int()\n",
    "                    & (noun.view(-1) == pred_class_noun.view(-1)).int()\n",
    "                )\n",
    "                .float()\n",
    "                .sum()\n",
    "                .item()\n",
    "            )\n",
    "\n",
    "            count += 200\n",
    "        print(\"Train Accuracy/Verb(Top 1)\", correct_verb / count)\n",
    "        print(\"Train Accuracy/Noun(Top 1)\", correct_noun / count)\n",
    "        print(\"Train Accuracy/Action(Top 1)\", correct_action / count)\n",
    "\n",
    "\n",
    "\n",
    "        correct_verb = 0\n",
    "        correct_noun = 0\n",
    "        correct_action = 0\n",
    "        count = 0\n",
    "        for idx, sample_batch in enumerate(val_loader):\n",
    "            data = sample_batch[\"data\"]\n",
    "            verb = sample_batch[\"verb_id\"]\n",
    "            noun = sample_batch[\"noun_id\"]\n",
    "\n",
    "            _out = model(data)\n",
    "            verb_out = _out[\"verb_out\"]\n",
    "            noun_out = _out[\"noun_out\"]\n",
    "\n",
    "            pred_class_verb = torch.argmax(verb_out, dim=2)\n",
    "\n",
    "            correct_verb += (\n",
    "                (verb.view(-1) == pred_class_verb.view(-1))\n",
    "                .float()\n",
    "                .sum()\n",
    "                .item()\n",
    "            )\n",
    "\n",
    "            pred_class_noun = torch.argmax(noun_out, dim=2)\n",
    "\n",
    "            correct_noun += (\n",
    "                (noun.view(-1) == pred_class_noun.view(-1))\n",
    "                .float()\n",
    "                .sum()\n",
    "                .item()\n",
    "            )\n",
    "\n",
    "            # Accuracy Action\n",
    "            correct_action += (\n",
    "                (\n",
    "                    (verb.view(-1) == pred_class_verb.view(-1)).int()\n",
    "                    & (noun.view(-1) == pred_class_noun.view(-1)).int()\n",
    "                )\n",
    "                .float()\n",
    "                .sum()\n",
    "                .item()\n",
    "            )\n",
    "\n",
    "            count += 200\n",
    "        print(\"Val Accuracy/Verb(Top 1)\", correct_verb / count)\n",
    "        print(\"Val Accuracy/Noun(Top 1)\", correct_noun / count)\n",
    "        print(\"Val Accuracy/Action(Top 1)\", correct_action / count)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python36864bitshahnawazconda3bcaaaaee8534c10a6bcb78b8ae88eca",
   "display_name": "Python 3.6.8 64-bit ('shahnawaz': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}