CIFARSuper20_Rocket_Unlearn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from dataset import *\n",
    "from model import ResNet18\n",
    "from unlearn import *\n",
    "from metrics import UnLearningScore\n",
    "from utils import *\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "train_ds = CustomCIFAR100(root='.', train=True,download=True, transform=transform_train)\n",
    "valid_ds = CustomCIFAR100(root='.', train=False,download=True, transform=transform_train)\n",
    "\n",
    "batch_size = 256\n",
    "train_dl = DataLoader(train_ds, batch_size, shuffle=True, num_workers=32, pin_memory=True)\n",
    "valid_dl = DataLoader(valid_ds, batch_size, num_workers=32, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 100\n",
    "classwise_train = {}\n",
    "for i in range(num_classes):\n",
    "    classwise_train[i] = []\n",
    "\n",
    "for img, label, clabel in train_ds:\n",
    "    classwise_train[label].append((img, label, clabel))\n",
    "    \n",
    "classwise_test = {}\n",
    "for i in range(num_classes):\n",
    "    classwise_test[i] = []\n",
    "\n",
    "for img, label, clabel in valid_ds:\n",
    "    classwise_test[label].append((img, label, clabel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train the model\n",
    "device = 'cuda'\n",
    "model = ResNet18(num_classes = 20, pretrained = True).to(device)\n",
    "epochs = 5\n",
    "history = fit_one_cycle(epochs, model, train_dl, valid_dl, device = device)\n",
    "#torch.save(model.state_dict(), \"ResNET18_CIFAR100Super20_Pretrained_ALL_CLASSES_5_Epochs.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load the trained model\n",
    "device = 'cuda'\n",
    "model = ResNet18(num_classes = 20, pretrained = True).to(device)\n",
    "model.load_state_dict(torch.load(\"ResNET18_CIFAR100Super20_Pretrained_ALL_CLASSES_5_Epochs.pt\", map_location='cuda'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Forgetting Rocket\n",
    "The Rocket is class 69 in CIFAR100 and belongs to Super Class 19 (Vehicles) in CIFAR Super 20."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the forget and retain validation data\n",
    "forget_valid = []\n",
    "forget_classes = [69]\n",
    "for cls in range(num_classes):\n",
    "    if cls in forget_classes:\n",
    "        for img, label, clabel in classwise_test[cls]:\n",
    "            forget_valid.append((img, label, clabel))\n",
    "\n",
    "retain_valid = []\n",
    "for cls in range(num_classes):\n",
    "    if cls not in forget_classes:\n",
    "        for img, label, clabel in classwise_test[cls]:\n",
    "            retain_valid.append((img, label, clabel))\n",
    "            \n",
    "forget_train = []\n",
    "for cls in range(num_classes):\n",
    "    if cls in forget_classes:\n",
    "        for img, label, clabel in classwise_train[cls]:\n",
    "            forget_train.append((img, label, clabel))\n",
    "\n",
    "retain_train = []\n",
    "for cls in range(num_classes):\n",
    "    if cls not in forget_classes:\n",
    "        for img, label, clabel in classwise_train[cls]:\n",
    "            retain_train.append((img, label, clabel))\n",
    "\n",
    "forget_valid_dl = DataLoader(forget_valid, batch_size, num_workers=32, pin_memory=True)\n",
    "\n",
    "retain_valid_dl = DataLoader(retain_valid, batch_size, num_workers=32, pin_memory=True)\n",
    "\n",
    "forget_train_dl = DataLoader(forget_train, batch_size, num_workers=32, pin_memory=True)\n",
    "retain_train_dl = DataLoader(retain_train, batch_size, num_workers=32, pin_memory=True, shuffle = True)\n",
    "import random\n",
    "retain_train_subset = random.sample(retain_train, int(0.3*len(retain_train)))\n",
    "retain_train_subset_dl = DataLoader(retain_train_subset, batch_size, num_workers=32, pin_memory=True, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Loss': 0.535236120223999, 'Acc': 85.77934265136719}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Performance of Fully trained model on retain set\n",
    "evaluate(model, retain_valid_dl, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Loss': 0.5363734364509583, 'Acc': 82.0}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Performance of Fully trained model on retain set\n",
    "evaluate(model, forget_valid_dl, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrain the model from Scratch\n",
    "Create Retrained Model (Gold model). This is the model trained from scratch without forget data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda'\n",
    "gold_model = ResNet18(num_classes = 20, pretrained = True).to(device)\n",
    "epochs = 5\n",
    "history = fit_one_cycle(epochs, gold_model, retain_train_dl, retain_valid_dl, device = device)\n",
    "torch.save(gold_model.state_dict(), \"ResNET18_CIFAR100Super20_Pretrained_Gold_Class69_5_Epochs.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = 'cuda'\n",
    "gold_model = ResNet18(num_classes = 20, pretrained = True).to(device)\n",
    "gold_model.load_state_dict(torch.load(\"ResNET18_CIFAR100Super20_Pretrained_Gold_Class69_5_Epochs.pt\", map_location=device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Loss': 7.545389175415039, 'Acc': 3.0}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate gold model on forget set\n",
    "evaluate(gold_model, forget_valid_dl, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Loss': 0.5325239896774292, 'Acc': 85.76885223388672}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate gold model on retain set\n",
    "evaluate(gold_model, retain_valid_dl, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## UnLearning via proposed method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/users/visionintelligence/Vikram/anaconda3/lib/python3.9/site-packages/torch/nn/functional.py:2747: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Unlearning Loss 0.004115822724997997\n"
     ]
    }
   ],
   "source": [
    "device = 'cuda'\n",
    "unlearning_teacher = ResNet18(num_classes = 20, pretrained = False).to(device).eval()\n",
    "student_model = ResNet18(num_classes = 20, pretrained = False).to(device)\n",
    "student_model.load_state_dict(torch.load(\"ResNET18_CIFAR100Super20_Pretrained_ALL_CLASSES_5_Epochs.pt\", map_location = device))\n",
    "model = model.eval()\n",
    "\n",
    "KL_temperature = 1\n",
    "\n",
    "optimizer = torch.optim.Adam(student_model.parameters(), lr = 0.0001)\n",
    "\n",
    "blindspot_unlearner(model = student_model, unlearning_teacher = unlearning_teacher, full_trained_teacher = model, \n",
    "          retain_data = retain_train_subset, forget_data = forget_train, epochs = 1, optimizer = optimizer, lr = 0.0001, \n",
    "          batch_size = 256, num_workers = 32, device = device, KL_temperature = KL_temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Loss': 3.3266074657440186, 'Acc': 3.0}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# performance of unlearned model on forget set\n",
    "evaluate(student_model, forget_valid_dl, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Loss': 0.5810623168945312, 'Acc': 84.57299041748047}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# performance of unlearned model on retain set\n",
    "evaluate(student_model, retain_valid_dl, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Measure ZRF (Unlearning Score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Score: 0.8766639232635498\n",
      "Our Score: 0.99411541223526\n",
      "Gold Score: 0.9299044013023376\n",
      "JS Div: 0.04860961437225342\n"
     ]
    }
   ],
   "source": [
    "print(\"Initial Score: {}\".format(UnLearningScore(model, unlearning_teacher, forget_valid_dl, 256, 'cuda')))\n",
    "print(\"Our Score: {}\".format(UnLearningScore(student_model, unlearning_teacher, forget_valid_dl, 256, 'cuda')))\n",
    "print(\"Gold Score: {}\".format(UnLearningScore(gold_model, unlearning_teacher, forget_valid_dl, 256, 'cuda')))\n",
    "print(\"JS Div: {}\".format(1-UnLearningScore(gold_model, student_model, forget_valid_dl, 256, 'cuda')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unlearning using Amnesiac unlearning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlearninglabels = list(range(20))\n",
    "unlearninglabels.remove(19)\n",
    "unlearning_train_set = []\n",
    "for cls in range(num_classes):\n",
    "    if cls in forget_classes:\n",
    "        for img, label, clabel in classwise_train[cls]:\n",
    "            unlearning_train_set.append((img, label, random.choice(unlearninglabels)))\n",
    "\n",
    "\n",
    "\n",
    "for cls in range(num_classes):\n",
    "    if cls not in forget_classes:\n",
    "        for img, label, clabel in classwise_train[cls]:\n",
    "            unlearning_train_set.append((img, label, clabel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlearning_train_set_dl = DataLoader(unlearning_train_set, batch_size, num_workers = 32, pin_memory = True, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [0], last_lr: 0.00010, train_loss: 0.0953, val_loss: 0.5882, val_acc: 84.9121\n",
      "Epoch [1], last_lr: 0.00010, train_loss: 0.0638, val_loss: 0.6049, val_acc: 84.9187\n",
      "Epoch [2], last_lr: 0.00010, train_loss: 0.0413, val_loss: 0.6110, val_acc: 84.9233\n"
     ]
    }
   ],
   "source": [
    "device = 'cuda'\n",
    "student_model = ResNet18(num_classes = 20, pretrained = True).to(device)\n",
    "student_model.load_state_dict(torch.load(\"ResNET18_CIFAR100Super20_Pretrained_ALL_CLASSES_5_Epochs.pt\", map_location = 'cuda'))\n",
    "epochs = 3\n",
    "\n",
    "history = fit_one_unlearning_cycle(epochs, student_model, unlearning_train_set_dl, retain_valid_dl, device = device, lr = 0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Forget Performance: {'Loss': 4.92495059967041, 'Acc': 2.0}\n",
      "Retain Performance: {'Loss': 0.6110122799873352, 'Acc': 84.92332458496094}\n"
     ]
    }
   ],
   "source": [
    "print(\"Forget Performance: {}\".format(evaluate(student_model, forget_valid_dl, device)))\n",
    "print(\"Retain Performance: {}\".format(evaluate(student_model, retain_valid_dl, device)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unlearning using UNSIR (Class 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 20\n",
    "classwise_train = {}\n",
    "for i in range(num_classes):\n",
    "    classwise_train[i] = []\n",
    "\n",
    "for img, label, clabel in train_ds:\n",
    "    classwise_train[clabel].append((img, label, clabel))\n",
    "    \n",
    "classwise_test = {}\n",
    "for i in range(num_classes):\n",
    "    classwise_test[i] = []\n",
    "\n",
    "for img, label, clabel in valid_ds:\n",
    "    classwise_test[clabel].append((img, label, clabel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the forget and retain validation data\n",
    "forget_valid = []\n",
    "forget_classes = [0]\n",
    "for cls in range(num_classes):\n",
    "    if cls in forget_classes:\n",
    "        for img, label, clabel in classwise_test[cls]:\n",
    "            forget_valid.append((img, label, clabel))\n",
    "\n",
    "retain_valid = []\n",
    "for cls in range(num_classes):\n",
    "    if cls not in forget_classes:\n",
    "        for img, label, clabel in classwise_test[cls]:\n",
    "            retain_valid.append((img, label, clabel))\n",
    "            \n",
    "forget_train = []\n",
    "for cls in range(num_classes):\n",
    "    if cls in forget_classes:\n",
    "        for img, label, clabel in classwise_train[cls]:\n",
    "            forget_train.append((img, label, clabel))\n",
    "\n",
    "retain_train = []\n",
    "for cls in range(num_classes):\n",
    "    if cls not in forget_classes:\n",
    "        for img, label, clabel in classwise_train[cls]:\n",
    "            retain_train.append((img, label, clabel))\n",
    "\n",
    "forget_valid_dl = DataLoader(forget_valid, batch_size, num_workers=32, pin_memory=True)\n",
    "\n",
    "retain_valid_dl = DataLoader(retain_valid, batch_size, num_workers=32, pin_memory=True)\n",
    "\n",
    "forget_train_dl = DataLoader(forget_train, batch_size, num_workers=32, pin_memory=True)\n",
    "retain_train_dl = DataLoader(retain_train, batch_size, num_workers=32, pin_memory=True, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#collect some samples from each class\n",
    "num_samples = 500\n",
    "retain_samples = []\n",
    "for i in range(num_classes):\n",
    "    if i not in forget_classes:\n",
    "        retain_samples += classwise_train[i][:num_samples]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise_batch_size = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = 'cuda'\n",
    "student_model = ResNet18(num_classes = 20, pretrained = False).to(device)\n",
    "student_model.load_state_dict(torch.load(\"ResNET18_CIFAR100Super20_Pretrained_ALL_CLASSES_5_Epochs.pt\", map_location = device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise = UNSIR_noise(batch_size, 3, 224, 224).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "forget_class_label = 0\n",
    "num_epochs = 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 15047.9326171875\n",
      "Loss: 6428.56494140625\n",
      "Loss: 2585.14208984375\n",
      "Loss: 1118.1822509765625\n",
      "Loss: 594.6854858398438\n",
      "Loss: 380.76507568359375\n",
      "Loss: 258.22991943359375\n",
      "Loss: 175.17173767089844\n",
      "Loss: 121.2085189819336\n",
      "Loss: 86.78948974609375\n",
      "Loss: 66.66294860839844\n",
      "Loss: 56.79840087890625\n",
      "Loss: 52.446632385253906\n",
      "Loss: 48.63398742675781\n",
      "Loss: 47.29644775390625\n",
      "Loss: 46.700103759765625\n",
      "Loss: 46.97292709350586\n",
      "Loss: 46.851234436035156\n",
      "Loss: 47.27818298339844\n",
      "Loss: 47.98065185546875\n",
      "Loss: 47.55046081542969\n",
      "Loss: 48.1744270324707\n",
      "Loss: 48.738792419433594\n",
      "Loss: 49.70948791503906\n",
      "Loss: 48.98561477661133\n",
      "Loss: 49.938560485839844\n",
      "Loss: 50.545745849609375\n",
      "Loss: 49.86770248413086\n",
      "Loss: 49.83490753173828\n",
      "Loss: 50.902198791503906\n",
      "Loss: 49.465301513671875\n",
      "Loss: 50.633384704589844\n",
      "Loss: 51.02934265136719\n",
      "Loss: 50.88955307006836\n",
      "Loss: 51.475547790527344\n",
      "Loss: 52.239593505859375\n",
      "Loss: 51.24016571044922\n",
      "Loss: 51.59583282470703\n",
      "Loss: 51.716270446777344\n",
      "Loss: 51.37892532348633\n",
      "Loss: 52.572391510009766\n",
      "Loss: 52.83474349975586\n",
      "Loss: 53.65574645996094\n",
      "Loss: 52.888275146484375\n",
      "Loss: 53.729156494140625\n",
      "Loss: 54.304771423339844\n",
      "Loss: 54.037353515625\n",
      "Loss: 55.88550567626953\n",
      "Loss: 54.78221893310547\n",
      "Loss: 54.159847259521484\n"
     ]
    }
   ],
   "source": [
    "noise =  UNSIR_noise_train(noise, student_model, forget_class_label, num_epochs,\\\n",
    "                           noise_batch_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "noisy_loader = UNSIR_create_noisy_loader(noise, forget_class_label\\\n",
    "                                         , retain_samples, batch_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [0], last_lr: 0.00010, train_loss: 0.0222, val_loss: 0.6516, val_acc: 84.0152\n"
     ]
    }
   ],
   "source": [
    "#impair step\n",
    "epochs = 1\n",
    "history = fit_one_unlearning_cycle(epochs, student_model, noisy_loader, retain_valid_dl, device = device, lr = 0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Forget Performance: {'Loss': 4.435159683227539, 'Acc': 20.274078369140625}\n",
      "Retain Performance: {'Loss': 0.6515841484069824, 'Acc': 84.01521301269531}\n"
     ]
    }
   ],
   "source": [
    "print(\"Forget Performance: {}\".format(evaluate(student_model, forget_valid_dl, device)))\n",
    "print(\"Retain Performance: {}\".format(evaluate(student_model, retain_valid_dl, device)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [0], last_lr: 0.00010, train_loss: 0.0178, val_loss: 0.6395, val_acc: 84.6217\n"
     ]
    }
   ],
   "source": [
    "#repair step\n",
    "other_samples = []\n",
    "for i in range(len(retain_samples)):\n",
    "    other_samples.append((retain_samples[i][0].cpu(), torch.tensor(retain_samples[i][2]),\n",
    "                            torch.tensor(retain_samples[i][2])))    \n",
    "\n",
    "heal_loader = torch.utils.data.DataLoader(other_samples, batch_size=batch_size, shuffle = True)\n",
    "epochs = 1\n",
    "history = fit_one_unlearning_cycle(epochs, student_model, heal_loader, retain_valid_dl, device = device, lr = 0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Forget Performance: {'Loss': 4.577937602996826, 'Acc': 17.43404197692871}\n",
      "Retain Performance: {'Loss': 0.639464259147644, 'Acc': 84.62171173095703}\n"
     ]
    }
   ],
   "source": [
    "print(\"Forget Performance: {}\".format(evaluate(student_model, forget_valid_dl, device)))\n",
    "print(\"Retain Performance: {}\".format(evaluate(student_model, retain_valid_dl, device)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}