EC-GAN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SSL-GAN.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "baEYxXC5pmRS",
        "llqFIh4IqAeV"
      ],
      "toc_visible": true,
      "mount_file_id": "https://github.com/ayaanzhaque/GANs-Research/blob/master/SSL_GAN.ipynb",
      "authorship_tag": "ABX9TyM4b9ZntMm7x2YFE+t5VOgX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayaanzhaque/EC-GAN/blob/main/EC-GAN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YPziPBDQpgN7"
      },
      "source": [
        "# GAN Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HGJjhQPdfzSL"
      },
      "source": [
        "import math\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pdb\n",
        "\n",
        "# torch imports\n",
        "import torch\n",
        "from torch.utils.data import DataLoader,Dataset\n",
        "from torch import optim,nn\n",
        "import torchvision\n",
        "import torchvision.datasets as datasets\n",
        "import torchvision.transforms as transforms\n",
        "import torch.nn.functional as F\n",
        "import torchvision.utils as vutils"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WB-2iFGCf1eK"
      },
      "source": [
        "# Models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R_aBQc3df4Uq"
      },
      "source": [
        "## ResNet 18"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XaW_Qyu0HZII"
      },
      "source": [
        "# ResNet Classifier\n",
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, in_planes, planes, stride=1):\n",
        "        super(BasicBlock, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "\n",
        "        self.shortcut = nn.Sequential()\n",
        "        if stride != 1 or in_planes != self.expansion*planes:\n",
        "            self.shortcut = nn.Sequential(\n",
        "                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),\n",
        "                nn.BatchNorm2d(self.expansion*planes)\n",
        "            )\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.bn2(self.conv2(out))\n",
        "        out += self.shortcut(x)\n",
        "        out = F.relu(out)\n",
        "        return out\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "    def __init__(self, block, num_blocks, num_classes=10):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.in_planes = 16\n",
        "        self.embDim = 128 * block.expansion\n",
        "        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(16)\n",
        "        self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)\n",
        "        self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)\n",
        "        self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)\n",
        "        self.layer4 = self._make_layer(block, 128, num_blocks[3], stride=2)\n",
        "        self.linear = nn.Linear(128 * block.expansion, num_classes)\n",
        "    def _make_layer(self, block, planes, num_blocks, stride):\n",
        "        strides = [stride] + [1]*(num_blocks-1)\n",
        "        layers = []\n",
        "        for stride in strides:\n",
        "            layers.append(block(self.in_planes, planes, stride))\n",
        "            self.in_planes = planes * block.expansion\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.layer1(out)\n",
        "        out = self.layer2(out)\n",
        "        out = self.layer3(out)\n",
        "        out = self.layer4(out)\n",
        "        out = F.avg_pool2d(out, 4)\n",
        "        emb = out.view(out.size(0), -1)\n",
        "        out = self.linear(emb)\n",
        "        return out#, emb\n",
        "    def get_embedding_dim(self):\n",
        "        return self.embDim\n",
        "\n",
        "def ResNet18():\n",
        "    return ResNet(BasicBlock, [2,2,2,2])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dRzBaZTmgBvo"
      },
      "source": [
        "## DC Generator\n",
        "\n",
        "This is the DC Generator PyTorch Implementation. We edited this model into a 32x32 version instead of the standard 64x64 version. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0_hsQeu_f_RF"
      },
      "source": [
        "class DCGAN_generator(nn.Module):\n",
        "  \"\"\"\n",
        "\n",
        "  Attributes\n",
        "  ----------\n",
        "    ngpu : int\n",
        "      The number of available GPU devices\n",
        "\n",
        "  \"\"\"\n",
        "  def __init__(self, ngpu):\n",
        "    \"\"\"Init function\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "      ngpu : int\n",
        "        The number of available GPU devices\n",
        "\n",
        "    \"\"\"\n",
        "    super(DCGAN_generator, self).__init__()\n",
        "    self.ngpu = ngpu\n",
        "    \n",
        "    nz = 100 # noise dimension\n",
        "    ngf = 64 # number of features map on the first layer\n",
        "    nc = 3 # number of channels\n",
        "\n",
        "    self.main = nn.Sequential(\n",
        "      # input is Z, going into a convolution\n",
        "      nn.ConvTranspose2d(     nz, ngf * 4, 4, 1, 0, bias=False),\n",
        "      nn.BatchNorm2d(ngf * 4),\n",
        "      nn.ReLU(True),\n",
        "      # state size. (ngf*8) x 4 x 4\n",
        "      nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False),\n",
        "      nn.BatchNorm2d(ngf * 2),\n",
        "      nn.ReLU(True),\n",
        "      # state size. (ngf*4) x 8 x 8\n",
        "      nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False),\n",
        "      nn.BatchNorm2d(ngf),\n",
        "      nn.ReLU(True),\n",
        "      # state size. (ngf*2) x 16 x 16\n",
        "      nn.ConvTranspose2d(ngf, nc, 4, 2, 1, bias=False),\n",
        "      nn.Tanh()\n",
        "      # state size. (nc) x 64 x 64\n",
        "    )\n",
        "\n",
        "  def forward(self, input):\n",
        "    \"\"\"Forward function\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    input : :py:class:`torch.Tensor`\n",
        "    \n",
        "    Returns\n",
        "    -------\n",
        "    :py:class:`torch.Tensor`\n",
        "      the output of the generator (i.e. an image)\n",
        "\n",
        "    \"\"\"\n",
        "    output = self.main(input)\n",
        "    return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_VVmcHa8ghaU"
      },
      "source": [
        "## DC Discriminator\n",
        "\n",
        "This is the DC Discriminator PyTorch Implementation. We edited this model into a 32x32 version instead of the standard 64x64 version. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o5d6GAUIglSx"
      },
      "source": [
        "class DCGAN_discriminator(nn.Module):\n",
        "  \"\"\" \n",
        "\n",
        "  Attributes\n",
        "  ----------\n",
        "    ngpu : int\n",
        "      The number of available GPU devices\n",
        "\n",
        "  \"\"\"\n",
        "  def __init__(self, ngpu):\n",
        "    \"\"\"Init function\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "      ngpu : int\n",
        "        The number of available GPU devices\n",
        "\n",
        "    \"\"\"\n",
        "    super(DCGAN_discriminator, self).__init__()\n",
        "    self.ngpu = ngpu\n",
        "        \n",
        "    ndf = 64\n",
        "    nc = 3\n",
        "       \n",
        "    self.main = nn.Sequential(\n",
        "      nn.Conv2d(nc, ndf, 4, 2, 1, bias=False),\n",
        "      nn.BatchNorm2d(ndf),\n",
        "      nn.LeakyReLU(0.2, inplace=True),\n",
        "\n",
        "      nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),\n",
        "      nn.BatchNorm2d(ndf * 2),\n",
        "      nn.LeakyReLU(0.2, inplace=True),\n",
        "      # state size. (ndf*4) x 8 x 8\n",
        "      nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),\n",
        "      nn.BatchNorm2d(ndf * 4),\n",
        "      nn.LeakyReLU(0.2, inplace=True),\n",
        "      # state size. (ndf*8) x 4 x 4\n",
        "      nn.Conv2d(ndf * 4, 1, 4, 1, 0, bias=False),\n",
        "      nn.Sigmoid()\n",
        "    )\n",
        "\n",
        "  def forward(self, input):\n",
        "    \"\"\"Forward function\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    input : :py:class:`torch.Tensor`\n",
        "    \n",
        "    Returns\n",
        "    -------\n",
        "    :py:class:`torch.Tensor`\n",
        "      the output of the generator (i.e. an image)\n",
        "\n",
        "    \"\"\"\n",
        "    output = self.main(input)\n",
        "\n",
        "    return output.view(-1, 1).squeeze(1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UAAD2FEkg_K3"
      },
      "source": [
        "# Data Pre-Processing\n",
        "Since the SVHN Dataset is already formatted well, there is little preprocessing needed other than converting the images into Tensors and using the PyTorch DataLoaders. We have created a process for restricting the dataset size."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WehTUhcnhMOF"
      },
      "source": [
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor()])\n",
        "\n",
        "transform_train = transforms.Compose([\n",
        "    transforms.RandomCrop(32, padding=4),\n",
        "    transforms.RandomRotation(10),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
        "])\n",
        "\n",
        "transform_test = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
        "])\n",
        "\n",
        "# regular data loaders\n",
        "batch_size = 64\n",
        "trainset = datasets.SVHN(\"/content\", split='train', download = True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n",
        "                                          shuffle=True, num_workers=2,)\n",
        "\n",
        "testset = datasets.SVHN(\"/content\", split='test', download = True, transform=transform)\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
        "                                         shuffle=False, num_workers=2)\n",
        "\n",
        "#create subsets\n",
        "dataSizeConstant = 0.1\n",
        "valDataFraction = 0.1\n",
        "subset = np.random.permutation([i for i in range(len(trainset))])\n",
        "subTrain = subset[:int(len(trainset) * (dataSizeConstant)]\n",
        "subTrainSet = datasets.SVHN(\"/content\", split = \"train\", download = True, transform = transform)\n",
        "subTrainLoader = DataLoader(subTrainSet, batch_size = batch_size, shuffle= False, num_workers= 2, sampler = torch.utils.data.SubsetRandomSampler(subTrain))\n",
        "\n",
        "# restrict validation size if needed\n",
        "# subset = np.random.permutation([i for i in range(len(trainset))])\n",
        "# SubTest = subset[: int(len(trainset) * (dataSizeConstant * valDataFraction))]\n",
        "# subTestSet = datasets.SVHN(\"/content\", split = \"train\", download = True, transform = transform)\n",
        "# subTestLoader = DataLoader(subTestSet, batch_size = batch_size, shuffle= False, num_workers= 2, sampler = torch.utils.data.SubsetRandomSampler(SubTest))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W4sE1eLjhjr6"
      },
      "source": [
        "# Prepare for Training\n",
        "We will initialize all of our models, optimizers, and loss functions. We will set important hyperparameters."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2gAmUj7Ah1eA"
      },
      "source": [
        "# define device \n",
        "device = torch.device(\"cuda:0\")\n",
        "\n",
        "# data for plotting purposes\n",
        "generatorLosses = []\n",
        "discriminatorLosses = []\n",
        "classifierLosses = []\n",
        "\n",
        "#training starts\n",
        "\n",
        "epochs = 100\n",
        "\n",
        "# models\n",
        "netG = DCGAN_generator(1)\n",
        "netD = DCGAN_discriminator(1)\n",
        "netC = ResNet18()\n",
        "\n",
        "netG.to(device)\n",
        "netD.to(device)\n",
        "netC.to(device)\n",
        "\n",
        "# optimizers \n",
        "optD = optim.Adam(netD.parameters(), lr=0.0002, betas=(0.5, 0.999), weight_decay = 1e-3) \n",
        "optG = optim.Adam(netG.parameters(), lr=0.0002, betas=(0.5, 0.999)) \n",
        "optC = optim.Adam(netC.parameters(), lr=0.0002, betas=(0.5, 0.999), weight_decay = 1e-3) \n",
        "\n",
        "advWeight = 0.1 # adversarial weight\n",
        "\n",
        "loss = nn.BCELoss()\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "file = open(\"ExternalClassifier.txt\", \"w\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pfkf_nq2iwhe"
      },
      "source": [
        "# Training Loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NN_Nx5dqi0LC"
      },
      "source": [
        "def train(datasetLoader):\n",
        "  file.write(text)\n",
        "  for epoch in range(epochs):\n",
        "    netC.train()\n",
        "\n",
        "    running_loss = 0.0\n",
        "    total_train = 0\n",
        "    correct_train = 0\n",
        "    for i, data in enumerate(subTrainLoader, 0):\n",
        "      \n",
        "      dataiter = iter(subTrainLoader)\n",
        "      inputs, labels = dataiter.next()\n",
        "      inputs, labels = inputs.to(device), labels.to(device)\n",
        "      tmpBatchSize = len(labels)  \n",
        "\n",
        "      # create label arrays \n",
        "      true_label = torch.ones(tmpBatchSize, 1, device=device)\n",
        "      fake_label = torch.zeros(tmpBatchSize, 1, device=device)\n",
        "\n",
        "      r = torch.randn(tmpBatchSize, 100, 1, 1, device=device)\n",
        "      fakeImageBatch = netG(r)\n",
        "\n",
        "      real_cpu = data[0].to(device)\n",
        "      batch_size = real_cpu.size(0)\n",
        "\n",
        "      # train discriminator on real images\n",
        "      predictionsReal = netD(inputs)\n",
        "      lossDiscriminator = loss(predictionsReal, true_label) #labels = 1\n",
        "      lossDiscriminator.backward(retain_graph = True)\n",
        "\n",
        "      # train discriminator on fake images\n",
        "      predictionsFake = netD(fakeImageBatch)\n",
        "      lossFake = loss(predictionsFake, fake_label)  #labels = 0\n",
        "      lossFake.backward(retain_graph= True)\n",
        "      optD.step() # update discriminator parameters    \n",
        "\n",
        "      # train generator \n",
        "      optG.zero_grad()\n",
        "      predictionsFake = netD(fakeImageBatch)\n",
        "      lossGenerator = loss(predictionsFake, true_label) #labels = 1\n",
        "      lossGenerator.backward(retain_graph = True)\n",
        "      optG.step()\n",
        "\n",
        "      torch.autograd.set_detect_anomaly(True)\n",
        "      fakeImageBatch = fakeImageBatch.detach().clone()\n",
        "\n",
        "      # train classifier on real data\n",
        "      predictions = netC(inputs)\n",
        "      realClassifierLoss = criterion(predictions, labels)\n",
        "      realClassifierLoss.backward(retain_graph=True)\n",
        "      \n",
        "      optC.step()\n",
        "      optC.zero_grad()\n",
        "\n",
        "      # update the classifer on fake data\n",
        "      predictionsFake = netC(fakeImageBatch)\n",
        "      # get a tensor of the labels that are most likely according to model\n",
        "      predictedLabels = torch.argmax(predictionsFake, 1) # -> [0 , 5, 9, 3, ...]\n",
        "      confidenceThresh = .2\n",
        "\n",
        "      # psuedo labeling threshold\n",
        "      probs = F.softmax(predictionsFake, dim=1)\n",
        "      mostLikelyProbs = np.asarray([probs[i, predictedLabels[i]].item() for  i in range(len(probs))])\n",
        "      toKeep = mostLikelyProbs > confidenceThresh\n",
        "      if sum(toKeep) != 0:\n",
        "          fakeClassifierLoss = criterion(predictionsFake[toKeep], predictedLabels[toKeep]) * advWeight\n",
        "          fakeClassifierLoss.backward()\n",
        "          \n",
        "      optC.step()\n",
        "\n",
        "      # reset the gradients\n",
        "      optD.zero_grad()\n",
        "      optG.zero_grad()\n",
        "      optC.zero_grad()\n",
        "\n",
        "      # save losses for graphing\n",
        "      generatorLosses.append(lossGenerator.item())\n",
        "      discriminatorLosses.append(lossDiscriminator.item())\n",
        "      classifierLosses.append(realClassifierLoss.item())\n",
        "\n",
        "      # get train accurcy \n",
        "      if(i % 100 == 0):\n",
        "        netC.eval()\n",
        "        # accuracy\n",
        "        _, predicted = torch.max(predictions, 1)\n",
        "        total_train += labels.size(0)\n",
        "        correct_train += predicted.eq(labels.data).sum().item()\n",
        "        train_accuracy = 100 * correct_train / total_train\n",
        "        text = (\"Train Accuracy: \" + str(train_accuracy))\n",
        "        file.write(text + '\\n')\n",
        "        netC.train()\n",
        "\n",
        "    print(\"Epoch \" + str(epoch) + \"Complete\")\n",
        "    \n",
        "    # save gan image\n",
        "    gridOfFakeImages = torchvision.utils.make_grid(fakeImageBatch.cpu())\n",
        "    torchvision.utils.save_image(gridOfFakeImages, \"/content/gridOfFakeImages/\" + str(epoch) + '_' + str(i) + '.png')\n",
        "    validate()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "23mj3-JRi6ME"
      },
      "source": [
        "def validate():\n",
        "  netC.eval()\n",
        "  correct = 0\n",
        "  total = 0\n",
        "  with torch.no_grad():\n",
        "      for data in testloader:\n",
        "          inputs, labels = data\n",
        "          inputs, labels = data[0].to(device), data[1].to(device)\n",
        "          outputs = netC(inputs)\n",
        "          _, predicted = torch.max(outputs.data, 1)\n",
        "          total += labels.size(0)\n",
        "          correct += (predicted == labels).sum().item()\n",
        "\n",
        "  accuracy = (correct / total) * 100 \n",
        "\n",
        "  print('Accuracy of the network on the 10000 test images: %d %%' % (\n",
        "      100 * correct / total))\n",
        "\n",
        "  text = (\"Test Accuracy: \" + str(accuracy) + \"\\n\")\n",
        "  file.write(text)\n",
        "  netC.train()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eO12e0gLi4LW"
      },
      "source": [
        "train(subTrainLoader)\n",
        "file.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T5BEZtGgjBvI"
      },
      "source": [
        "## Plot Losses"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PspKHII2i2qQ"
      },
      "source": [
        "# plot losses\n",
        "plt.figure(figsize=(10,5))\n",
        "plt.title(\"Loss of Models\")\n",
        "plt.plot(generatorLosses,label=\"Generator\")\n",
        "plt.plot(discriminatorLosses,label=\"Discriminator\")\n",
        "plt.plot(classifierLosses, label = \"Classifier\")\n",
        "plt.xlabel(\"Batches\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "baEYxXC5pmRS"
      },
      "source": [
        "# Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jb_01A1i8NZr"
      },
      "source": [
        "Classifier with different dataset sizes and generated images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AiC1FN2n8w5o"
      },
      "source": [
        "import math\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pdb\n",
        "\n",
        "# torch imports\n",
        "import torch\n",
        "from torch.utils.data import DataLoader,Dataset\n",
        "from torch import optim,nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "import torchvision.datasets as datasets\n",
        "import torchvision.transforms as transforms\n",
        "import torch.utils.data\n",
        "import torchvision.utils as vutils\n",
        "from torch.autograd import Variable\n",
        "\n",
        "# model\n",
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, in_planes, planes, stride=1):\n",
        "        super(BasicBlock, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "\n",
        "        self.shortcut = nn.Sequential()\n",
        "        if stride != 1 or in_planes != self.expansion*planes:\n",
        "            self.shortcut = nn.Sequential(\n",
        "                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),\n",
        "                nn.BatchNorm2d(self.expansion*planes)\n",
        "            )\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.bn2(self.conv2(out))\n",
        "        out += self.shortcut(x)\n",
        "        out = F.relu(out)\n",
        "        return out\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "    def __init__(self, block, num_blocks, num_classes=10):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.in_planes = 16\n",
        "        self.embDim = 128 * block.expansion\n",
        "        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(16)\n",
        "        self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)\n",
        "        self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)\n",
        "        self.layer3 = self._make_layer(block, 64, num_blocks[2], stride=2)\n",
        "        self.layer4 = self._make_layer(block, 128, num_blocks[3], stride=2)\n",
        "        self.linear = nn.Linear(128 * block.expansion, num_classes)\n",
        "    def _make_layer(self, block, planes, num_blocks, stride):\n",
        "        strides = [stride] + [1]*(num_blocks-1)\n",
        "        layers = []\n",
        "        for stride in strides:\n",
        "            layers.append(block(self.in_planes, planes, stride))\n",
        "            self.in_planes = planes * block.expansion\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.layer1(out)\n",
        "        out = self.layer2(out)\n",
        "        out = self.layer3(out)\n",
        "        out = self.layer4(out)\n",
        "        out = F.avg_pool2d(out, 4)\n",
        "        emb = out.view(out.size(0), -1)\n",
        "        out = self.linear(emb)\n",
        "        return out#, emb\n",
        "    def get_embedding_dim(self):\n",
        "        return self.embDim\n",
        "\n",
        "def ResNet18():\n",
        "    return ResNet(BasicBlock, [2,2,2,2])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89Fpit5ukpvl"
      },
      "source": [
        "#data preprocessing\n",
        "\n",
        "file = open(\"StandardClassifier.txt\", \"w\")\n",
        "\n",
        "transform_augment = transforms.Compose([\n",
        "    transforms.RandomCrop(32, padding=4),\n",
        "    transforms.RandomRotation(10),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
        "])\n",
        "\n",
        "transform = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n",
        "])\n",
        "\n",
        "batch_size = 64\n",
        "trainset = datasets.SVHN(\"/content\", split='train', download = True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n",
        "                                          shuffle=True, num_workers=2,)\n",
        "\n",
        "testset = datasets.SVHN(\"/content\", split='test', download = True, transform=transform_test)\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
        "                                         shuffle=False, num_workers=2)\n",
        "\n",
        "#create subsets\n",
        "dataSizeConstant = 0.25\n",
        "valDataFraction = 0.1\n",
        "subset = np.random.permutation([i for i in range(len(trainset))])\n",
        "subTrain = subset[:int(len(trainset) * (dataSizeConstant))]\n",
        "subTrainSet = datasets.SVHN(\"/content\", split = \"train\", download = True, transform = transform_train)\n",
        "subTrainLoader = DataLoader(subTrainSet, batch_size = batch_size, shuffle= False, num_workers= 2, sampler = torch.utils.data.SubsetRandomSampler(subTrain))\n",
        "\n",
        "# subset = np.random.permutation([i for i in range(len(trainset))])\n",
        "# SubTest = subset[: int(len(trainset) * (dataSizeConstant * valDataFraction))]\n",
        "# subTestSet = datasets.SVHN(\"/content\", split = \"train\", download = True, transform = transform)\n",
        "# subTestLoader = DataLoader(subTestSet, batch_size = batch_size, shuffle= False, num_workers= 2, sampler = torch.utils.data.SubsetRandomSampler(SubTest))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQd7meXAkrzz"
      },
      "source": [
        "# define device \n",
        "device = torch.device(\"cuda:0\")\n",
        "\n",
        "# data for plotting purposes\n",
        "modelLoss = []\n",
        "\n",
        "# model\n",
        "# model = ResNet(BasicBlock, [2,2,2,2])\n",
        "model = ResNet18()\n",
        "model.to(device)\n",
        "\n",
        "opt = optim.Adam(model.parameters(), lr=0.0001, betas=(0.5, 0.999)) \n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "epochs = 100"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QbjU4HaWkNvP"
      },
      "source": [
        "#training starts\n",
        "\n",
        "def train(datasetLoader):\n",
        "  text = (\"Datasize: \" + str(dataSizeConstant) + \"\\n\")\n",
        "  file.write(text)\n",
        "  for epoch in range(epochs):\n",
        "    model.train()\n",
        "\n",
        "    running_loss = 0.0\n",
        "    total_train = 0\n",
        "    correct_train = 0\n",
        "    for i, data in enumerate(datasetLoader, 0):\n",
        "      dataiter = iter(datasetLoader)\n",
        "      inputs, labels = dataiter.next()\n",
        "      inputs, labels = inputs.to(device), labels.to(device)\n",
        "\n",
        "      opt.zero_grad()\n",
        "\n",
        "      outputs = model(inputs)\n",
        "      modelLoss = criterion(outputs, labels) # error line\n",
        "      modelLoss.backward()\n",
        "      opt.step()\n",
        "\n",
        "      model.eval()\n",
        "      # accuracy\n",
        "      _, predicted = torch.max(outputs, 1)\n",
        "      total_train += labels.size(0)\n",
        "      correct_train += predicted.eq(labels.data).sum().item()\n",
        "      train_accuracy = 100 * correct_train / total_train                                   \n",
        "\n",
        "      # save generated images \n",
        "      if(i % 1 == 0):\n",
        "        text = (\"Train Accuracy: \" + str(train_accuracy))\n",
        "        file.write(text + '\\n')\n",
        "\n",
        "\n",
        "\n",
        "    print(\"Epoch \" + str(epoch) + \"Complete\")\n",
        "    print(\"Loss: \" + str(modelLoss.item()))\n",
        "    validate()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fzdclBANkY_H"
      },
      "source": [
        "# validation \n",
        "def validate():\n",
        "  model.eval()\n",
        "  correct = 0\n",
        "  total = 0\n",
        "  with torch.no_grad():\n",
        "      for data in testloader:\n",
        "          inputs, labels = data\n",
        "          inputs, labels = data[0].to(device), data[1].to(device)\n",
        "          outputs = model(inputs)\n",
        "          _, predicted = torch.max(outputs.data, 1)\n",
        "          total += labels.size(0)\n",
        "          correct += (predicted == labels).sum().item()\n",
        "\n",
        "  accuracy = (correct / total) * 100 \n",
        "\n",
        "  print('Accuracy of the network on the 10000 test images: %d %%' % (\n",
        "      100 * correct / total))\n",
        "\n",
        "  text = (\"Test Accuracy: \" + str(accuracy) + \"\\n\")\n",
        "  file.write(text)\n",
        "  model.train()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwrkvQqBioF3"
      },
      "source": [
        "train(subTrainLoader)\n",
        "file.close() "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bCD4mztw9KQK"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}