Image_reconstruction.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DyJ7PvTOk_v6"
      },
      "source": [
        "#Kaggle Instruction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LNAUiDliMDW8"
      },
      "outputs": [],
      "source": [
        "from IPython.display import clear_output\n",
        "\n",
        "! pip install kaggle\n",
        "\n",
        "!mkdir ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "!chmod 600 ~/.kaggle/kaggle.json\n",
        "\n",
        "!pip install torchvision\n",
        "!kaggle competitions download -c cifar-10\n",
        "!unzip /content/cifar-10.zip\n",
        "\n",
        "!sudo apt-get install p7zip-full\n",
        "!7za x /content/train.7z\n",
        "\n",
        "clear_output()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u_L5X2UiZtFp"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt \n",
        "import seaborn as sns\n",
        "import cv2\n",
        "from PIL import Image\n",
        "import imageio\n",
        "import os\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "import torchvision as v\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "\n",
        "from tqdm import tqdm\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a80WaO-SiM-1"
      },
      "outputs": [],
      "source": [
        "!cp /content/train.zip /content/drive/MyDrive/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SUNxaizx9i5Y"
      },
      "source": [
        "# Data Generation\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sAq4niydZwb5"
      },
      "outputs": [],
      "source": [
        "Data_Labels= pd.DataFrame(columns=['First_image','Second_image','Target'])\n",
        "if not os.path.isdir('/content/Cifar_Multi_Class_Reconstruction'):\n",
        "    os.mkdir('/content/Cifar_Multi_Class_Reconstruction/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X6DSNsqs_PGw"
      },
      "outputs": [],
      "source": [
        "def dataset_generation(data_Path,destination,csv_destination):\n",
        "    image_names = []\n",
        "    image = []\n",
        "\n",
        "    for i in sorted(os.listdir(data_Path)):\n",
        "        image_names.append(imageio.imread(os.path.join('/content/train',i)))\n",
        "        image.append(i)\n",
        "\n",
        "    for i in tqdm(range(10)):\n",
        "      for k in ((range(100*i,(100)*(i+1)))):\n",
        "          for j in range(k,100*(i+1)):\n",
        "            combine_image = ((image_names[k] + image_names[j])/2)\n",
        "            combine_image.astype(int)\n",
        "            combine_image = Image.fromarray(combine_image.astype(np.uint8))\n",
        "            combine_image.save(os.path.join(destination,\"{}_{}.jpg\".format(image[k],image[j])))\n",
        "            Data_Labels.loc[len(Data_Labels.index)] = [image[k],image[j],\"{}_{}.jpg\".format(image[k],image[j])] \n",
        "    Data_Labels.to_csv(csv_destination)\n",
        "    return image_names "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "72YbjG3ya5Iw"
      },
      "outputs": [],
      "source": [
        "Data_Labels.to_csv('/content/Cifar_Multi_Class_Reconstruction.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b8VkUATTir_N"
      },
      "outputs": [],
      "source": [
        "dataset_generation('/content/train','/content/Cifar_Multi_Class_Reconstruction/','/content/Cifar_Multi_Class_Reconstruction.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V-wEQ5GmixKa"
      },
      "outputs": [],
      "source": [
        "import shutil\n",
        "des = '/content/drive/MyDrive/Cifar-1000-image-combination'\n",
        "# a = []\n",
        "for i in os.listdir('/content/Cifar_Multi_Class_Reconstruction'):\n",
        "  # a.append(os.path.join('/content/Cifar_Multi_Class_Reconstruction',i))\n",
        "  shutil.copy(os.path.join('/content/Cifar_Multi_Class_Reconstruction',i), des)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "92EwKpJgX-Ir",
        "outputId": "83c7bda8-3128-4fd9-ee08-3221e4ba3b3d"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/Cifar-1000-image-combination.zip'"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import shutil\n",
        "shutil.make_archive('/content/Cifar-1000-image-combination', 'zip', '/content/Cifar_Multi_Class_Reconstruction/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mhR6JDgIVTd8"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import glob\n",
        "\n",
        "files = glob.glob('/content/drive/MyDrive/Cifar-1000-image-combination/*')\n",
        "for f in files:\n",
        "    os.remove(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iikcA5kwumf0",
        "outputId": "0fda6dc3-6b99-4e25-a7e4-8aff4f31fe56"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "6810"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import shutil\n",
        "import os\n",
        "# des = '/content/drive/MyDrive/Cifar-1000-image-combination'\n",
        "a = []\n",
        "len(os.listdir('/content/train'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JbbQn3IJwwR-",
        "outputId": "fffe488a-7ed5-43a2-c119-d102c74639b6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  Cifar-1000-image-combination.zip\n",
            "  End-of-central-directory signature not found.  Either this file is not\n",
            "  a zipfile, or it constitutes one disk of a multi-part archive.  In the\n",
            "  latter case the central directory and zipfile comment will be found on\n",
            "  the last disk(s) of this archive.\n",
            "unzip:  cannot find zipfile directory in one of Cifar-1000-image-combination.zip or\n",
            "        Cifar-1000-image-combination.zip.zip, and cannot find Cifar-1000-image-combination.zip.ZIP, period.\n"
          ]
        }
      ],
      "source": [
        "!unzip Cifar-1000-image-combination.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uGIC-8wxw4k1"
      },
      "outputs": [],
      "source": [
        "ls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JGXaHcEVisLe"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "29-Le8azitBF"
      },
      "source": [
        "#Data Prepration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rnf2uA36a-Jh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ace34a25-55f2-45e9-c75c-0e631a692247"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ro0Yov2uAHhS"
      },
      "source": [
        "# DataLoader\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V4sFNH85-IJ3"
      },
      "outputs": [],
      "source": [
        "!unzip '/content/drive/MyDrive/Cifar-1000-image-combination.zip' -d '/content/Cifar_Multi_Class_Reconstruction/'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 142,
      "metadata": {
        "id": "4ApIOvqiTMze"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset\n",
        "from torchvision.io import read_image\n",
        "import pandas as pd\n",
        "import torchvision\n",
        "import os\n",
        "\n",
        "from torchvision import transforms\n",
        "from torchvision import transforms, utils\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "root_directory = '/content/'\n",
        "\n",
        "class ImageDataLoader(Dataset):\n",
        "\n",
        "    def __init__(self , image_directory, label_directory,transform=None):\n",
        "        self.image_labels= pd.read_csv(label_directory)\n",
        "        self.image_directory = image_directory\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.image_labels.index)\n",
        "\n",
        "    def __getitem__(self , index):\n",
        "        # print(self.image_labels.iloc[index,3])\n",
        "        image_path = os.path.join(self.image_directory,str(self.image_labels.iloc[index,3]))\n",
        "        image = read_image(image_path)\n",
        "        image_label_1 =read_image(os.path.join('/content/train',str(self.image_labels.iloc[index,1])))\n",
        "        image_label_2 =read_image(os.path.join('/content/train',str(self.image_labels.iloc[index,2])))\n",
        "\n",
        "        # image = torch.from_numpy(image)\n",
        "        # image_label_1 = torch.from_numpy(image_label_1)\n",
        "        # image_label_2 = torch.from_numpy(image_label_2)b\n",
        "        transform1 = transforms.Compose([transforms.ToPILImage()])\n",
        "        image = transform1(image)\n",
        "        image_label_1 = transform1(image_label_1)\n",
        "        image_label_2 = transform1(image_label_2)\n",
        "\n",
        "        if self.transform != None :\n",
        "          image = self.transform(image)\n",
        "          image_label_1 = self.transform(image_label_1)\n",
        "          image_label_2 = self.transform(image_label_2)\n",
        "        c = torch.cat((image_label_1, image_label_2), 1)\n",
        "        return image , c\n",
        "        # return image , image"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 143,
      "metadata": {
        "id": "IWG8CXwNXi8Y"
      },
      "outputs": [],
      "source": [
        "composed = transforms.Compose([transforms.Resize(32),\n",
        "  #  transforms.CenterCrop(15),\n",
        "   transforms.ToTensor(),\n",
        "  #  transforms.Normalize(\n",
        "      #  mean=[0.485, 0.456, 0.406],\n",
        "      #  std=[0.229, 0.224, 0.225]\n",
        "  #  )]\n",
        "])\n",
        "# preprocess = T.Compose([\n",
        "  #  T.Resize(256),\n",
        "  #  T.CenterCrop(224),\n",
        "  #  T.ToTensor(),\n",
        "  #  T.Normalize(\n",
        "      #  mean=[0.485, 0.456, 0.406],\n",
        "      #  std=[0.229, 0.224, 0.225]\n",
        "  #  )\n",
        "# ])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 144,
      "metadata": {
        "id": "FNI3WdrgH1XK"
      },
      "outputs": [],
      "source": [
        "train = ImageDataLoader('/content/Cifar_Multi_Class_Reconstruction','/content/Cifar_Multi_Class_Reconstruction.csv',composed)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_set, val_set = torch.utils.data.random_split(train, [45000, 5500])\n"
      ],
      "metadata": {
        "id": "Izj3PGCtfEwH"
      },
      "execution_count": 145,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 146,
      "metadata": {
        "id": "TuQqkEzRIDs7"
      },
      "outputs": [],
      "source": [
        "\n",
        "# train_dataloader = DataLoader(train, batch_size=32 ,shuffle=True)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataloader = DataLoader(train_set, batch_size=32 ,shuffle=True)\n",
        "valid_loader = DataLoader(val_set, batch_size=32 ,shuffle=True)"
      ],
      "metadata": {
        "id": "P7VfXlIEfPq6"
      },
      "execution_count": 147,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "5nPqhWPPe7uu"
      },
      "execution_count": 147,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 148,
      "metadata": {
        "id": "n3PBkOaXGKpa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2524b199-b6c3-457a-e550-5c828a81d66b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[tensor([[[[0.3255, 0.4275, 0.3451,  ..., 0.1490, 0.1412, 0.2392],\n",
              "           [0.3098, 0.3686, 0.2588,  ..., 0.2275, 0.2275, 0.2196],\n",
              "           [0.2039, 0.2863, 0.2431,  ..., 0.0784, 0.1686, 0.2039],\n",
              "           ...,\n",
              "           [0.0000, 0.0000, 0.5176,  ..., 0.0824, 0.0196, 0.0824],\n",
              "           [0.4549, 0.0000, 0.0000,  ..., 0.0275, 0.0039, 0.4863],\n",
              "           [0.0000, 0.2314, 0.3137,  ..., 0.4235, 0.0549, 0.0157]],\n",
              " \n",
              "          [[0.2471, 0.3490, 0.2745,  ..., 0.1451, 0.1373, 0.2353],\n",
              "           [0.2353, 0.2941, 0.1961,  ..., 0.2235, 0.2235, 0.2157],\n",
              "           [0.1451, 0.2314, 0.1961,  ..., 0.0784, 0.1686, 0.2039],\n",
              "           ...,\n",
              "           [0.0510, 0.0667, 0.5765,  ..., 0.0863, 0.0157, 0.0784],\n",
              "           [0.5647, 0.0353, 0.0353,  ..., 0.0314, 0.0000, 0.4824],\n",
              "           [0.0667, 0.3412, 0.3843,  ..., 0.4275, 0.0510, 0.0118]],\n",
              " \n",
              "          [[0.2549, 0.3569, 0.2745,  ..., 0.1765, 0.1608, 0.2588],\n",
              "           [0.2431, 0.3020, 0.2000,  ..., 0.2471, 0.2431, 0.2353],\n",
              "           [0.1569, 0.2314, 0.1961,  ..., 0.0863, 0.1686, 0.2039],\n",
              "           ...,\n",
              "           [0.0588, 0.0784, 0.6039,  ..., 0.0667, 0.0000, 0.0627],\n",
              "           [0.5098, 0.0000, 0.0039,  ..., 0.0118, 0.0000, 0.4667],\n",
              "           [0.0000, 0.2549, 0.3294,  ..., 0.4078, 0.0353, 0.0000]]],\n",
              " \n",
              " \n",
              "         [[[0.1098, 0.1059, 0.1333,  ..., 0.4549, 0.4118, 0.3882],\n",
              "           [0.2824, 0.3098, 0.3137,  ..., 0.4235, 0.4039, 0.4157],\n",
              "           [0.2941, 0.3137, 0.3333,  ..., 0.4196, 0.4078, 0.4157],\n",
              "           ...,\n",
              "           [0.2000, 0.1765, 0.1882,  ..., 0.2196, 0.2118, 0.2078],\n",
              "           [0.1333, 0.1922, 0.1765,  ..., 0.1725, 0.1216, 0.0784],\n",
              "           [0.2627, 0.3255, 0.1373,  ..., 0.2431, 0.2039, 0.1647]],\n",
              " \n",
              "          [[0.1412, 0.1294, 0.1373,  ..., 0.4235, 0.3529, 0.3255],\n",
              "           [0.2941, 0.3176, 0.3020,  ..., 0.3922, 0.3569, 0.3608],\n",
              "           [0.2706, 0.2863, 0.3020,  ..., 0.3882, 0.3608, 0.3686],\n",
              "           ...,\n",
              "           [0.1176, 0.1059, 0.1373,  ..., 0.1961, 0.1922, 0.1922],\n",
              "           [0.1412, 0.2000, 0.1725,  ..., 0.1686, 0.1059, 0.0627],\n",
              "           [0.3137, 0.3647, 0.1529,  ..., 0.2471, 0.1843, 0.1451]],\n",
              " \n",
              "          [[0.0000, 0.0000, 0.0667,  ..., 0.3412, 0.2627, 0.2235],\n",
              "           [0.1255, 0.1725, 0.2275,  ..., 0.3176, 0.2627, 0.2549],\n",
              "           [0.1373, 0.1725, 0.2118,  ..., 0.3137, 0.2745, 0.2667],\n",
              "           ...,\n",
              "           [0.2392, 0.2078, 0.2039,  ..., 0.0941, 0.1686, 0.1961],\n",
              "           [0.2235, 0.2588, 0.1961,  ..., 0.0510, 0.1020, 0.1216],\n",
              "           [0.3765, 0.4039, 0.1569,  ..., 0.1216, 0.2000, 0.2314]]],\n",
              " \n",
              " \n",
              "         [[[0.1255, 0.1255, 0.1255,  ..., 0.0235, 0.0078, 0.0000],\n",
              "           [0.1216, 0.1333, 0.1373,  ..., 0.0431, 0.0275, 0.0157],\n",
              "           [0.1098, 0.1333, 0.1490,  ..., 0.0824, 0.0627, 0.0549],\n",
              "           ...,\n",
              "           [0.0000, 0.0196, 0.0000,  ..., 0.1137, 0.2667, 0.2157],\n",
              "           [0.0000, 0.0824, 0.0510,  ..., 0.1725, 0.2431, 0.2667],\n",
              "           [0.0392, 0.0039, 0.0000,  ..., 0.0000, 0.0000, 0.1608]],\n",
              " \n",
              "          [[0.1412, 0.1373, 0.1412,  ..., 0.0588, 0.0510, 0.0471],\n",
              "           [0.1294, 0.1412, 0.1529,  ..., 0.0588, 0.0549, 0.0510],\n",
              "           [0.1176, 0.1373, 0.1569,  ..., 0.0627, 0.0549, 0.0549],\n",
              "           ...,\n",
              "           [0.4353, 0.4980, 0.4392,  ..., 0.4314, 0.5490, 0.4824],\n",
              "           [0.4627, 0.5529, 0.5176,  ..., 0.3765, 0.4667, 0.4941],\n",
              "           [0.5059, 0.4706, 0.4196,  ..., 0.0863, 0.1765, 0.3686]],\n",
              " \n",
              "          [[0.1529, 0.1569, 0.1882,  ..., 0.0000, 0.0000, 0.0000],\n",
              "           [0.1098, 0.1294, 0.1647,  ..., 0.0000, 0.0000, 0.0000],\n",
              "           [0.0314, 0.0667, 0.1059,  ..., 0.0784, 0.0980, 0.1020],\n",
              "           ...,\n",
              "           [0.3608, 0.4000, 0.3098,  ..., 0.5490, 0.5961, 0.4863],\n",
              "           [0.4275, 0.5059, 0.4392,  ..., 0.5294, 0.5451, 0.5412],\n",
              "           [0.4980, 0.4471, 0.3647,  ..., 0.2510, 0.2784, 0.4392]]],\n",
              " \n",
              " \n",
              "         ...,\n",
              " \n",
              " \n",
              "         [[[0.3451, 0.3647, 0.3608,  ..., 0.3255, 0.3412, 0.2941],\n",
              "           [0.3804, 0.4039, 0.3765,  ..., 0.3333, 0.3216, 0.3098],\n",
              "           [0.4235, 0.4392, 0.4039,  ..., 0.3333, 0.3098, 0.3176],\n",
              "           ...,\n",
              "           [0.2431, 0.2980, 0.3294,  ..., 0.5216, 0.4275, 0.3255],\n",
              "           [0.2902, 0.3255, 0.3333,  ..., 0.3725, 0.3843, 0.3804],\n",
              "           [0.3490, 0.3608, 0.3451,  ..., 0.3412, 0.3412, 0.3647]],\n",
              " \n",
              "          [[0.2745, 0.3059, 0.3176,  ..., 0.2902, 0.2745, 0.2118],\n",
              "           [0.3137, 0.3451, 0.3412,  ..., 0.2980, 0.2549, 0.2235],\n",
              "           [0.3686, 0.4000, 0.3686,  ..., 0.2941, 0.2471, 0.2431],\n",
              "           ...,\n",
              "           [0.2549, 0.3098, 0.3294,  ..., 0.5176, 0.4000, 0.2902],\n",
              "           [0.3020, 0.3373, 0.3373,  ..., 0.3569, 0.3451, 0.3294],\n",
              "           [0.3647, 0.3725, 0.3490,  ..., 0.3216, 0.2902, 0.3098]],\n",
              " \n",
              "          [[0.2353, 0.2863, 0.3490,  ..., 0.2314, 0.3059, 0.2863],\n",
              "           [0.2745, 0.3255, 0.3608,  ..., 0.2392, 0.2941, 0.3059],\n",
              "           [0.3176, 0.3647, 0.3804,  ..., 0.2471, 0.2902, 0.3294],\n",
              "           ...,\n",
              "           [0.1804, 0.2431, 0.2824,  ..., 0.4078, 0.3765, 0.3098],\n",
              "           [0.2196, 0.2627, 0.2824,  ..., 0.3098, 0.3765, 0.4000],\n",
              "           [0.2745, 0.2902, 0.2941,  ..., 0.2980, 0.3569, 0.4078]]],\n",
              " \n",
              " \n",
              "         [[[0.2078, 0.2118, 0.2118,  ..., 0.1961, 0.1882, 0.1843],\n",
              "           [0.1804, 0.1804, 0.1804,  ..., 0.1686, 0.1647, 0.1608],\n",
              "           [0.1490, 0.1490, 0.1490,  ..., 0.1373, 0.1294, 0.1294],\n",
              "           ...,\n",
              "           [0.3843, 0.4118, 0.3529,  ..., 0.1490, 0.1451, 0.1176],\n",
              "           [0.2392, 0.1569, 0.2039,  ..., 0.2039, 0.1608, 0.1255],\n",
              "           [0.1608, 0.1020, 0.2824,  ..., 0.0706, 0.0706, 0.1137]],\n",
              " \n",
              "          [[0.2784, 0.2863, 0.2863,  ..., 0.2588, 0.2549, 0.2510],\n",
              "           [0.2510, 0.2510, 0.2549,  ..., 0.2314, 0.2314, 0.2275],\n",
              "           [0.2078, 0.2078, 0.2118,  ..., 0.1961, 0.1882, 0.1882],\n",
              "           ...,\n",
              "           [0.4196, 0.4706, 0.4588,  ..., 0.0588, 0.0863, 0.0667],\n",
              "           [0.4314, 0.3608, 0.4118,  ..., 0.1490, 0.0902, 0.0431],\n",
              "           [0.4392, 0.3765, 0.5412,  ..., 0.0314, 0.0000, 0.0157]],\n",
              " \n",
              "          [[0.3569, 0.3529, 0.3412,  ..., 0.3490, 0.3333, 0.3216],\n",
              "           [0.3451, 0.3373, 0.3216,  ..., 0.3216, 0.3098, 0.2980],\n",
              "           [0.3294, 0.3216, 0.3098,  ..., 0.2863, 0.2706, 0.2706],\n",
              "           ...,\n",
              "           [0.4157, 0.4510, 0.4235,  ..., 0.4157, 0.4118, 0.3765],\n",
              "           [0.4471, 0.3529, 0.3882,  ..., 0.4902, 0.4784, 0.4549],\n",
              "           [0.4549, 0.3765, 0.5255,  ..., 0.3686, 0.4157, 0.4784]]],\n",
              " \n",
              " \n",
              "         [[[0.0902, 0.1529, 0.1765,  ..., 0.2863, 0.3333, 0.3765],\n",
              "           [0.0824, 0.2471, 0.2314,  ..., 0.2078, 0.2392, 0.2667],\n",
              "           [0.0314, 0.1333, 0.1804,  ..., 0.1373, 0.1490, 0.1608],\n",
              "           ...,\n",
              "           [0.0784, 0.0000, 0.0510,  ..., 0.2157, 0.2275, 0.2471],\n",
              "           [0.0000, 0.0196, 0.1451,  ..., 0.2784, 0.3059, 0.2863],\n",
              "           [0.0667, 0.1098, 0.0000,  ..., 0.2941, 0.2745, 0.2588]],\n",
              " \n",
              "          [[0.0980, 0.1529, 0.1804,  ..., 0.3216, 0.3765, 0.4157],\n",
              "           [0.0824, 0.2431, 0.2235,  ..., 0.2471, 0.2824, 0.3098],\n",
              "           [0.0118, 0.1098, 0.1608,  ..., 0.1765, 0.1922, 0.2039],\n",
              "           ...,\n",
              "           [0.0706, 0.0000, 0.0314,  ..., 0.1961, 0.2000, 0.2196],\n",
              "           [0.0000, 0.0039, 0.1294,  ..., 0.2510, 0.2706, 0.2510],\n",
              "           [0.0392, 0.0863, 0.0000,  ..., 0.2667, 0.2392, 0.2235]],\n",
              " \n",
              "          [[0.0941, 0.1843, 0.2627,  ..., 0.3020, 0.3608, 0.4118],\n",
              "           [0.0824, 0.2745, 0.3098,  ..., 0.2157, 0.2588, 0.2941],\n",
              "           [0.0275, 0.1490, 0.2510,  ..., 0.1412, 0.1608, 0.1725],\n",
              "           ...,\n",
              "           [0.2235, 0.1333, 0.1922,  ..., 0.1725, 0.1765, 0.1961],\n",
              "           [0.2275, 0.2353, 0.3294,  ..., 0.2196, 0.2431, 0.2235],\n",
              "           [0.3412, 0.3608, 0.1961,  ..., 0.2353, 0.2118, 0.1961]]]]),\n",
              " tensor([[[[0.8941, 0.9686, 0.7686,  ..., 0.6941, 0.6549, 0.7137],\n",
              "           [0.8353, 0.9294, 0.7451,  ..., 0.6863, 0.5765, 0.6824],\n",
              "           [0.7451, 0.8000, 0.7020,  ..., 0.5333, 0.5569, 0.6078],\n",
              "           ...,\n",
              "           [0.6745, 0.6118, 0.5569,  ..., 0.6549, 0.6353, 0.6706],\n",
              "           [0.5490, 0.6039, 0.6863,  ..., 0.5882, 0.5922, 0.5098],\n",
              "           [0.6471, 0.6353, 0.6078,  ..., 0.4941, 0.5804, 0.6078]],\n",
              " \n",
              "          [[0.8980, 0.9686, 0.7922,  ..., 0.7098, 0.6745, 0.7608],\n",
              "           [0.8314, 0.9255, 0.7608,  ..., 0.6980, 0.5961, 0.7333],\n",
              "           [0.7333, 0.8000, 0.7020,  ..., 0.5843, 0.6196, 0.6863],\n",
              "           ...,\n",
              "           [0.6588, 0.6000, 0.5529,  ..., 0.6549, 0.6314, 0.6667],\n",
              "           [0.5412, 0.5922, 0.6667,  ..., 0.5843, 0.5882, 0.5059],\n",
              "           [0.6314, 0.6118, 0.5765,  ..., 0.4902, 0.5765, 0.6078]],\n",
              " \n",
              "          [[0.8471, 0.9176, 0.7294,  ..., 0.6745, 0.6510, 0.7647],\n",
              "           [0.7647, 0.8745, 0.7216,  ..., 0.6863, 0.6039, 0.7529],\n",
              "           [0.6510, 0.7373, 0.6863,  ..., 0.5725, 0.6314, 0.7059],\n",
              "           ...,\n",
              "           [0.6118, 0.5529, 0.4902,  ..., 0.6824, 0.6510, 0.6784],\n",
              "           [0.4863, 0.5647, 0.6549,  ..., 0.5804, 0.5843, 0.4902],\n",
              "           [0.6314, 0.5961, 0.5412,  ..., 0.4706, 0.5765, 0.6000]]],\n",
              " \n",
              " \n",
              "         [[[0.9216, 0.9098, 0.9098,  ..., 0.1961, 0.2078, 0.1843],\n",
              "           [0.9569, 0.9373, 0.9333,  ..., 0.1647, 0.1569, 0.1451],\n",
              "           [0.9059, 0.9176, 0.9333,  ..., 0.1569, 0.1412, 0.1333],\n",
              "           ...,\n",
              "           [0.4902, 0.4902, 0.5569,  ..., 0.5765, 0.6000, 0.5569],\n",
              "           [0.5020, 0.5294, 0.6000,  ..., 0.6196, 0.6078, 0.5608],\n",
              "           [0.5294, 0.5569, 0.5686,  ..., 0.6314, 0.6000, 0.5490]],\n",
              " \n",
              "          [[0.8980, 0.8863, 0.8863,  ..., 0.2157, 0.2235, 0.2000],\n",
              "           [0.9333, 0.9137, 0.9098,  ..., 0.1843, 0.1725, 0.1569],\n",
              "           [0.8824, 0.8941, 0.9098,  ..., 0.1725, 0.1529, 0.1451],\n",
              "           ...,\n",
              "           [0.3882, 0.3804, 0.4431,  ..., 0.4784, 0.4941, 0.4431],\n",
              "           [0.4000, 0.4196, 0.4863,  ..., 0.5216, 0.5020, 0.4471],\n",
              "           [0.4275, 0.4471, 0.4510,  ..., 0.5137, 0.4863, 0.4431]],\n",
              " \n",
              "          [[0.9373, 0.9255, 0.9255,  ..., 0.1922, 0.2196, 0.2078],\n",
              "           [0.9725, 0.9529, 0.9490,  ..., 0.1686, 0.1765, 0.1725],\n",
              "           [0.9216, 0.9333, 0.9490,  ..., 0.1725, 0.1686, 0.1725],\n",
              "           ...,\n",
              "           [0.2353, 0.2275, 0.2706,  ..., 0.3451, 0.3647, 0.3020],\n",
              "           [0.2471, 0.2667, 0.3176,  ..., 0.3647, 0.3529, 0.3020],\n",
              "           [0.2745, 0.2941, 0.3020,  ..., 0.3647, 0.3373, 0.3059]]],\n",
              " \n",
              " \n",
              "         [[[0.5843, 0.5686, 0.5882,  ..., 0.3804, 0.3490, 0.3569],\n",
              "           [0.5686, 0.5961, 0.6431,  ..., 0.4314, 0.4275, 0.4235],\n",
              "           [0.5373, 0.5647, 0.5922,  ..., 0.4196, 0.4353, 0.4314],\n",
              "           ...,\n",
              "           [0.6392, 0.6392, 0.6353,  ..., 0.6392, 0.6314, 0.6314],\n",
              "           [0.6353, 0.6353, 0.6353,  ..., 0.6392, 0.6314, 0.6314],\n",
              "           [0.6314, 0.6314, 0.6353,  ..., 0.6353, 0.6314, 0.6314]],\n",
              " \n",
              "          [[0.5569, 0.5451, 0.5608,  ..., 0.3647, 0.3333, 0.3333],\n",
              "           [0.5020, 0.5255, 0.5686,  ..., 0.3804, 0.3765, 0.3647],\n",
              "           [0.4667, 0.4941, 0.5216,  ..., 0.3765, 0.3922, 0.3804],\n",
              "           ...,\n",
              "           [0.7020, 0.7020, 0.7020,  ..., 0.7059, 0.6980, 0.6980],\n",
              "           [0.6980, 0.6980, 0.6980,  ..., 0.7059, 0.6980, 0.6980],\n",
              "           [0.6941, 0.6980, 0.6980,  ..., 0.7020, 0.6980, 0.6980]],\n",
              " \n",
              "          [[0.5490, 0.5373, 0.5569,  ..., 0.3294, 0.2902, 0.2902],\n",
              "           [0.4784, 0.4902, 0.5294,  ..., 0.3255, 0.3059, 0.2902],\n",
              "           [0.4588, 0.4706, 0.4941,  ..., 0.3059, 0.3059, 0.2863],\n",
              "           ...,\n",
              "           [0.7647, 0.7647, 0.7647,  ..., 0.7686, 0.7608, 0.7608],\n",
              "           [0.7608, 0.7608, 0.7608,  ..., 0.7686, 0.7608, 0.7608],\n",
              "           [0.7569, 0.7608, 0.7608,  ..., 0.7647, 0.7608, 0.7608]]],\n",
              " \n",
              " \n",
              "         ...,\n",
              " \n",
              " \n",
              "         [[[0.5176, 0.5373, 0.5412,  ..., 0.4902, 0.4431, 0.4745],\n",
              "           [0.5333, 0.5020, 0.5529,  ..., 0.6039, 0.4353, 0.4235],\n",
              "           [0.5059, 0.5843, 0.5216,  ..., 0.4667, 0.5255, 0.4431],\n",
              "           ...,\n",
              "           [0.1804, 0.1412, 0.1412,  ..., 0.3647, 0.3451, 0.2902],\n",
              "           [0.2392, 0.1569, 0.1255,  ..., 0.2627, 0.2314, 0.2000],\n",
              "           [0.2039, 0.1451, 0.1294,  ..., 0.2196, 0.2000, 0.1843]],\n",
              " \n",
              "          [[0.4980, 0.5176, 0.5373,  ..., 0.4824, 0.4431, 0.4392],\n",
              "           [0.5098, 0.5059, 0.5647,  ..., 0.5569, 0.4392, 0.4235],\n",
              "           [0.4863, 0.5725, 0.5294,  ..., 0.4275, 0.5176, 0.4549],\n",
              "           ...,\n",
              "           [0.1765, 0.1333, 0.1333,  ..., 0.3451, 0.3255, 0.2667],\n",
              "           [0.2353, 0.1490, 0.1137,  ..., 0.2431, 0.2118, 0.1765],\n",
              "           [0.2000, 0.1373, 0.1176,  ..., 0.2000, 0.1804, 0.1608]],\n",
              " \n",
              "          [[0.4824, 0.4941, 0.5216,  ..., 0.4549, 0.4235, 0.4078],\n",
              "           [0.4549, 0.4706, 0.5098,  ..., 0.4627, 0.3843, 0.4235],\n",
              "           [0.3843, 0.5451, 0.5137,  ..., 0.3020, 0.4431, 0.4549],\n",
              "           ...,\n",
              "           [0.1137, 0.0784, 0.0863,  ..., 0.1765, 0.1725, 0.1373],\n",
              "           [0.1686, 0.1020, 0.0824,  ..., 0.1333, 0.1137, 0.0980],\n",
              "           [0.1333, 0.0941, 0.0980,  ..., 0.1451, 0.1333, 0.1255]]],\n",
              " \n",
              " \n",
              "         [[[0.5647, 0.5725, 0.5804,  ..., 0.5765, 0.5686, 0.5569],\n",
              "           [0.5725, 0.5765, 0.5843,  ..., 0.5882, 0.5804, 0.5725],\n",
              "           [0.5647, 0.5686, 0.5765,  ..., 0.5843, 0.5765, 0.5686],\n",
              "           ...,\n",
              "           [0.4353, 0.3922, 0.3765,  ..., 0.6118, 0.6078, 0.6157],\n",
              "           [0.4078, 0.4039, 0.3490,  ..., 0.6510, 0.6157, 0.6078],\n",
              "           [0.4745, 0.4235, 0.3647,  ..., 0.6078, 0.5961, 0.5725]],\n",
              " \n",
              "          [[0.6745, 0.6824, 0.6902,  ..., 0.6902, 0.6824, 0.6667],\n",
              "           [0.6902, 0.6941, 0.7020,  ..., 0.7020, 0.6941, 0.6824],\n",
              "           [0.6863, 0.6902, 0.6980,  ..., 0.6941, 0.6863, 0.6784],\n",
              "           ...,\n",
              "           [0.2980, 0.2627, 0.2588,  ..., 0.4353, 0.4275, 0.4431],\n",
              "           [0.2667, 0.2667, 0.2235,  ..., 0.4706, 0.4314, 0.4353],\n",
              "           [0.3333, 0.2863, 0.2392,  ..., 0.4314, 0.4157, 0.4000]],\n",
              " \n",
              "          [[0.7569, 0.7608, 0.7686,  ..., 0.7608, 0.7529, 0.7373],\n",
              "           [0.7686, 0.7725, 0.7804,  ..., 0.7725, 0.7647, 0.7529],\n",
              "           [0.7647, 0.7686, 0.7765,  ..., 0.7686, 0.7608, 0.7490],\n",
              "           ...,\n",
              "           [0.2196, 0.2000, 0.2078,  ..., 0.2784, 0.2902, 0.3098],\n",
              "           [0.1843, 0.2078, 0.1804,  ..., 0.3216, 0.3020, 0.3098],\n",
              "           [0.2196, 0.1961, 0.1686,  ..., 0.2824, 0.2863, 0.2824]]],\n",
              " \n",
              " \n",
              "         [[[0.3373, 0.5373, 0.4863,  ..., 0.8157, 0.8824, 0.9529],\n",
              "           [0.3176, 0.5804, 0.5686,  ..., 0.5373, 0.6706, 0.8000],\n",
              "           [0.2314, 0.3961, 0.4627,  ..., 0.4078, 0.4745, 0.5451],\n",
              "           ...,\n",
              "           [0.4510, 0.4549, 0.4745,  ..., 0.6039, 0.6471, 0.6784],\n",
              "           [0.4941, 0.5059, 0.5020,  ..., 0.6980, 0.7176, 0.7137],\n",
              "           [0.5216, 0.5176, 0.5255,  ..., 0.7333, 0.7255, 0.7176]],\n",
              " \n",
              "          [[0.4000, 0.5804, 0.5333,  ..., 0.8471, 0.8980, 0.9569],\n",
              "           [0.3882, 0.6275, 0.6196,  ..., 0.6314, 0.7333, 0.8235],\n",
              "           [0.3176, 0.4784, 0.5412,  ..., 0.5059, 0.5686, 0.6118],\n",
              "           ...,\n",
              "           [0.4314, 0.4314, 0.4471,  ..., 0.5529, 0.5961, 0.6314],\n",
              "           [0.4627, 0.4745, 0.4745,  ..., 0.6353, 0.6588, 0.6549],\n",
              "           [0.4902, 0.4902, 0.4980,  ..., 0.6706, 0.6627, 0.6549]],\n",
              " \n",
              "          [[0.3686, 0.5961, 0.5176,  ..., 0.8000, 0.8667, 0.9451],\n",
              "           [0.3529, 0.6431, 0.5882,  ..., 0.5294, 0.6314, 0.7725],\n",
              "           [0.2235, 0.4039, 0.4667,  ..., 0.4392, 0.4588, 0.5333],\n",
              "           ...,\n",
              "           [0.3961, 0.3882, 0.3961,  ..., 0.5098, 0.5569, 0.5922],\n",
              "           [0.4157, 0.4157, 0.4078,  ..., 0.5882, 0.6078, 0.6039],\n",
              "           [0.4392, 0.4275, 0.4235,  ..., 0.6196, 0.6118, 0.6039]]]])]"
            ]
          },
          "metadata": {},
          "execution_count": 148
        }
      ],
      "source": [
        "next(iter(train_dataloader))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9M9A3zj5EnvP"
      },
      "source": [
        "# Model Arch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XTaTYXUIDRW6"
      },
      "outputs": [],
      "source": [
        "\n",
        "class Block(nn.Module):\n",
        "    def __init__(self, in_ch, out_ch):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv2d(in_ch, out_ch, 3)\n",
        "        self.relu  = nn.ReLU()\n",
        "        self.conv2 = nn.Conv2d(out_ch, out_ch, 3)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        return self.conv2(self.relu(self.conv1(x)))\n",
        "\n",
        "\n",
        "class Encoder(nn.Module):\n",
        "    def __init__(self, chs=(3,64,128,256,512,1024)):\n",
        "        super().__init__()\n",
        "        self.enc_blocks = nn.ModuleList([Block(chs[i], chs[i+1]) for i in range(len(chs)-1)])\n",
        "        self.pool       = nn.MaxPool2d(2)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        ftrs = []\n",
        "        for block in self.enc_blocks:\n",
        "            x = block(x)\n",
        "            ftrs.append(x)\n",
        "            x = self.pool(x)\n",
        "        return ftrs\n",
        "\n",
        "\n",
        "class Decoder(nn.Module):\n",
        "    def __init__(self, chs=(1024, 512, 256, 128, 64)):\n",
        "        super().__init__()\n",
        "        self.chs         = chs\n",
        "        self.upconvs    = nn.ModuleList([nn.ConvTranspose2d(chs[i], chs[i+1], 2, 2) for i in range(len(chs)-1)])\n",
        "        self.dec_blocks = nn.ModuleList([Block(chs[i], chs[i+1]) for i in range(len(chs)-1)]) \n",
        "        \n",
        "    def forward(self, x, encoder_features):\n",
        "        for i in range(len(self.chs)-1):\n",
        "            x        = self.upconvs[i](x)\n",
        "            enc_ftrs = self.crop(encoder_features[i], x)\n",
        "            x        = torch.cat([x, enc_ftrs], dim=1)\n",
        "            x        = self.dec_blocks[i](x)\n",
        "        return x\n",
        "    \n",
        "    def crop(self, enc_ftrs, x):\n",
        "        _, _, H, W = x.shape\n",
        "        enc_ftrs   = torchvision.transforms.CenterCrop([H, W])(enc_ftrs)\n",
        "        return enc_ftrs\n",
        "\n",
        "\n",
        "class UNet(nn.Module):\n",
        "    def __init__(self, enc_chs=(3,64,128,256,512,1024), dec_chs=(1024, 512, 256, 128, 64), num_class=1, retain_dim=False, out_sz=(572,572)):\n",
        "        super().__init__()\n",
        "        self.encoder     = Encoder(enc_chs)\n",
        "        self.decoder     = Decoder(dec_chs)\n",
        "        self.head        = nn.Conv2d(dec_chs[-1], num_class, 1)\n",
        "        self.retain_dim  = retain_dim\n",
        "\n",
        "    def forward(self, x):\n",
        "        enc_ftrs = self.encoder(x)\n",
        "        out      = self.decoder(enc_ftrs[::-1][0], enc_ftrs[::-1][1:])\n",
        "        out      = self.head(out)\n",
        "        if self.retain_dim:\n",
        "            out = F.interpolate(out, out_sz)\n",
        "        return out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sgIYxGKhE6sz",
        "outputId": "b3c1de52-2bc4-48d3-d17c-520b72866999"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Autoencoder(\n",
              "  (enc1): Linear(in_features=1024, out_features=256, bias=True)\n",
              "  (enc2): Linear(in_features=256, out_features=128, bias=True)\n",
              "  (enc3): Linear(in_features=128, out_features=64, bias=True)\n",
              "  (enc4): Linear(in_features=64, out_features=32, bias=True)\n",
              "  (enc5): Linear(in_features=32, out_features=16, bias=True)\n",
              "  (dec1): Linear(in_features=16, out_features=32, bias=True)\n",
              "  (dec2): Linear(in_features=32, out_features=64, bias=True)\n",
              "  (dec3): Linear(in_features=64, out_features=128, bias=True)\n",
              "  (dec4): Linear(in_features=128, out_features=256, bias=True)\n",
              "  (dec5): Linear(in_features=256, out_features=2048, bias=True)\n",
              ")"
            ]
          },
          "execution_count": 295,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "net = Autoencoder()\n",
        "criterion = nn.MSELoss()\n",
        "optimizer = optim.Adam(net.parameters(), lr=0.01, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\n",
        "net.to('cuda:0')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "PFYrR4GRFbKs",
        "outputId": "0f94c7a7-eb5e-4193-bdcf-7330bc431d30"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([2, 3, 32, 32])\n"
          ]
        },
        {
          "ename": "RuntimeError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-296-fb7a2afd4853>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorchsummary\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torchsummary/torchsummary.py\u001b[0m in \u001b[0;36msummary\u001b[0;34m(model, input_size, batch_size, device)\u001b[0m\n\u001b[1;32m     70\u001b[0m     \u001b[0;31m# make a forward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m     \u001b[0;31m# print(x.shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m     \u001b[0;31m# remove these hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1188\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1189\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1190\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1191\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-287-5d3c26aabb49>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menc1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menc2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menc3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1206\u001b[0m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbw_hook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetup_input_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1208\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1209\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_global_forward_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1210\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (192x32 and 1024x256)"
          ]
        }
      ],
      "source": [
        "from torchsummary import summary\n",
        "\n",
        "summary(net,(3,32,32))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iB5cYoxyCIF1"
      },
      "source": [
        "#Simple MLP"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "plUaNfXnCKGH"
      },
      "outputs": [],
      "source": [
        "class Autoencoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Autoencoder, self).__init__()\n",
        "        # encoder\n",
        "        self.enc1 = nn.Linear(in_features=32*32, out_features=256)\n",
        "        self.enc2 = nn.Linear(in_features=256, out_features=128)\n",
        "        self.enc3 = nn.Linear(in_features=128, out_features=64)\n",
        "        self.enc4 = nn.Linear(in_features=64, out_features=32)\n",
        "        self.enc5 = nn.Linear(in_features=32, out_features=16)\n",
        "        # decoder \n",
        "        self.dec1 = nn.Linear(in_features=16, out_features=32)\n",
        "        self.dec2 = nn.Linear(in_features=32, out_features=64)\n",
        "        self.dec3 = nn.Linear(in_features=64, out_features=128)\n",
        "        self.dec4 = nn.Linear(in_features=128, out_features=256)\n",
        "        self.dec5 = nn.Linear(in_features=256, out_features=32*32*2)\n",
        "    def forward(self, x):\n",
        "        print(x.size())\n",
        "        x = F.relu(self.enc1(x))\n",
        "        x = F.relu(self.enc2(x))\n",
        "        x = F.relu(self.enc3(x))\n",
        "        x = F.relu(self.enc4(x))\n",
        "        x = F.relu(self.enc5(x))\n",
        "        x = F.relu(self.dec1(x))\n",
        "        x = F.relu(self.dec2(x))\n",
        "        x = F.relu(self.dec3(x))\n",
        "        x = F.relu(self.dec4(x))\n",
        "        x = F.relu(self.dec5(x))\n",
        "        return x\n",
        "net = Autoencoder()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7jIwwKUVLFtf"
      },
      "outputs": [],
      "source": [
        "class A(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(A, self).__init__()\n",
        "\n",
        "        \"\"\" encoder \"\"\"\n",
        "        self.conv1 = nn.Conv2d(3, 32, kernel_size=(3,3))\n",
        "        self.batchnorm1_e = nn.BatchNorm2d(32)\n",
        "\n",
        "        self.conv2 = nn.Conv2d(32, 64, kernel_size=(4, 4), stride=3)\n",
        "        self.batchnorm2_e = nn.BatchNorm2d(64)\n",
        "\n",
        "        self.conv3 = nn.Conv2d(64, 128, kernel_size=(3, 3), stride=3)\n",
        "        self.batchnorm3_e = nn.BatchNorm2d(128)\n",
        "\n",
        "        self.maxpool2x2 = nn.MaxPool2d(2)   # not in usage\n",
        "\n",
        "        \"\"\" decoder \"\"\"\n",
        "        self.upsample2x2 = nn.Upsample(scale_factor=2)   # not in usage\n",
        "\n",
        "        self.deconv1 = nn.ConvTranspose2d(128, 64, kernel_size=(3, 3), stride=3)\n",
        "        self.batchnorm1 = nn.BatchNorm2d(64)\n",
        "\n",
        "        self.deconv2 = nn.ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=3)\n",
        "        self.batchnorm2 = nn.BatchNorm2d(32)\n",
        "\n",
        "        self.deconv3 = nn.ConvTranspose2d(32, 6, kernel_size=(5, 5))\n",
        "        self.batchnorm3 = nn.BatchNorm2d(6)\n",
        "    \n",
        "\n",
        "    def forward(self, x, train_: bool=True, print_: bool=False, return_bottlenecks: bool=False):\n",
        "\n",
        "        \"\"\" encoder \"\"\"\n",
        "        x = self.conv1(x)\n",
        "        # print(x.size())\n",
        "        x = self.batchnorm1_e(x)\n",
        "        x = F.relu(x)\n",
        "\n",
        "        x = self.conv2(x)\n",
        "        x = self.batchnorm2_e(x)\n",
        "        x = F.relu(x)\n",
        "\n",
        "        x = self.conv3(x)\n",
        "        x = self.batchnorm3_e(x)\n",
        "        bottlenecks = F.relu(x)\n",
        "\n",
        "        \"\"\" decoder \"\"\"\n",
        "        x = self.deconv1(bottlenecks)\n",
        "        x = self.batchnorm1(x)\n",
        "        x = F.relu(x)\n",
        "\n",
        "        x = self.deconv2(x)\n",
        "        x = self.batchnorm2(x)\n",
        "        x = F.relu(x)\n",
        "\n",
        "        x = self.deconv3(x)\n",
        "        x = torch.sigmoid(x)\n",
        "\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bIfi7_l_LF00"
      },
      "outputs": [],
      "source": [
        "class ConvAutoencoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(ConvAutoencoder, self).__init__()\n",
        "       \n",
        "        #Encoder\n",
        "        self.conv1 = nn.Conv2d(3, 1024, 3, padding=1)  \n",
        "        self.conv2 = nn.Conv2d(1024, 512, 3, padding=1)\n",
        "        self.conv3 = nn.Conv2d(512, 256, 3, padding=1)\n",
        "        self.conv4 = nn.Conv2d(256, 128, 3, padding=1)\n",
        "        self.conv5 = nn.Conv2d(128, 64, 3, padding=1)\n",
        "\n",
        "        self.pool = nn.MaxPool2d(2, 2)\n",
        "       \n",
        "        #Decoder\n",
        "        # self.t_conv1 = nn.ConvTranspose2d(64, 128, 2, stride=2)\n",
        "        # self.t_conv2 = nn.ConvTranspose2d(128, 6, 2, stride=1)\n",
        "        self.t_conv1 = nn.ConvTranspose2d(64, 128, 2)\n",
        "        self.t_conv2 = nn.ConvTranspose2d(128, 256, 2, stride=2)\n",
        "        self.t_conv3 = nn.ConvTranspose2d(256, 512, 2, stride=2)\n",
        "        self.t_conv4 = nn.ConvTranspose2d(512, 1024, 2, stride=1)\n",
        "        self.t_conv5 = nn.ConvTranspose2d(1024, 6, 2)\n",
        "\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = F.relu(self.conv1(x))\n",
        "        x = self.pool(x)\n",
        "        x = F.relu(self.conv2(x))\n",
        "        x = F.relu(self.conv3(x))\n",
        "        x = F.relu(self.conv4(x))\n",
        "        x = self.pool(x)\n",
        "        x = F.relu(self.conv5(x))\n",
        "        x = self.pool(x)\n",
        "        x = F.relu(self.t_conv1(x))\n",
        "        x = F.relu(self.t_conv2(x))\n",
        "        x = F.relu(self.t_conv3(x))\n",
        "        x = F.relu(self.t_conv4(x))\n",
        "        x = F.relu(self.t_conv5(x))\n",
        "        \n",
        "        return x"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rFxbAQjZLwP7"
      },
      "source": [
        "# More Complex Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 100,
      "metadata": {
        "id": "Ki0z1fubLy-L"
      },
      "outputs": [],
      "source": [
        "class Encoder(nn.Module):\n",
        "\n",
        "    def __init__(self,\n",
        "                 num_input_channels : int,\n",
        "                 base_channel_size : int,\n",
        "                 latent_dim : int,\n",
        "                 act_fn : object = nn.LeakyReLU):\n",
        "        super().__init__()\n",
        "        c_hid = base_channel_size\n",
        "        self.net = nn.Sequential(\n",
        "            nn.Conv2d(num_input_channels, c_hid, kernel_size=3, padding=1, stride=2), # 32x32 => 16x16\n",
        "            nn.BatchNorm2d(c_hid),\n",
        "            act_fn(),\n",
        "            nn.Conv2d(c_hid, c_hid, kernel_size=3, padding=1),\n",
        "            nn.BatchNorm2d(c_hid),\n",
        "            act_fn(),\n",
        "            nn.Conv2d(c_hid, 2*c_hid, kernel_size=3, padding=1, stride=2), # 16x16 => 8x8\n",
        "            nn.BatchNorm2d(2*c_hid),\n",
        "            act_fn(),\n",
        "            nn.Conv2d(2*c_hid, 2*c_hid, kernel_size=3, padding=1),\n",
        "            nn.BatchNorm2d(2*c_hid),\n",
        "            act_fn(),\n",
        "            nn.Conv2d(2*c_hid, 2*c_hid, kernel_size=3, padding=1, stride=2), # 8x8 => 4x4\n",
        "            nn.BatchNorm2d(2*c_hid),\n",
        "            act_fn(),\n",
        "            nn.Flatten(), # Image grid to single feature vector\n",
        "            nn.Linear(2*16*c_hid, latent_dim),\n",
        "            nn.BatchNorm1d(latent_dim),\n",
        "            act_fn(),\n",
        "            nn.Linear(latent_dim, 16),\n",
        "            nn.BatchNorm1d(16),\n",
        "            act_fn(),\n",
        "            nn.Linear(16, 8),\n",
        "            nn.BatchNorm1d(8),\n",
        "            act_fn(),\n",
        "            nn.Linear(8, 4),\n",
        "            nn.BatchNorm1d(4),\n",
        "            act_fn()\n",
        "\n",
        "\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.net(x)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 108,
      "metadata": {
        "id": "3i1h75RgL1Mm"
      },
      "outputs": [],
      "source": [
        "class Decoder(nn.Module):\n",
        "\n",
        "    def __init__(self,\n",
        "                 num_input_channels : int,\n",
        "                 base_channel_size : int,\n",
        "                 latent_dim : int,\n",
        "                 act_fn : object = nn.LeakyReLU):\n",
        "        super().__init__()\n",
        "        c_hid = base_channel_size\n",
        "        self.linear = nn.Sequential(\n",
        "            nn.Linear(4, 8),\n",
        "            nn.BatchNorm1d(8),\n",
        "            act_fn(),\n",
        "            nn.Linear(8, 16),\n",
        "            nn.BatchNorm1d(16),\n",
        "            act_fn(),\n",
        "            nn.Linear(16, latent_dim),\n",
        "            nn.BatchNorm1d(latent_dim),\n",
        "            act_fn(),\n",
        "            nn.Linear(latent_dim, 2*16*c_hid),\n",
        "            nn.BatchNorm1d(2*16*c_hid),\n",
        "            act_fn()\n",
        "        )\n",
        "        self.net = nn.Sequential(\n",
        "            nn.ConvTranspose2d(2*c_hid, 2*c_hid, kernel_size=3, output_padding=1, padding=1, stride=2), # 4x4 => 8x8\n",
        "            nn.BatchNorm2d(2*c_hid),\n",
        "            act_fn(),\n",
        "            nn.Conv2d(2*c_hid, 2*c_hid, kernel_size=3, padding=1),\n",
        "            nn.BatchNorm2d(2*c_hid),\n",
        "            act_fn(),\n",
        "            nn.ConvTranspose2d(2*c_hid, c_hid, kernel_size=3, output_padding=1, padding=1, stride=2), # 8x8 => 16x16\n",
        "            nn.BatchNorm2d(c_hid),\n",
        "            act_fn(),\n",
        "            nn.Conv2d(c_hid, c_hid, kernel_size=3, padding=1),\n",
        "            nn.BatchNorm2d(c_hid),\n",
        "            act_fn(),\n",
        "            nn.ConvTranspose2d(c_hid, num_input_channels*2, kernel_size=3, output_padding=1, padding=1, stride=2), # 16x16 => 32x32\n",
        "            nn.Sigmoid() # The input images is scaled between -1 and 1, hence the output has to be bounded as well\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.linear(x)\n",
        "        x = x.reshape(x.shape[0], -1, 4, 4)\n",
        "        x = self.net(x)\n",
        "        return x\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 109,
      "metadata": {
        "id": "f8G1RVzTTfUJ"
      },
      "outputs": [],
      "source": [
        "encoder = Encoder(3, 32, 64)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 110,
      "metadata": {
        "id": "qWCfDP43L1VP"
      },
      "outputs": [],
      "source": [
        "class Autoencoder(nn.Module):\n",
        "\n",
        "    def __init__(self,\n",
        "                 base_channel_size: int,\n",
        "                 latent_dim: int,\n",
        "                 encoder_class : object = Encoder,\n",
        "                 decoder_class : object = Decoder,\n",
        "                 num_input_channels: int = 3,\n",
        "                 width: int = 32,\n",
        "                 height: int = 32):\n",
        "        super().__init__()\n",
        "        self.encoder = encoder_class\n",
        "        self.decoder = decoder_class(num_input_channels, base_channel_size, latent_dim)\n",
        "        self.example_input_array = torch.zeros(2, num_input_channels, width, height)\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        The forward function takes in an image and returns the reconstructed image\n",
        "        \"\"\"\n",
        "        z = self.encoder(x)\n",
        "        x_hat = self.decoder(z)\n",
        "        return x_hat"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "K5RxZ5YVpzna"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_OGfApyUL1a8"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.utils.data as Data\n",
        "import torchvision      \n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "class AutoEncoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(AutoEncoder, self).__init__()\n",
        "\n",
        "        \n",
        "        self.encoder = nn.Sequential(\n",
        "            nn.Conv2d(\n",
        "                in_channels=3,      \n",
        "                out_channels=16,    \n",
        "                kernel_size=3,      \n",
        "                stride=1,          \n",
        "                padding=1,      \n",
        "            ),      \n",
        "            nn.LeakyReLU(),    \n",
        "            nn.Conv2d(\n",
        "                in_channels=16,     \n",
        "                out_channels=32,    \n",
        "                kernel_size=3,      \n",
        "                stride=1,          \n",
        "                padding=1,      \n",
        "            ),     \n",
        "            nn.LeakyReLU(),    \n",
        "            nn.MaxPool2d(kernel_size=2),     \n",
        "            nn.Conv2d(\n",
        "                in_channels=32,     \n",
        "                out_channels=32,    \n",
        "                kernel_size=5,      \n",
        "                stride=1,          \n",
        "                padding=2,      \n",
        "            ),      \n",
        "            nn.LeakyReLU(),    \n",
        "            nn.Conv2d(\n",
        "                in_channels=32,     \n",
        "                out_channels=64,    \n",
        "                kernel_size=5,      \n",
        "                stride=1,          \n",
        "                padding=2,      \n",
        "            ),      \n",
        "            nn.LeakyReLU(),    \n",
        "            nn.MaxPool2d(kernel_size=2),    \n",
        "        )\n",
        "        \n",
        "        self.decoder = nn.Sequential(\n",
        "                nn.ConvTranspose2d(\n",
        "                in_channels=64,     \n",
        "                out_channels=32,    \n",
        "                kernel_size=2,      \n",
        "                stride=2,          \n",
        "                padding=0,      \n",
        "            ),     \n",
        "            nn.LeakyReLU(),        \n",
        "            nn.Conv2d(\n",
        "                in_channels=32,     \n",
        "                out_channels=32,    \n",
        "                kernel_size=5,      \n",
        "                stride=1,          \n",
        "                padding=2,      \n",
        "            ),      \n",
        "            nn.LeakyReLU(),    \n",
        "           nn.ConvTranspose2d(\n",
        "                in_channels=32,     \n",
        "                out_channels=16,    \n",
        "                kernel_size=5,      \n",
        "                stride=1,          \n",
        "                padding=2,      \n",
        "            ),\n",
        "            nn.Conv2d(\n",
        "                in_channels=16,     \n",
        "                out_channels=16,    \n",
        "                kernel_size=5,      \n",
        "                stride=1,          \n",
        "                padding=2,     \n",
        "            ),      \n",
        "            nn.LeakyReLU(),           \n",
        "             nn.ConvTranspose2d(\n",
        "                in_channels=16,     \n",
        "                out_channels=16,    \n",
        "                kernel_size=2,      \n",
        "                stride=2,          \n",
        "                padding=0,      \n",
        "            ), \n",
        "            nn.LeakyReLU(),\n",
        "            nn.Conv2d(\n",
        "                in_channels=16,     \n",
        "                out_channels=16,    \n",
        "                kernel_size=3,      \n",
        "                stride=1,          \n",
        "                padding=1,      \n",
        "            ),     \n",
        "            nn.LeakyReLU(),                \n",
        "           nn.ConvTranspose2d(\n",
        "                in_channels=16,     \n",
        "                out_channels=3,    \n",
        "                kernel_size=5,      \n",
        "                stride=1,          \n",
        "                padding=2,      \n",
        "            ),\n",
        "            nn.Conv2d(\n",
        "                in_channels=3,     \n",
        "                out_channels=6,    \n",
        "                kernel_size=3,      \n",
        "                stride=1,          \n",
        "                padding=1,      \n",
        "            ),    \n",
        "            nn.ReLU(),  \n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        encoded = self.encoder(x)\n",
        "        decoded = self.decoder(encoded)\n",
        "        return decoded"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "net =  Autoencoder(base_channel_size=32, latent_dim=64, encoder_class=encoder)\n",
        "optimizer = optim.Adam(net.parameters(), lr=0.1, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\n",
        "net.to('cuda:0')\n",
        "criterion = nn.MSELoss()\n"
      ],
      "metadata": {
        "id": "J_OKuscWp2zc"
      },
      "execution_count": 111,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_hist, acc_hist = [], []\n",
        "loss_hist_val, acc_hist_val = [], []\n",
        "\n",
        "for epoch in range(140):\n",
        "    net.train()\n",
        "\n",
        "    running_loss = 0.0\n",
        "    correct = 0\n",
        "    for data in train_dataloader:\n",
        "        batch, labels = data\n",
        "        batch = batch.float()\n",
        "        # labels[0] = torch.tensor(labels[0]).to('cuda:0')\n",
        "        # labels[1] = torch.tensor(labels[1]).to('cuda:0')\n",
        "\n",
        "        batch , labels = batch.to('cuda:0') , labels.to('cuda:0')\n",
        "        batch, labels = batch.cuda(), labels.cuda() # add this line\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        # optimizer2.zero_grad()\n",
        "\n",
        "        # outputs = net(batch)\n",
        "        output1 = outputs[:,3:,:,:]\n",
        "        output2 = outputs[:,:3,:,:]\n",
        "        # # print(outputs.size())\n",
        "\n",
        "        loss1 = criterion(output1, labels[:,:,32:,:].float())\n",
        "        loss2 = criterion(output2, labels[:,:,:32,:].float())\n",
        "        loss = loss1 + loss2\n",
        "\n",
        "        # outputs1 = net(batch)\n",
        "        # loss1 = criterion(outputs1, labels[:,:,32:,:].float())\n",
        "        # plt.imshow()\n",
        "        # loss = criterion(outputs1, labels.float())\n",
        "\n",
        "        loss.backward()\n",
        "\n",
        "        # outputs2 = net2(batch)\n",
        "        # loss2 = criterion(outputs1, labels[:,:,32:,:].float())\n",
        "        # loss2 = criterion(outputs1, labels.float())\n",
        "\n",
        "        # loss2.backward(retain_graph=True)\n",
        "\n",
        "        optimizer.step()\n",
        "        # optimizer2.step()\n",
        "\n",
        "        # compute training statistics\n",
        "        # _, predicted = torch.max(outputs, 1)\n",
        "        # print(loss)\n",
        "\n",
        "        # correct += (predicted == labels).sum().item()\n",
        "        running_loss += loss.item()\n",
        "\n",
        "    avg_loss = running_loss / len(train_set)\n",
        "    loss_hist.append(avg_loss)\n",
        "\n",
        "    # validation statistics\n",
        "    net.eval()\n",
        "    loss_val = 0.0\n",
        "    correct_val = 0\n",
        "    for data in valid_loader:\n",
        "      batch, labels = data\n",
        "        batch = batch.float()\n",
        "        batch , labels = batch.to('cuda:0') , labels.to('cuda:0')\n",
        "        batch, labels = batch.cuda(), labels.cuda() # add this line\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        output1 = outputs[:,3:,:,:]\n",
        "        output2 = outputs[:,:3,:,:]\n",
        "\n",
        "        loss1 = criterion(output1, labels[:,:,32:,:].float())\n",
        "        loss2 = criterion(output2, labels[:,:,:32,:].float())\n",
        "        loss = loss1 + loss2\n",
        "\n",
        "        loss = criterion(outputs1, labels.float())\n",
        "        loss_val += loss.item()\n",
        "    avg_loss_val = loss_val / len(val_set)\n",
        "    loss_hist_val.append(avg_loss_val)\n",
        "  net.train()\n",
        "  print('[epoch %d] loss: %.5f accuracy: %.4f val loss: %.5f val accuracy: %.4f' % (epoch + 1, avg_loss, avg_loss_val))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 130
        },
        "id": "PVz6rJunqEK1",
        "outputId": "d5b3f183-29ce-4a2e-f2ad-b2a282809474"
      },
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndentationError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<tokenize>\"\u001b[0;36m, line \u001b[0;32m78\u001b[0m\n\u001b[0;31m    net.train()\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mIndentationError\u001b[0m\u001b[0;31m:\u001b[0m unindent does not match any outer indentation level\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=10, verbose=True, min_lr=0)\n",
        "\n",
        "loss_hist, acc_hist = [], []\n",
        "loss_hist_val, acc_hist_val = [], []\n",
        "\n",
        "for epoch in range(10):\n",
        "  running_loss = 0.0\n",
        "  correct = 0\n",
        "  for data in train_dataloader:\n",
        "    batch, labels = data\n",
        "    batch = batch.float()\n",
        "    batch, labels = batch.to(device), labels.to(device)\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "    outputs = net(batch)\n",
        "    # loss = criterion(outputs, labels)\n",
        "\n",
        "    output1 = outputs[:,3:,:,:]\n",
        "    output2 = outputs[:,:3,:,:]\n",
        "    # # print(outputs.size())\n",
        "\n",
        "    loss1 = criterion(output1, labels[:,:,32:,:].float())\n",
        "    loss2 = criterion(output2, labels[:,:,:32,:].float())\n",
        "    loss = loss1 + loss2\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    # compute training statistics\n",
        "    # _, predicted = torch.max(outputs, 1)\n",
        "    # correct += (predicted == labels).sum().item()\n",
        "    running_loss += loss.item()\n",
        "\n",
        "  avg_loss = running_loss / len(train_set)\n",
        "  # avg_acc = correct /len(train_set)\n",
        "  loss_hist.append(avg_loss)\n",
        "  # acc_hist.append(avg_acc)\n",
        "\n",
        "  # validation statistics\n",
        "  net.eval()\n",
        "  with torch.no_grad():\n",
        "    loss_val = 0.0\n",
        "    correct_val = 0\n",
        "    for data in valid_loader:\n",
        "      batch, labels = data\n",
        "      batch = batch.float()\n",
        "      batch, labels = batch.to(device), labels.to(device)\n",
        "      outputs = net(batch)\n",
        "      # loss = criterion(outputs, labels)\n",
        "\n",
        "      output1 = outputs[:,3:,:,:]\n",
        "      output2 = outputs[:,:3,:,:]\n",
        "      # # print(outputs.size())\n",
        "\n",
        "      loss1 = criterion(output1, labels[:,:,32:,:].float())\n",
        "      loss2 = criterion(output2, labels[:,:,:32,:].float())\n",
        "      loss = loss1 + loss2\n",
        "      # loss = criterion(outputs, labels)\n",
        "      # _, predicted = torch.max(outputs, 1)\n",
        "      # correct_val += (predicted == labels).sum().item()\n",
        "      loss_val += loss.item()\n",
        "    avg_loss_val = loss_val / len(val_set)\n",
        "    # avg_acc_val = correct_val / len(val_set)\n",
        "    loss_hist_val.append(avg_loss_val)\n",
        "    # acc_hist_val.append(avg_acc_val)\n",
        "  net.train()\n",
        "\n",
        "  scheduler.step(avg_loss_val)\n",
        "  print('[epoch %d] loss: %.5f val loss: %.5f' % (epoch + 1, avg_loss, avg_loss_val))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rv3nVE-Ofx8L",
        "outputId": "0a777119-6429-4b38-c21e-5aafb4f561bd"
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[epoch 1] loss: 0.00380 val loss: 0.00360\n",
            "[epoch 2] loss: 0.00353 val loss: 0.00354\n",
            "[epoch 3] loss: 0.00346 val loss: 0.00343\n",
            "[epoch 4] loss: 0.00344 val loss: 0.00350\n",
            "[epoch 5] loss: 0.00344 val loss: 0.00352\n",
            "[epoch 6] loss: 0.00342 val loss: 0.00342\n",
            "[epoch 7] loss: 0.00341 val loss: 0.00339\n",
            "[epoch 8] loss: 0.00340 val loss: 0.00337\n",
            "[epoch 9] loss: 0.00340 val loss: 0.00350\n",
            "[epoch 10] loss: 0.00340 val loss: 0.00343\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.rcParams[\"figure.figsize\"] = [7.50, 3.50]\n",
        "plt.rcParams[\"figure.autolayout\"] = True\n",
        "line1, = plt.plot(loss_hist,label=\"train\")\n",
        "line2, = plt.plot(loss_hist_val,label=\"validation\")\n",
        "leg = plt.legend(loc='upper left')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 261
        },
        "id": "Gmw3my7Bp5-e",
        "outputId": "dd1d4dd1-e3d8-473a-9296-a78bb8edcd2f"
      },
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 540x252 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhQAAAD0CAYAAADDob9OAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV1dn+8e+TmSQkQAhKEhBElHmMwdZ5xgnECXCotg6tQ60d3r7a9tdWX21ttWhrHepYawVEEEEccEKpVRFQQCYVEGWUgEyBzFm/P9ZOCGEKmfY5yf25rnPlnL3X3mediMmdtZ+9ljnnEBEREamPmLA7ICIiItFPgUJERETqTYFCRERE6k2BQkREROpNgUJERETqLS7sDjSE9u3buy5duoTdDRERkWZv7ty5G51zmTW3N4tA0aVLF+bMmRN2N0RERJo9M/tqb9t1yUNERETqTYFCRERE6k2BQkREROqtWdRQ7E1paSmrV6+mqKgo7K40C0lJSeTk5BAfHx92V0REJAI120CxevVqWrduTZcuXTCzsLsT1ZxzbNq0idWrV9O1a9ewuyMiIhGoVpc8zGyomX1mZsvM7Na97E80s+eC/bPMrEu1fbcF2z8zszODbUlm9pGZzTezRWZ2e7X2p5rZx2Y2z8zeM7Mj6vLBioqKyMjIUJhoAGZGRkaGRntERGSfDhgozCwWeBA4C+gFjDazXjWaXQ1sds4dAdwH/Ck4thcwCugNDAUeCs5XDJzinOsPDACGmtkxwbkeBi5zzg0AxgK/qeuHU5hoOPpeiojI/tRmhCIPWOacW+GcKwHGA8NrtBkOPB08nwicav430HBgvHOu2Dn3JbAMyHNeQdA+PnhUrqPugLTgeTqwtg6fq14KS8pZuXEHpeUVTf3WIiIiUak2gSIbWFXt9epg217bOOfKgK1Axv6ONbNYM5sHbADecM7NCtpcA7xiZquBK4C799YpM7vOzOaY2Zz8/PxafIzaM4NtRaVs3llS53Ns2bKFhx566KCPO/vss9myZUud31dERCQMod026pwrDy5r5AB5ZtYn2PVT4GznXA7wFDBmH8c/6pzLdc7lZmbuMQNovSTFx5KSGMfmHSU45w58wF7sK1CUlZXt97hXXnmFNm3a1Ok9RUREwlKbQLEG6FTtdU6wba9tzCwOf6liU22Odc5tAWbg6ygygf7VRiueA75bq0/SwDJSEiguq2BH8f4DwL7ceuutLF++nAEDBnD00Udz/PHHM2zYMHr18uUn559/PoMHD6Z37948+uijVcd16dKFjRs3snLlSnr27Mm1115L7969OeOMMygsLGyQzyYiItLQanPb6Gygu5l1xYeBUcClNdpMBa4EPgAuAt52zjkzmwqMNbMxQBbQHfgoCA6lzrktZtYKOB1fyLkZSDezI51znwfbl9T3Q97+0iIWr9120MftLCkjNsZIjIvdY1+vrDR+d17vfR579913s3DhQubNm8c777zDOeecw8KFC6tuu3zyySdp164dhYWFHH300Vx44YVkZGTsdo4vvviCcePG8dhjj3HJJZcwadIkLr/88oP+HCIiIo3tgIHCOVdmZjcB04FY4Enn3CIzuwOY45ybCjwBPGNmy4Bv8aGDoN0EYDFQBtzonCs3s47A08EdHzHABOfcNAAzuxaYZGYV+IDxgwb+zLUWFxNDaUUFCUB973HIy8vbbQ6Hv/3tb0yePBmAVatW8cUXX+wRKLp27cqAAQMAGDx4MCtXrqxnL0RERBpHrSa2cs69ArxSY9tvqz0vAi7ex7F3AXfV2LYAGLiP9pOBybXpV23tbyRhf4pKy/n8m+10TE8is3VSvfqQkpJS9fydd97hzTff5IMPPiA5OZmTTjppr3M8JCYmVj2PjY3VJQ8REYlYWstjP5LiY0lJiOPbOhRntm7dmu3bt+9139atW2nbti3JycksXbqUDz/8sCG6KyIiEppmO/V2Q2mXmsCqb3eyo7iM1KTar2ORkZHBscceS58+fWjVqhWHHHJI1b6hQ4fyyCOP0LNnT4466iiOOeaY/ZxJREQk8lldb4uMJLm5uW7OnDm7bVuyZAk9e/as97krKhxL1m+jdWIcnTNSDnxAM9ZQ31MREYleZjbXOZdbc7sueRxATIzRNjmBrUVllGnmTBERkb1SoKiFdikJOOfqNXOmiIhIc6ZAUQu7ijNL6zxzpoiISHOmQFFL7VITKC4rZ0dxedhdERERiTgKFLWUnhRPbIzx7Q5d9hAREalJgaKWdhVnlqo4U0REpAYFioOwqziztMHPnZqaCsDatWu56KKL9trmpJNOoubtsTXdf//97Ny5s+q1lkMXEZGmoEBxEOozc2ZtZWVlMXHixDofXzNQaDl0ERFpCgoUB6ldSlCcWbL/4sxbb72VBx98sOr173//e+68805OPfVUBg0aRN++fZkyZcoex61cuZI+ffoAUFhYyKhRo+jZsycjRozYbS2P66+/ntzcXHr37s3vfvc7wC84tnbtWk4++WROPvlkYNdy6ABjxoyhT58+9OnTh/vvv7/q/bRMuoiI1FfLmHr71Vth/acNcqo2OBJKyinv0AdG/GWf7UaOHMktt9zCjTfeCMCECROYPn06N998M2lpaWzcuJFjjjmGYcOGYbb3tUwffvhhkpOTWbJkCQsWLGDQoEFV++666y7atWtHeXk5p556KgsWLODmm29mzJgxzJgxg/bt2+92rrlz5/LUU08xa9YsnHMMGTKEE088kbZt22qZdBERqTeNUBwkw4iLMUrKK/ZbnDlw4EA2bNjA2rVrmT9/Pm3btuXQQw/lV7/6Ff369eO0005jzZo1fPPNN/s8x8yZM6t+sffr149+/fpV7ZswYQKDBg1i4MCBLFq0iMWLF++33++99x4jRowgJSWF1NRULrjgAv7zn/8AWiZdRETqr2WMUJx1d4OezpWWs/ab7bidpWS2Ttxnu4svvpiJEyeyfv16Ro4cybPPPkt+fj5z584lPj6eLl267HXZ8gP58ssvuffee5k9ezZt27blqquuqtN5KmmZdBERqS+NUNRBUnwsybUozhw5ciTjx49n4sSJXHzxxWzdupUOHToQHx/PjBkz+Oqrr/b7PieccAJjx44FYOHChSxYsACAbdu2kZKSQnp6Ot988w2vvvpq1TH7Wjb9+OOP58UXX2Tnzp3s2LGDyZMnc/zxx9fl44uIiOyhZYxQNIJ2KQms3ryTHSXlpCbu/dvYu3dvtm/fTnZ2Nh07duSyyy7jvPPOo2/fvuTm5tKjR4/9vsf111/P97//fXr27EnPnj0ZPHgwAP3792fgwIH06NGDTp06ceyxx1Ydc9111zF06FCysrKYMWNG1fZBgwZx1VVXkZeXB8A111zDwIEDdXlDREQahJYvr6OqZc2T4uncLrnR3ieSaPlyERHR8uUNrGrmzELNnCkiIqJAUQ9tG3HmTBERkWjSrANFY1/OaVXL4szmoLl/PhERqZ9mGyiSkpLYtGlTo/8irJw5c+cBZs6MZs45Nm3aRFJSUthdERGRCNVs7/LIyclh9erV5OfnN+r7OOfI31rE9vWxtEtJaNT3ClNSUhI5OTlhd0NERCJUsw0U8fHxdO3atUne67kpCxk3exWzbjuVts04VIiIiOxLs73k0ZRGD+lMSVkFL3yyJuyuiIiIhEKBogH0ODSNgZ3bMO6jr1W8KCIiLZICRQMZndeZZRsKmPPV5rC7IiIi0uQUKBrIef2yaJ0Ux9hZX4fdFRERkSanQNFAWiXEMmJgNi9/uo4tO0vC7o6IiEiTUqBoQKOODoozP1ZxpoiItCwKFA2oV1YaAzqpOFNERFoeBYoGdmleZ77YUMBcFWeKiEgLokDRwM7t35HWiSrOFBGRlkWBooElJ8Rx/sBspqk4U0REWhAFikYwOs8XZ07WzJkiItJCKFA0gl5ZafRXcaaIiLQgtQoUZjbUzD4zs2Vmdute9iea2XPB/llm1qXavtuC7Z+Z2ZnBtiQz+8jM5pvZIjO7vVr7/5jZvOCx1sxerP/HbHqX5nXi828K+PhrFWeKiEjzd8BAYWaxwIPAWUAvYLSZ9arR7Gpgs3PuCOA+4E/Bsb2AUUBvYCjwUHC+YuAU51x/YAAw1MyOAXDOHe+cG+CcGwB8ALxQ/4/Z9M7rn0VqYhzPqjhTRERagNqMUOQBy5xzK5xzJcB4YHiNNsOBp4PnE4FTzcyC7eOdc8XOuS+BZUCe8wqC9vHBY7drA2aWBpwCROUIhS/OzOLlBevYurM07O6IiIg0qtoEimxgVbXXq4Nte23jnCsDtgIZ+zvWzGLNbB6wAXjDOTerxjnPB95yzm2r3UeJPKPzOlNcVsHkT1aH3RUREZFGFVpRpnOuPLiskQPkmVmfGk1GA+P2dbyZXWdmc8xsTn5+fmN2tc56Z6XTPyedcR+tUnGmiIg0a7UJFGuATtVe5wTb9trGzOKAdGBTbY51zm0BZuBrLAjO0R5/qeXlfXXKOfeocy7XOZebmZlZi48RjtF5nfnsm+18/PWWsLsiIiLSaGoTKGYD3c2sq5kl4Issp9ZoMxW4Mnh+EfC283+STwVGBXeBdAW6Ax+ZWaaZtQEws1bA6cDSaue7CJjmnCuq6weLFJXFmZo5U0REmrMDBoqgJuImYDqwBJjgnFtkZneY2bCg2RNAhpktA34G3BocuwiYACwGXgNudM6VAx2BGWa2AB9Y3nDOTav2tqPYz+WOaJKSGMfwAVlMW7BWxZkiItJsWXO4tp+bm+vmzJkTdjf2aeGarZz7wHvcPqw3V363S9jdERERqTMzm+ucy625XTNlNoE+2en0y0nXzJkiItJsKVA0kdF5nVm6fjufrFJxpoiIND8KFE3kvP5ZpCTEqjhTRESaJQWKJpKaGMfwgdm+OLNQxZkiItK8KFA0oUvzOlNUWsGUeVrWXEREmhcFiibUJzudvtnpjJ2l4kwREWleFCiaWGVx5jwVZ4qISDOiQNHEhg1QcaaIiDQ/ChRNLDUxjmEDsnlpwVq2Fak4U0REmgcFihBUFWd+ouJMERFpHhQoQtA3J50+2Wk8q+JMERFpJhQoQlJZnDl/9dawuyIiIlJvChQhGdY/i+SEWMbO+irsroiIiNSbAkVIWifFM3xAFi/NX6fiTBERiXoKFCEandeZwtJypsxbG3ZXRERE6kWBIkR9s9PpnZWmmTNFRCTqKVCEyMwYndeZJeu2sUDFmSIiEsUUKEI2fEBlcaZmzhQRkeilQBGy1knxDOufxdT5a9mu4kwREYlSChQRQMWZIiIS7RQoIkC/nHR6dVRxpoiIRC8FighgZowe0pnF67bx6RoVZ4qISPRRoIgQwwdk0So+lnEfqThTRESijwJFhEhLiue8/h2ZMk/FmSIiEn0UKCLIpUMOY2dJOVPnqzhTRESiiwJFBOmfk07Pjmm67CEiIlFHgSKCmBmX5nVi4ZptfKqZM0VEJIooUESY4QOzSYqPYaxGKUREJIooUESYtKR4zuuXxdR5aygoLgu7OyIiIrWiQBGBLh3SmR0l5UzVzJkiIhIlFCgi0IBObehxaGsVZ4qISNRQoIhAZsalQzrz6ZqtKs4UEZGooEARoYYP8MWZ42ZrlEJERCKfAkWESm8Vz7n9spjyiYozRUQk8ilQRLDK4syXNHOmiIhEOAWKCDZQxZkiIhIlFCgimJkxOq8zC1ZvZaGWNRcRkQhWq0BhZkPN7DMzW2Zmt+5lf6KZPRfsn2VmXartuy3Y/pmZnRlsSzKzj8xsvpktMrPbq7U3M7vLzD43syVmdnP9P2b0On9gNolxMRqlEBGRiHbAQGFmscCDwFlAL2C0mfWq0exqYLNz7gjgPuBPwbG9gFFAb2Ao8FBwvmLgFOdcf2AAMNTMjgnOdRXQCejhnOsJjK/XJ4xyVcWZ89ayQ8WZIiISoWozQpEHLHPOrXDOleB/wQ+v0WY48HTwfCJwqplZsH28c67YOfclsAzIc15B0D4+eLjg9fXAHc65CgDn3IY6frb62bIqlLfdm0uHdKaguEzFmSIiErFqEyiygeq/XVcH2/baxjlXBmwFMvZ3rJnFmtk8YAPwhnNuVtCmGzDSzOaY2atm1n1vnTKz64I2c/Lz82vxMQ7ClzPhr/3h5V/Azm8b9tx1MKhzG446RMWZIiISuUIrynTOlTvnBgA5QJ6Z9Ql2JQJFzrlc4DHgyX0c/6hzLtc5l5uZmdmwnTukD+T+AOY8CX8bCLP+AeWlDfseB8EXZ3ZivoozRUQkQtUmUKzB1zRUygm27bWNmcUB6cCm2hzrnNsCzMDXWIAfxXgheD4Z6FeLPjas5HZwzr3wo/egY3949ZfwyHGw/O0m70qlEQNzSIyLYbxmzhQRkQhUm0AxG+huZl3NLAFfZDm1RpupwJXB84uAt51zLtg+KrgLpCvQHfjIzDLNrA2AmbUCTgeWBse/CJwcPD8R+LxuH60BHNILvjcFRo2FsiJ4ZgSMGw2bljd5V9KT4zmnX0de/ETFmSIiEnkOGCiCmoibgOnAEmCCc26Rmd1hZsOCZk8AGWa2DPgZcGtw7CJgArAYeA240TlXDnQEZpjZAnxgecM5Ny04193AhWb2KfBH4JqG+ah1ZAY9zoEbP4LTfu/rKx4cAm/8Foq2NWlXLguKM6ctUHGmiIhEFvMDCdEtNzfXzZkzp2nebPt6eOsOmPcspHSA034H/S+FmMYvR3HOceb9M2mVEMeUG49t9PcTERGpyczmBnWOu9FMmQer9aFw/kNw7dvQ9jCYciM8djJ8PevAx9ZT5cyZ81dtYdFaFWeKiEjkUKCoq+zBcPUbcMFjUPANPHkGTLoGttasV21YI4KZM8d/FDnzZIiIiChQ1IcZ9LsEbpoDJ/wPLJ4Kf8+Fd/8MpYWN8pZtkhM4p29HXvxkDTtLVJwpIiKRQYGiISSmwim/gZtmQ/fTYcZd8PejYdFkaIQaldFDOrO9uIxp89c1+LlFRETqQoGiIbU9DC75F1z1MiS1geevgqfOhnXzG/Rtcg9rS/cOqYzVzJkiIhIhFCgaQ5fj4Ifvwrn3w8bP4B8nwtSboaBhpgivLM6ct2oLi9c27a2rIiIie6NA0VhiYiH3+/Djj+GYG/xtpg8Mgvf/DmUl9T79BYOySdDMmSIiEiEUKBpbqzYw9A9w/QfQKQ9e/zU8/B34/PV6nbayOHPyxyrOFBGR8ClQNJXMI+HySXDp8/712Ivh3xdBft1nFh+dFxRnLlBxpoiIhEuBoqkdeYYfrTjzD7Bqlh+teO1XULjloE91dJe2HNEhVcuai4hI6BQowhCXAN+50ddXDLwcPnzI11fMeQoqymt9msrizE++3sKSdSrOFBGR8ChQhCk1E877q78jJLMHTLvF3xGy8r1an+KCgUFxpkYpREQkRAoUkaBjfz93xcX/hKIt8M9zYMKVsPmrAx7aNiWBs/scygufrKGwpPajGyIiIg1JgSJSmEHvEX62zZN/DZ9Phwfz4O07oWTHfg8dndeZ7UVa1lxERMKjQBFp4lvBib+EH8+BnufBzHvggVxYMGGf03jndW1Ht8wUFWeKiEhoFCgiVXoOXPg4/GA6pHaAF66FJ86ANXP3aFpZnPnx11tYul7FmSIi0vQUKCJd52Pg2hkw/EHYvBIeOwVevAG2r9+t2YWDckiI1bLmIiISDgWKaBAT428v/fFcOPYW+PR5eGAw/GcMlBYBvjjzrL6HMunj1SrOFBGRJqdAEU2S0uD02+GGD6HrifDW7fDQEFgyDZyrKs58+VPNnCkiIk1LgSIaZXSD0WPhiskQlwTPXQb/Gs6QlPUcruJMEREJgQJFNOt2Cvzov3DWPbBuPvbIcTyQPpblX33NZ+u3h907ERFpQRQool1sHAy5Dm7+BHKvpteaibyT+DOWTxsD5VqFVEREmoYCRXOR3A7OuRf70Xt8k3IUZ68eQ/mkaxQqRESkSShQNDeH9OLbC57nD6WjiV08mbIXfnhQC46JiIjUhQJFM3RMtwxijruFP5eNJG7RRDaOvQ4qKsLuloiINGMKFM2QmXHrWT048Qd/5PG4UbRfNpF5D19FcWlp2F0TEZFmSoGiGRtyeAajfvF33s78HgPyp/D6PVeweM3WsLslIiLNkAJFM5eaFM8pN/yNr3pcy3klrzL7ket48O0vKCvXJRDZh8LNsPxtWDc/7J6ISBSJC7sD0gTMOGzkPRRNi+HKuf/g0bdjuWTJDfxl5EC6tk8Ju3cSptIi+GahX3Su8rFpmd9nMTDiUeh3cbh9FJGooEDRUpiRdO6fcDEVXDf7MWLy4zn7r6P41dk9ufyYwzCzsHsoja2iAjZ+7kPD2o/91/ULoSKorUk9FLIHQ//RkDUQ3rsPJl8HrgL6jwy37yIS8RQoWhIz7Ox7oKKMa+Y+RUbbVH46pYLXF3/Dny/qR8f0VmH3UBqKc7Bt7e4jD2vnQUkwg2pCa8geCN+9yYeI7MGQlrX7OTp/B8aNhMk/BFcOAy5t+s8hIlFDgaKlMYNzxkBFKSM++Tdd+7Zh9NLjOfO+mdwxvA/DB2RptCIaFW6BtZ8E4SEYfSgIlriPiYdD+/hRhsrwkNHdr2K7PwnJMPo5GD8aXrzBz2cy6IrG/ywiEpUUKFqimBg4729QXsaABX/nv8enc82yY7nluXm8vng9d57fl3YpCWH3UvZlj7qHj2HTF7v2ZxwBh5+4Kzwc0gfik+r2XgnJMHo8jL8Upt7kL38MvrJhPoeINCsKFC1VTCyc/xBUlNHu/buYeMZdPNJrKPe98TkffbmZP13Yl1N7HhJ2L6WiwoeF6pcudqt7OASyc6H/KB8esgZCqzYN24f4VjBqHDx3Obx0s7/8kfuDhn0PEYl65pwLuw/1lpub6+bMmRN2N6JTeRlM+gEsngJn3cPiTqP42YR5LF2/nZG5nfjNuT1pnRQfdi9bjr3VPRRv8/sSWkPWgF0jD5V1D011iaqsGJ67Ar6YDmffC3nXNs37ikhEMbO5zrncmts1QtHSxcbBhU/46+Ov/g+9zo1jyk1Xcv+bX/CPd5fz3+Ub+cvF/RlyeEbYPW1+qtc9VH7dvs7vq6x76HvxrvDQvrsfWQpLXCKMfAYmXAmv/MIXfg65Lrz+iEhEqdUIhZkNBf4KxAKPO+furrE/EfgXMBjYBIx0zq0M9t0GXA2UAzc756abWRIwE0jEh5qJzrnfBe3/CZwIVE7peJVzbt7++qcRigZQVuKHtL+YDsMegEHfY+5X3/KzCfP5+tudXHNcV35+xlEkxYf4Cy2alRX7SxW7zfdQo+6h+shDfeoeGltZCUz8PiydBkPvhmOuD7tHItKE9jVCccBAYWaxwOfA6cBqYDYw2jm3uFqbG4B+zrkfmdkoYIRzbqSZ9QLGAXlAFvAmcCRQAaQ45wrMLB54D/iJc+7DIFBMc85NrO2HU6BoIKVFvvhu+dtw/sMwYDQ7isv446tL+PeHX9O9Qyr3jRxAn+z0sHsa+bavh+UzqtU9fLqr7iGlA+TkQvaganUPbcPt78EqL/WhYslLcMZd/vZTEWkR6nPJIw9Y5pxbEZxoPDAcWFytzXDg98HzicDfzd97OBwY75wrBr40s2VAnnPuA6AgaB8fPKK/mCPaxSfBqGdh3CiYcgPExJHS72LuPL8vp/c6lF9OnM/5D/6Xm0/tzg0ndSMuVjO378E5+OQZeO02KCmAhFQfGL5zQ7W6h+ymq3toLLHxcNFTMOkaeP3XvlDz2J+E3SsRCVFtAkU2sKra69XAkH21cc6VmdlWICPY/mGNY7OhauRjLnAE8KBzbla1dneZ2W+Bt4Bbg0AiTaGyon/sJX6WxNg46D2CE4/M5PVbTuS3Uxcy5o3PeWvpBsZc0p9umalh9zhyFGyAqTfD569Cl+PhzD/AIb3DrXtoTLHxvv7GYuCN3/o6nON/FnavROrPOZj9OBzaDzrX/HUn+xLan5jOuXLn3AAgB8gzsz7BrtuAHsDRQDvgf/d2vJldZ2ZzzGxOfn5+k/S5xaice6DTEJh4tR/WBtKT4/nrqIE8eOkgvtq0g7P/+h+e+u+XVFRocInFU+GhY/zlojP/CN+bCh37Nd8wUSk2Di54DPpcBG/dDjPvCbtHIvU38x5fePzkmfDG73wNlBxQbQLFGqBTtdc5wba9tjGzOCAdX5x5wGOdc1uAGcDQ4PU65xUDT+EvuezBOfeocy7XOZebmZlZi48hByUxFS573l/nf/778NlrVbvO6deR1285gWOPaM/tLy3m8idmsWZLYYidDVHRVpj8I5hwBaTnwA9n+ssbB5qFsjmJjYMR/4C+l8Dbd8I7fwq7RyJ1t3ASzLjLh+RB34P/3g+PnQrfLD7wsS1cbX7qzQa6m1lXM0sARgFTa7SZClROn3cR8Lbz1Z5TgVFmlmhmXYHuwEdmlmlmbQDMrBW+4HNp8Lpj8NWA84GF9fmAUg+JreHySf72xQlXwBdvVu3qkJbEE1fmcvcFfZm/agtD75vJxLmraQ7zmtTainfhoe/Cgglwwi/hmregQ4+wexWO2DgY8YhfWOydP8CMP/phY5Fosmo2TL4eOh3jJ/4b9jc/WluwHh49Ed5/wE82J3t1wEDhnCsDbgKmA0uACc65RWZ2h5kNC5o9AWQERZc/A24Njl0ETMAXcL4G3OicKwc6AjPMbAE+sLzhnJsWnOtZM/sU+BRoD9zZMB9V6iQpHa6YDJk9gjtAZlTtMjNG5XXmtVtOoGfHNH7x/Hx++MxcNhY08+HB0kJ49Vb41zBfyHr163DKr31NQUsWEwvDH4QBl8G7d/u/8hQqJFps+dqvW5PW0RenxyX67UedBTd8CN3PgNd/A0+f59vKHjRTptTOzm/hn+fCtyv8pZCux++2u7zC8eR7X3LP9M9onRTHHy7oy5m9Dw2ps41o7Sfwwg9h42eQdx2cdruvOZFdKir8FN2fPAPH/xxO+X/Rf1eLNG9F23y9xNY1cM0bkHnUnm2cg3lj4dX/9f+ez/qzn/K+Bf7b3tdtoy3oQq/US3I7+N4UaHuYvwPkq/d32x0bY1x7wuFMu/k4Dk1P4ofPzOXnE+azrag0pA43sPIyXxvw+GlQvB0ufwHOvh7ExYQAABmgSURBVEdhYm8qF58bfBX85y/w5u81UiGRq7wMJl0N+Z/BJf/ce5gAHxwGXgbX/9dPPPdiUDu1Y1OTdjeSKVBI7aVm+rsX0rLh2Yth1Ud7NDnykNZMvuFYbj7lCF6ct4ah983k/WUbQ+hsA9r4BTx5hq8N6D0Cbngfjjg17F5FtpgYOOc+yL3aF7W98f8UKiQyvf5r+OJ1/wdCt1MO3L7tYXDVNDj9Dvh8ur+76/Ppjd/PKKBAIQen9SFw5UuQ2gH+fSGsnrtHk4S4GH52xlFMuv67JMXHcunjs/j91EUUlpSH0OF6qKiAWY/CI8f7Sz0XPQUXPh59s1qGJSYGzvkLHH2tL2ab/muFCoksHz0Gsx6BIdfD0VfX/riYWD+R27UzICXTj9q+9BMoLjjwsc2YaiikbrauhqfOhqItftQia8BemxWWlPOn15byz/dXcnhmCmMuGcCATg28vHZj2LrGzxa64h044nS/vklax7B7FZ2cg9du3fWDe+gfW+R1Z4kwy96EZy+BI06D0ePqPmdMWbG/Xfr9B6BtF7jgUei019kOmg3VUEjDSs/xw36J6fDM+X6tir1olRDL74f15tlrhlBUUs6FD7/PmNc/o7Q8Qm+9cs7fBvrQd/wlnXPv80WoChN1ZxYsInYjzHoYXv2lRiokXBuW+vl1MnvARU/UbwK6uEQ44//gqpf9bLFPnglv/Z9fRK+F0QiF1M+3X8I/z4GyIrhyGhzSa59NtxWVcvvUxUz6eDV9stMYc8kAjjykdRN29gB2fgvTfgqLX4ScPD+vQka3sHvVfDjnaynef8DXVpx9b8uaAEwiw46N8Ngp/vbva9+GNp0OfExtFW3z6/jM+zd07A8jHm2Wc9NohEIaR7uuvqYiJt7Py5D/2T6bpiXF85dL+vPI5YNZt6WIcx94j8dmrqA8Eqbu/vx1X1y19GU49bfwg9cUJhqaGZz+f3DsLTDnCXj5p5okSJpW5YrKBd/4CasaMkwAJKXB+Q/CyGf9ZeF/nAAfPNRi/p0rUEj9ZXTzlz8wP+nLxmX7bT60z6FM/+kJnHhkJne9soTRj33Iqm93Nk1fayou8MVUYy+G5Az/F8vxP2/+a3CExQxO+73/Hs/9J0z7SYv5YSshcw6m/hhWzYLzH4acwY33Xj3P9ZNhdTsZpt8Gzwz3AaOZU6CQhtG+ux+pqCj3oeLbFftvnprIo1cM5t6L+7Nk7TaG3j+T52Z/3bRTd389Cx45DuY+Dd/9sa/Y7tiv6d6/pTLzk12d8Ev4+F/+h3xFlN0BJNFn5j3w6QQ4+TfQ54LGf7/UDn4U5Ly/+bvhKqfpbwZlBvuiQCENp0MPP/lVWSE8PQw2f7Xf5mbGRYNzeO2nJ9C/Uxv+d9KnXP30HDZsL2rcfpaV+MmWnhoKrtwXU51xp59GW5qGmZ+u/KTgevOUGxUqpPFULvjVbySc8Iume18zGHwlXP+e//n4wrUw8fu+XqsZUlGmNLx18/0oRVI6fP9Vf0fIAVRUOJ7+YCV3v7qU5IRY7hrRl7P7NsKdFd8s8lNnf/MpDLwCzvyDv+4p4Xn3z7t+2J//sC43ScNaPccXjnccAFdO3bVGR1OrKPeTvM34AyS397UWR5wWTl/qaV9FmQoU0jjWzIV/ne/rEr7/CqRl1eqwZRsK+PmEecxfvZXDM1PonZVOr45p9MpKo1fHNDJb1/GHQUU5fPB3f794UrqfV+Kos+p2Lml4M++Ft//PLxk94h9+9VKR+trytb+jIyHFrwac0j7sHvk/uF64DvKXwtHX+Bk3E1LC7tVBUaCQprdqNjwzws+uedXL0Lp2i4WVlVfwzIdf8f7yTSxeu401Wwqr9mW2TtwtYPTKSqNLRgqxMfuZKGnzSr8k8dfvQ49z4by/RsYPFtnde/f5S1G9L4ALHlOokPqpzYJfYSktgrfugA8fhHbd/GRYOXv8fo5YChQSjq8+8FN0p+f4UJGaedCn2LqzlMXrtvnHWv/1i2+2UxbcbtoqPpYeHVvvFjR6HJpGq/gYv+Lla7eBxbTo1QGjxn//5ueq6DUcLnxCS8JL3ZSX+aXIl70Fl0+s3RodYfhypv9jZ/s6OOF/fH1HFPybV6CQ8Kx8D/59EbQ73N8JkpJR71MWl5WzbENBVcCo/Lq9qAyADraV+5Of5Lvls1nTJpdVJ/6Fbkf0rPslE2k6HzwI038FPc+DC5+EuISweyTR5tVb/ays54w5uDU6wlC4xS+JvmA8ZA30o3Ptu4fdq/1SoJBwrXgHxo70/6N8b6pfDr2BOedYvbmQjbMnctTs/0d8+Q4eir2C+wtOwQU3NNXpkok0vQ8f9ut/HHUOXPxPhQqpvdmPw8s/9+vGnHV32L2pvUUvwrRb/OWQ0++AvGsjdjRVgULCt+xNGDcaOvTyt5e2auBFwoq2wiu/9Em/4wBf3NehR90vmSToboNQzXoUXv0fOPIsuOTp8KrzJXo01IJfYdm+3t9CvexNf5lm+IO1LmhvSgoUEhk+nw7jL/Pz3F8xueFu2VzxLrx4g78WefzP4cRf7vda5IEumcQYdG2fQq+GustE6qbyr83uZ8Il/9JcIbJvG5bCE6dDeie4ejokRtA6QQfDOZjzJLz+G4hNgHPHQJ8Lw+7VbhQoJHIsfRkmfA+yB8PlL0Biat3PVVoIb97ur5dmHOFHJepYLV15yaR6wGiQu0ykfuY86RdtO+I0v0aCQoXU1JgLfoVl4zKYfJ2/Bb/PRXDOvdCqbdi9AhQoJNIsnuKXD+58jF8evC73Ya/9xE9StfEzyLsOTrsdEpIbvKtbdpawZN12XTIJ09x/+jVXup0Co8ZCfKuweySRorTIL0y4br6/kyyKbr88oPIyeG8MvPsnSOkA5z/k1wcJmQKFRJ5PJ/qpaLscB6Ofq30YKC+F/4yBmX8O/id7sMlvC6vNJZPM1omkt4onLSmetFbxwfO4as/jSWsVV2N/PK2T4ojRiMeePn7Gr/tx+IkwalyjhEeJMs7B5B/CgufgoqeaZo2OMKz52H/OjZ/DkB/5BfZCDNUKFBKZ5j/n/0fpdrL/JXGg4eyNX/hZ5tZ+DH0vhrPviZhhwOqXTBat3cb6rYVsKyxja2Ep24qCR2EZ24pK97s+kBmkJlYPGnuGjrRWcbuFleqvkxNisQitDq+3eWN9rUzX4/3CS1E2w6A0sHfvgRl3+gW/TvyfsHvTuEoL/cRvsx6B9kf6ybCyBobSFQUKiVyfBItDdT8DRv5779X8FRW+QO+N3/rQcc6YqP1rpKLCUVBSxrbCUh82gpBR9brI79tWuCuEVIWSwlJ2lOx/Ea24GPMhIxgNSUsKgki1YFK5P73q+a5QkhgX4Zdq5o+HF6+Hw46FS59TqGipFr7gF9rqN9LXTjXXEF3T8rfhxRthxwY48VY47qdNPqusAoVEtjlP+XuwjzobLn5693kHtq6BKTf4uSyOON2vw5HWCAuHRYnS8gq2V4aOon2Fkj2DyNZCf0xJecV+z58YF0Naq3gyUxM5PDOFwzNT6ZaZQrfMVA7PTCE5IQKmxF7wvC9Y6xTU4NSnsFeiT6Qs+BWWws3w8i9g4UTIOdoHqoxuTfb2ChQS+T56DF75hZ8h8aKnICYOPn3e/49TUQpn3gWDv99y/hJpJEWl5buHjOD5tmqjI1sLS1m/rYgV+TtYtXnnbpdostKTqkKG/5pKtw4pHJqW1LSXWj6d6C9/5Rztp1eO1tsE5eBULvgVn+zv6GjJ6/J8OhFe/pmvKzvjTsj9QZP8fFSgkOjwwUMw/Tbodb5/vfhF6DTEL2vdhAlcdikqLeerTTtZnl/AivwClufvCJ7voKC4rKpdckKsH9Fon1o1mtEtM5Wu7VMa746XRZNh4tXBLciTtBR9c1e0DZ4cCltXwdVvQIceYfcofFvX+EvGK2b4Edzhf6/1Qox1pUAh0eO/f/W1EjHxcPJtcOwt0TfjXQvgnGPD9mKWByFjRbWva7YU7jaqkd2mFd06pHJ4+xS6dUilW/C1Q+vE+o9qLJ4CE3/gC9Qun+SXp5fmp6Icxo2K/AW/wlBRAbMfC2rMkuG8+/0Ce41EgUKiy9KXoW0XOKR32D2ROigsKefLjTtYsbGA5RuCr8Goxs5qRaWpiXFVIxmVYePwzBS6ZKSQFH8QIXLJS/D8VX4G1stfaPhp3SV80bTgV1jyP/e1RWs/gX6j4Ow/N0rAVqAQkdA551i/rWhXyNhQwIqNO1i+oYC1W4uq2plBTttWvj6j2uWTwzNTyEzdx6jG0lf8DKyH9vHTukfI7cTSAKJ1wa8wlJfCzHtg5r1+jp8rpzb4WyhQiEhE21lSxor8HVUBo3JEY8XGAopKd92Z0jopbrc7Tyq/ds5IJnH5GzDhCujQE654sVFWtZUmtuwtePbi6F3wKyyr54DFQPagBj+1AoWIRKWKCse6bUV+NKNGUej6bbtGNWIMOrdLZnjKIn6cfzs7krP5qtulbOs6lIQ2OaQmxtE6KY7UxDhSk+KIj40J8VNJrTSXBb+aGQUKEWl2CorL+LIqYOwKGx03fcBt9jRHxqwBYHbFkbxWnser5Xmsxd9mmBQfQ2pi/K6QURk4kuJoHYSO1MR4UpPiSKvWxu+PD/bHkRCnYNIomuOCX82EAoWItBjlFY5NBcUUrVtC/Gcv0frLV0jdvASA/LQ+fJZxCgtan8gqDqGguIyColIKisvYXuQfBcX+UV5x4J+PiXExu418+GASXy2UBCGkcttu7fx2BZMayorh6WGwbl7zW/CrGVCgEJGWbdNyf4vp4in+FxXAof387XW9zof2R+zW3DlHUWkF24tLKagWNKoCR/UQUlxGQdV2/3p7tf21CSYJcTG7hZD0VvEclpESTCDm5/fIaduKuOZ+qaalLPgVxRQoREQqbV7pbzVdPAVWz/bbOvQOwsXwBp0wyTlHcVlFtUBSWhU6CqptqxlKvt1ZwsqNO9i8s7TqXPGxxmEZKRze3s9Seni1wtQ2yQn76UUUaUkLfkUpBQoRkb3ZumZXuPj6A8D51Rwrw8UhfUKd7n3zjpJgHo8d/q6XfH8HzNff7qS0fNfP73YpCUHQCMJGEDoOy0iOngLUlrrgV5RRoBAROZDt6324WDIVVr4HrgLadt0VLrIGRswvubLyClZtLmRFtdtrK0PHxoLiqnaxMUbndsm7Jg6rNrqRkZIQOUvdt/QFv6JIvQKFmQ0F/grEAo875+6usT8R+BcwGNgEjHTOrQz23QZcDZQDNzvnpptZEjATSATigInOud/VOOffgB845w64jKAChYg0uB0bYek0P3Kx4l1w5ZDeGXoN8+EiOxdiIvMv/62FpX6m0vxq83nk7+DLTTsoKds1p0daMKdH9dlKK0c1Dmqm0vra8jU8dirEt9KCX1GgzoHCzGKBz4HTgdXAbGC0c25xtTY3AP2ccz8ys1HACOfcSDPrBYwD8oAs4E3gSKACSHHOFZhZPPAe8BPn3IfB+XKBnwTnUaAQkXDt/BY+e9WHi+Vv+9VvW2ftChedhkTFhEvlFY61Wwp3mzSsMmzUnNMjO5ip9PD2qcFlFB86GmT9leq04FfU2VegiKvFsXnAMufciuBE44HhwOJqbYYDvw+eTwT+bv5f3HBgvHOuGPjSzJYBec65D4CCoH188HDB+WOBe4BLgREH8yFFRBpFcjsYeJl/FG6Bz6f7cDHnKZj1CKQeAj3O9eHisGMhtjY/WptebIzRqV0yndolc9JRu+/bUVzGlxt37LbY24r8Hcxa8S2Fpbuvv9K1/a47Tw6vdhfKQa8qW1EOk66G/KVw2fN1ChPlFY7S8gpKyisoLaugtHzX65KyCkrL/aOkzFU99/td0N4/iqsdW3k8DjAwrOpKl+GvelVuM/yGvW03oyp8Ve7b9Xz39n579fMEr/fyHr4ftsd5qH4cPhQec3jGQX9P66o2/+qzgVXVXq8GhuyrjXOuzMy2AhnB9g9rHJsNVcFhLnAE8KBzblbQ5iZgqnNu3f5SsJldB1wH0Llz51p8DBGRBtCqDfQf6R/F2+GL1324mD8O5jwByRm7wkXXEyA2Puwe10pKYhx9stPpk737YlIVFX79leojGsvzC5izcjNT5q3drW12m1ZBuEjhsIwU4mIt+KW++y/q0uCX+5mr7ue4Ta8zvsMtzPggndL3Zge//Ct/0VcLC8FxlcGhclst7sg9aGYQHxuDEfylG7yHw+Gcf+mcC742/Ps3lKG9D424QNEonHPlwAAzawNMNrM+wLfAxcBJtTj+UeBR8Jc8GrGrIiJ7l9ga+lzoHyU7YdmbPlwsnAQfPw1JbaDHOT5cHH5SVBYaxsQYWW1akdWmFcd13722ofqqspV3oKzYuINJH6+hoLhsr+dLiIshITaGS2Pe4Dj3PBNiz+XRnScTX7yT+DgjPjaG+NgYkhPiiI81EuL864Rge2WbqtexMUGbXW39w6od418n7rY/OEe190wIzhUbc/CXdJzbe9ioDCG72u0nmOxnn8NVCzb7OI/b/T0OesSonmoTKNYA1ec8zQm27a3NajOLA9LxxZkHPNY5t8XMZgBDgSX4EYtlwehEspktc87tPuOMiEikSUgOaiqGQWmRr7VYPAWWTIN5z0JiGhx1lg8X3U7xBYhRrlVCLL2y0uiVlbbbducc3+4owUG1X/5GbIz5SwDL3oJnn4TuZ3LJ6H9xSRTUnxxI5eWJ4FWYXQlNbQLFbKC7mXXFh4FR+PqG6qYCVwIfABcBbzvnnJlNBcaa2Rh8UWZ34CMzywRKgzDRCl/w+Sfn3MvAoZUnNbMChQkRiTrxSdDjbP8oK4Ev34XFL8LSl/0MkPEpcOSZPlx0Px0SUsLucYMyMzJS9zEas2EpPH8VZPaAi56IimJWqZ0DBoqgJuImYDr+ttEnnXOLzOwOYI5zbirwBPBMUHT5LT50ELSbgC/gLANudM6Vm1lH4OmgjiIGmOCcm9YYH1BEJFRxCT40dD8dzr0fVv4HFk/1810segHiWvl9vYb7kNGcV9TcsRHGXgJxSXDp+Ob9WVsgTWwlIhKGinL46v3gsshLULAeYhPhiFOh5zB/eaRVm7B72XC04FezUZ/bRkVEpKHFxELX4/3jrD/D6o92LV722SuAQWoHSMuCtGz/SM/e9TwtC1p39CMgkc45mPpjWPWhX/BLYaJZUqAQEQlbTAx0PsY/zrgL1n7sizq3rvJrjWxaDl/OhOJtNQ6sDB1BwEjP2TOAtO4Y/q2rM+/1tSMn/0arhzZjChQiIpEkJsb/Bb+3v+KLtsG2tbBtTfBYC1tX+6+blu0ndBwSBI4aIxyVAaQxQ8fCF/zqof1Gwgm/aJz3kIigQCEiEi2S0vxjfzNKVoWOIGhsXbMrgGz8Apa/AyXbaxwUhI70IGik5ewZQFofevChY/UcePF66HQMDHsgYhZWk8ahQCEi0pzUOnQEIWPrmt0DSP7nsHwGlBTUOMh8qNijpqNaAGndcde041u+hnGjfVAZ9WxUTuolB0eBQkSkpakKHT333aZo654jHJUBJP8zX+NRM3RYTHB5JRsKNkBZEVz5klYPbSEUKEREZE9J6f6xr9DhnK/XqDnCURlAcDD8Aa0e2oIoUIiIyMEz2xU6DukVdm8kAsSE3QERERGJfgoUIiIiUm8KFCIiIlJvChQiIiJSbwoUIiIiUm8KFCIiIlJvChQiIiJSbwoUIiIiUm/mnAu7D/VmZvnAVw182vbAxgY+pxwc/TcIl77/4dL3P3z6b7B3hznnMmtubBaBojGY2Rzn3F7WD5amov8G4dL3P1z6/odP/w0Oji55iIiISL0pUIiIiEi9KVDs26Nhd0D03yBk+v6HS9//8Om/wUFQDYWIiIjUm0YoREREpN4UKERERKTeFCj2wsyGmtlnZrbMzG4Nuz8tiZl1MrMZZrbYzBaZ2U/C7lNLZGaxZvaJmU0Luy8tkZm1MbOJZrbUzJaY2XfC7lNLYmY/DX7+LDSzcWaWFHafooECRQ1mFgs8CJwF9AJGm1mvcHvVopQBP3fO9QKOAW7U9z8UPwGWhN2JFuyvwGvOuR5Af/TfosmYWTZwM5DrnOsDxAKjwu1VdFCg2FMesMw5t8I5VwKMB4aH3KcWwzm3zjn3cfB8O/4HaXa4vWpZzCwHOAd4POy+tERmlg6cADwB4Jwrcc5tCbdXLU4c0MrM4oBkYG3I/YkKChR7ygZWVXu9Gv1CC4WZdQEGArPC7UmLcz/wS6Ai7I60UF2BfOCp4LLT42aWEnanWgrn3BrgXuBrYB2w1Tn3eri9ig4KFBKRzCwVmATc4pzbFnZ/WgozOxfY4JybG3ZfWrA4YBDwsHNuILADUC1XEzGztvhR6a5AFpBiZpeH26vooECxpzVAp2qvc4Jt0kTMLB4fJp51zr0Qdn9amGOBYWa2En+57xQz+3e4XWpxVgOrnXOVI3MT8QFDmsZpwJfOuXznXCnwAvDdkPsUFRQo9jQb6G5mXc0sAV+MMzXkPrUYZmb4a8dLnHNjwu5PS+Ocu805l+Oc64L/t/+2c05/nTUh59x6YJWZHRVsOhVYHGKXWpqvgWPMLDn4eXQqKoqtlbiwOxBpnHNlZnYTMB1f3fukc25RyN1qSY4FrgA+NbN5wbZfOedeCbFPIk3tx8CzwR81K4Dvh9yfFsM5N8vMJgIf4+86+wRNwV0rmnpbRERE6k2XPERERKTeFChERESk3hQoREREpN4UKERERKTeFChERESk3hQoREREpN4UKERERKTe/j9DarYckelo+AAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "RdHiUN0op4KM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "net.eval()"
      ],
      "metadata": {
        "id": "Kfpursi2nH2r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = []\n",
        "j = 0 \n",
        "net.eval()\n",
        "with torch.no_grad():\n",
        "    for data in train_dataloader:\n",
        "        batch, labels = data\n",
        "        batch, labels = batch.to(device), labels.to(device)\n",
        "        out = net(batch)\n",
        "        # plt.hist(out[1,3:,0,0].cpu().detach().numpy()*255)\n",
        "        plt.imshow(((out[1,3:,:,:].cpu().detach().numpy()*255).T).astype(int))\n",
        "        plt.show()\n",
        "        j = j +1 \n",
        "        if(j>20):\n",
        "          break\n",
        "        print(j)"
      ],
      "metadata": {
        "id": "3t9l-YHjnfQ4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "out[0,3:,:,:].cpu().detach().numpy()"
      ],
      "metadata": {
        "id": "Nfir5VltpqUU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data[0][0].size()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cp34Z-KInzHv",
        "outputId": "2dbcf23f-c481-4a45-b6ba-ddda1b7be8d1"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([3, 32, 32])"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "net(data[0][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        },
        "id": "F9R3z0PfnK5i",
        "outputId": "69df8687-f35b-4f3a-d785-0c0f3c614082"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-65-b3727e9be165>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1188\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1189\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1190\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1191\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-39-45eb573682fa>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m         \u001b[0mencoded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         \u001b[0mdecoded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoded\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdecoded\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1188\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1189\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1190\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1191\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1188\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1189\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1190\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1191\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    462\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 463\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_conv_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    464\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    465\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36m_conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m    457\u001b[0m                             \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstride\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    458\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[0;32m--> 459\u001b[0;31m         return F.conv2d(input, weight, bias, self.stride,\n\u001b[0m\u001b[1;32m    460\u001b[0m                         self.padding, self.dilation, self.groups)\n\u001b[1;32m    461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0! (when checking argument for argument weight in method wrapper___slow_conv2d_forward)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loss_hist"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LuDf79Z5j1jH",
        "outputId": "001e9136-973e-4060-fd57-279f5e682194"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.01829134351015091,\n",
              " 0.01829119467602836,\n",
              " 0.018291674719254174,\n",
              " 0.018288753596279357,\n",
              " 0.018291366510921055,\n",
              " 0.018289796302053664,\n",
              " 0.018290697213013966,\n",
              " 0.018292148091395698]"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZcJblZnwDkCO"
      },
      "outputs": [],
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "net1 =  Autoencoder(base_channel_size=32, latent_dim=64, encoder_class=encoder)\n",
        "net2 = Autoencoder(base_channel_size=32, latent_dim=64 , encoder_class=encoder)\n",
        "criterion = nn.MSELoss()\n",
        "optimizer1 = optim.Adam(net1.parameters(), lr=0.01, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\n",
        "optimizer2 = optim.Adam(net2.parameters(), lr=0.01, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\n",
        "\n",
        "net1.to('cuda:0')\n",
        "net2.to('cuda:0')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xO2T_MIjEP2-"
      },
      "outputs": [],
      "source": [
        "from torchsummary import summary\n",
        "\n",
        "summary(net1,(3,32,32))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9M-9O04HHrZJ"
      },
      "source": [
        "# Train "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4wlTsw_8Vnn7"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 467
        },
        "id": "ZoAfTvhgFe-2",
        "outputId": "8b942f01-e2e8-46bb-cfca-ae9b04ee889c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([32, 3, 32, 32])\n"
          ]
        },
        {
          "ename": "TypeError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-55-601a3636a6a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m         \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m         \u001b[0mloss1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mimshow\u001b[0;34m(X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, shape, filternorm, filterrad, imlim, resample, url, data, **kwargs)\u001b[0m\n\u001b[1;32m   2643\u001b[0m         \u001b[0mfilterrad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimlim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeprecation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_deprecated_parameter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2644\u001b[0m         resample=None, url=None, *, data=None, **kwargs):\n\u001b[0;32m-> 2645\u001b[0;31m     __ret = gca().imshow(\n\u001b[0m\u001b[1;32m   2646\u001b[0m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcmap\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnorm\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maspect\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maspect\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2647\u001b[0m         \u001b[0minterpolation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minterpolation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/matplotlib/__init__.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1563\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1564\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1565\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msanitize_sequence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1566\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1567\u001b[0m         \u001b[0mbound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_sig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/matplotlib/cbook/deprecation.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    356\u001b[0m                 \u001b[0;34mf\"%(removal)s.  If any parameter follows {name!r}, they \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    357\u001b[0m                 f\"should be pass as keyword, not positionally.\")\n\u001b[0;32m--> 358\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/matplotlib/cbook/deprecation.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    356\u001b[0m                 \u001b[0;34mf\"%(removal)s.  If any parameter follows {name!r}, they \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    357\u001b[0m                 f\"should be pass as keyword, not positionally.\")\n\u001b[0;32m--> 358\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    359\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mimshow\u001b[0;34m(self, X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, shape, filternorm, filterrad, imlim, resample, url, **kwargs)\u001b[0m\n\u001b[1;32m   5624\u001b[0m                               resample=resample, **kwargs)\n\u001b[1;32m   5625\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5626\u001b[0;31m         \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5627\u001b[0m         \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_alpha\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5628\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_clip_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/matplotlib/image.py\u001b[0m in \u001b[0;36mset_data\u001b[0;34m(self, A)\u001b[0m\n\u001b[1;32m    687\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mImage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mImage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m                 \u001b[0mA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpil_to_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Needed e.g. to apply png palette.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 689\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_A\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msafe_masked_invalid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m         if (self._A.dtype != np.uint8 and\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/matplotlib/cbook/__init__.py\u001b[0m in \u001b[0;36msafe_masked_invalid\u001b[0;34m(x, copy)\u001b[0m\n\u001b[1;32m    704\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    705\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msafe_masked_invalid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 706\u001b[0;31m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    707\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnative\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    708\u001b[0m         \u001b[0;31m# Note that the argument to `byteswap` is 'inplace',\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36m__array__\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m    953\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__array__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    954\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 955\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    956\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    957\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first."
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAD8CAYAAACVSwr3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAMbElEQVR4nO3bcYikd33H8ffHXFNpGrWYFeTuNJFeGq+2kHRJU4SaYlouKdz9YZE7CG1KyKE1UlAKKZZU4l9WakG41l6pRAWNp3+UBU8CtZGAeDEbEmPuQmQ9bXNRmjOm/iMaQ7/9YybtZL+7mSd3szO39f2ChXme+e3Md4fhfc8881yqCkma9IpFDyDpwmMYJDWGQVJjGCQ1hkFSYxgkNVPDkOQTSZ5O8tgm9yfJx5KsJXk0yTWzH1PSPA05Yrgb2PcS998I7Bn/HAb+4fzHkrRIU8NQVfcDP3yJJQeAT9XICeA1SV4/qwElzd+OGTzGTuDJie0z433fX78wyWFGRxVccsklv3XVVVfN4Oklbeahhx76QVUtvdzfm0UYBquqo8BRgOXl5VpdXZ3n00s/d5L8+7n83iy+lXgK2D2xvWu8T9I2NYswrAB/PP524jrgR1XVPkZI2j6mfpRI8lngeuCyJGeAvwZ+AaCqPg4cB24C1oAfA3+6VcNKmo+pYaiqQ1PuL+A9M5tI0sJ55aOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6TGMEhqDIOkxjBIagyDpMYwSGoMg6RmUBiS7EvyRJK1JHdscP8bktyX5OEkjya5afajSpqXqWFIchFwBLgR2AscSrJ33bK/Ao5V1dXAQeDvZz2opPkZcsRwLbBWVaer6jngHuDAujUFvGp8+9XA92Y3oqR5GxKGncCTE9tnxvsmfRC4OckZ4Djw3o0eKMnhJKtJVs+ePXsO40qah1mdfDwE3F1Vu4CbgE8naY9dVUerarmqlpeWlmb01JJmbUgYngJ2T2zvGu+bdCtwDKCqvga8ErhsFgNKmr8hYXgQ2JPkiiQXMzq5uLJuzX8AbwdI8mZGYfCzgrRNTQ1DVT0P3A7cCzzO6NuHk0nuSrJ/vOz9wG1JvgF8Frilqmqrhpa0tXYMWVRVxxmdVJzcd+fE7VPAW2c7mqRF8cpHSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUGAZJjWGQ1BgGSY1hkNQYBkmNYZDUDApDkn1JnkiyluSOTda8M8mpJCeTfGa2Y0qapx3TFiS5CDgC/D5wBngwyUpVnZpYswf4S+CtVfVsktdt1cCStt6QI4ZrgbWqOl1VzwH3AAfWrbkNOFJVzwJU1dOzHVPSPA0Jw07gyYntM+N9k64Erkzy1SQnkuzb6IGSHE6ymmT17Nmz5zaxpC03q5OPO4A9wPXAIeCfkrxm/aKqOlpVy1W1vLS0NKOnljRrQ8LwFLB7YnvXeN+kM8BKVf2sqr4DfItRKCRtQ0PC8CCwJ8kVSS4GDgIr69b8C6OjBZJcxuijxekZzilpjqaGoaqeB24H7gUeB45V1ckkdyXZP152L/BMklPAfcBfVNUzWzW0pK2VqlrIEy8vL9fq6upCnlv6eZHkoapafrm/55WPkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySmkFhSLIvyRNJ1pLc8RLr3pGkkizPbkRJ8zY1DEkuAo4ANwJ7gUNJ9m6w7lLgz4EHZj2kpPkacsRwLbBWVaer6jngHuDABus+BHwY+MkM55O0AEPCsBN4cmL7zHjf/0pyDbC7qr74Ug+U5HCS1SSrZ8+efdnDSpqP8z75mOQVwEeB909bW1VHq2q5qpaXlpbO96klbZEhYXgK2D2xvWu87wWXAm8BvpLku8B1wIonIKXta0gYHgT2JLkiycXAQWDlhTur6kdVdVlVXV5VlwMngP1VtbolE0vaclPDUFXPA7cD9wKPA8eq6mSSu5Ls3+oBJc3fjiGLquo4cHzdvjs3WXv9+Y8laZG88lFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWGQVJjGCQ1hkFSYxgkNYZBUmMYJDWDwpBkX5InkqwluWOD+9+X5FSSR5N8OckbZz+qpHmZGoYkFwFHgBuBvcChJHvXLXsYWK6q3wS+APzNrAeVND9DjhiuBdaq6nRVPQfcAxyYXFBV91XVj8ebJ4Bdsx1T0jwNCcNO4MmJ7TPjfZu5FfjSRnckOZxkNcnq2bNnh08paa5mevIxyc3AMvCRje6vqqNVtVxVy0tLS7N8akkztGPAmqeA3RPbu8b7XiTJDcAHgLdV1U9nM56kRRhyxPAgsCfJFUkuBg4CK5MLklwN/COwv6qenv2YkuZpahiq6nngduBe4HHgWFWdTHJXkv3jZR8Bfhn4fJJHkqxs8nCStoEhHyWoquPA8XX77py4fcOM55K0QF75KKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqTEMkhrDIKkxDJIawyCpMQySGsMgqRkUhiT7kjyRZC3JHRvc/4tJPje+/4Ekl896UEnzMzUMSS4CjgA3AnuBQ0n2rlt2K/BsVf0q8HfAh2c9qKT5GXLEcC2wVlWnq+o54B7gwLo1B4BPjm9/AXh7ksxuTEnztGPAmp3AkxPbZ4Df3mxNVT2f5EfAa4EfTC5Kchg4PN78aZLHzmXoBbmMdX/PBWw7zQrba97tNCvAr53LLw0Jw8xU1VHgKECS1apanufzn4/tNO92mhW217zbaVYYzXsuvzfko8RTwO6J7V3jfRuuSbIDeDXwzLkMJGnxhoThQWBPkiuSXAwcBFbWrVkB/mR8+4+Af6uqmt2YkuZp6keJ8TmD24F7gYuAT1TVySR3AatVtQL8M/DpJGvADxnFY5qj5zH3ImynebfTrLC95t1Os8I5zhv/YZe0nlc+SmoMg6Rmy8OwnS6nHjDr+5KcSvJoki8neeMi5pyY5yXnnVj3jiSVZGFfsw2ZNck7x6/vySSfmfeM62aZ9l54Q5L7kjw8fj/ctIg5x7N8IsnTm10XlJGPjf+WR5NcM/VBq2rLfhidrPw28CbgYuAbwN51a/4M+Pj49kHgc1s503nO+nvAL41vv3tRsw6dd7zuUuB+4ASwfKHOCuwBHgZ+Zbz9ugv5tWV0Uu/d49t7ge8ucN7fBa4BHtvk/puALwEBrgMemPaYW33EsJ0up546a1XdV1U/Hm+eYHRNx6IMeW0BPsTo/678ZJ7DrTNk1tuAI1X1LEBVPT3nGScNmbeAV41vvxr43hzne/EgVfcz+jZwMweAT9XICeA1SV7/Uo+51WHY6HLqnZutqarngRcup563IbNOupVRhRdl6rzjQ8bdVfXFeQ62gSGv7ZXAlUm+muREkn1zm64bMu8HgZuTnAGOA++dz2jn5OW+t+d7SfT/F0luBpaBty16ls0keQXwUeCWBY8y1A5GHyeuZ3Qkdn+S36iq/1roVJs7BNxdVX+b5HcYXcfzlqr670UPNgtbfcSwnS6nHjIrSW4APgDsr6qfzmm2jUyb91LgLcBXknyX0WfLlQWdgBzy2p4BVqrqZ1X1HeBbjEKxCEPmvRU4BlBVXwNeyeg/WF2IBr23X2SLT4rsAE4DV/B/J3F+fd2a9/Dik4/HFnQCZ8isVzM6KbVnETO+3HnXrf8Kizv5OOS13Qd8cnz7MkaHvq+9gOf9EnDL+PabGZ1jyALfD5ez+cnHP+TFJx+/PvXx5jDwTYzq/23gA+N9dzH6FxdGpf08sAZ8HXjTAl/cabP+K/CfwCPjn5VFzTpk3nVrFxaGga9tGH30OQV8Ezh4Ib+2jL6J+Oo4Go8Af7DAWT8LfB/4GaMjr1uBdwHvmnhtj4z/lm8OeR94SbSkxisfJTWGQVJjGCQ1hkFSYxgkNYZBUmMYJDX/AwqkUdV2nfELAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "for epoch in range(140):\n",
        "    net1.train()\n",
        "    net2.train()\n",
        "\n",
        "    running_loss = 0.0\n",
        "    correct = 0\n",
        "    for data in train_dataloader:\n",
        "        batch, labels = data\n",
        "        batch = batch.float()\n",
        "        # labels[0] = torch.tensor(labels[0]).to('cuda:0')\n",
        "        # labels[1] = torch.tensor(labels[1]).to('cuda:0')\n",
        "\n",
        "        batch , labels = batch.to('cuda:0') , labels.to('cuda:0')\n",
        "        batch, labels = batch.cuda(), labels.cuda() # add this line\n",
        "\n",
        "        optimizer1.zero_grad()\n",
        "        # optimizer2.zero_grad()\n",
        "\n",
        "        # outputs = net(batch)\n",
        "        # output1 = outputs[:,3:,:,:]\n",
        "        # output2 = outputs[:,:3,:,:]\n",
        "        # # print(outputs.size())\n",
        "\n",
        "        # loss1 = criterion(output1, labels[:,:,32:,:].float())\n",
        "        # loss2 = criterion(output2, labels[:,:,:32,:].float())\n",
        "        # loss = loss1 + loss2\n",
        "\n",
        "        outputs1 = net1(batch)\n",
        "        # loss1 = criterion(outputs1, labels[:,:,32:,:].float())\n",
        "        plt.imshow()\n",
        "        loss1 = criterion(outputs1, labels.float())\n",
        "\n",
        "        loss1.backward(retain_graph=True)\n",
        "\n",
        "        # outputs2 = net2(batch)\n",
        "        # loss2 = criterion(outputs1, labels[:,:,32:,:].float())\n",
        "        # loss2 = criterion(outputs1, labels.float())\n",
        "\n",
        "        # loss2.backward(retain_graph=True)\n",
        "\n",
        "        optimizer1.step()\n",
        "        # optimizer2.step()\n",
        "\n",
        "        # compute training statistics\n",
        "        # _, predicted = torch.max(outputs, 1)\n",
        "        # print(loss)\n",
        "\n",
        "        # correct += (predicted == labels).sum().item()\n",
        "        running_loss += loss1.item()\n",
        "    print(running_loss)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cMTOi0FyIWS_"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "SUNxaizx9i5Y",
        "29-Le8azitBF",
        "9M9A3zj5EnvP",
        "iB5cYoxyCIF1"
      ],
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}