experiment_number_of_test_predictions_2_cifar10_simplecnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Bka_bK83VFHh"
   },
   "source": [
    "## Experiment Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8D5NSPs_cJZe"
   },
   "source": [
    "### Random seed / PyTorch / CUDA related"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 243
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6214,
     "status": "ok",
     "timestamp": 1575020214134,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "pHbfpytEVFHu",
    "outputId": "2a30e001-783e-4de2-f1ce-8211ba20613d"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import datetime\n",
    "import os\n",
    "import sys\n",
    "import itertools\n",
    "\n",
    "# Use Google Colab\n",
    "use_colab = True\n",
    "\n",
    "# Is this notebook running on Colab?\n",
    "# If so, then google.colab package (github.com/googlecolab/colabtools)\n",
    "# should be available in this environment\n",
    "\n",
    "# Previous version used importlib, but we could do the same thing with\n",
    "# just attempting to import google.colab\n",
    "try:\n",
    "    from google.colab import drive\n",
    "    colab_available = True\n",
    "except:\n",
    "    colab_available = False\n",
    "\n",
    "if use_colab and colab_available:\n",
    "    drive.mount('/content/drive')\n",
    "    \n",
    "    # If there's a package I need to install separately, do it here\n",
    "    !pip install pyro-ppl\n",
    "    \n",
    "    # Install ronald_bdl package\n",
    "    !pip install git+https://github.com/ronaldseoh/ronald_bdl.git\n",
    "\n",
    "    # cd to the appropriate working directory under my Google Drive\n",
    "    %cd 'drive/My Drive/Colab Notebooks/bayesian-dl-experiments'\n",
    "    \n",
    "    # List the directory contents\n",
    "    !ls\n",
    "\n",
    "# IPython reloading magic\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Random seeds\n",
    "# Based on https://pytorch.org/docs/stable/notes/randomness.html\n",
    "random_seed = 682"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eag8z68GGbRu"
   },
   "source": [
    "### Third party libraries (NumPy, PyTorch, Pyro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6773,
     "status": "ok",
     "timestamp": 1575020214716,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "dHn9F3GMGbR1",
    "outputId": "97858cb4-449b-4120-901e-3eff9141c218"
   },
   "outputs": [],
   "source": [
    "# Third party libraries import\n",
    "import numpy as np\n",
    "import torch\n",
    "import pyro\n",
    "import matplotlib.pyplot as plt\n",
    "import tqdm\n",
    "\n",
    "# Print version information\n",
    "print(\"NumPy Version: \" + np.__version__)\n",
    "print(\"PyTorch Version: \" + torch.__version__)\n",
    "print(\"Pyro Version: \" + pyro.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7006,
     "status": "ok",
     "timestamp": 1575020214967,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "PrhSLV8pGbSB",
    "outputId": "adeab5e6-c83e-4ff0-8b87-e0e885bf0570"
   },
   "outputs": [],
   "source": [
    "# More imports...\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import random_split, DataLoader, RandomSampler\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from pyro.infer import SVI, Trace_ELBO, HMC, MCMC\n",
    "\n",
    "# Import model and dataset classes from ronald_bdl\n",
    "from ronald_bdl import models, datasets\n",
    "from ronald_bdl.models import utils\n",
    "\n",
    "# pyplot setting\n",
    "%matplotlib inline\n",
    "\n",
    "# torch.device / CUDA Setup\n",
    "use_cuda = True\n",
    "\n",
    "if use_cuda and torch.cuda.is_available():\n",
    "    torch_device = torch.device('cuda')\n",
    "\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "            \n",
    "    # Disable 'benchmark' mode\n",
    "    # Note: https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    use_pin_memory = True # Faster Host to GPU copies with page-locked memory\n",
    "\n",
    "    # CUDA libraries version information\n",
    "    print(\"CUDA Version: \" + str(torch.version.cuda))\n",
    "    print(\"cuDNN Version: \" + str(torch.backends.cudnn.version()))\n",
    "    print(\"CUDA Device Name: \" + str(torch.cuda.get_device_name()))\n",
    "    print(\"CUDA Capabilities: \"+ str(torch.cuda.get_device_capability()))    \n",
    "else:\n",
    "    torch_device = torch.device('cpu')\n",
    "    use_pin_memory = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XIFRoH3AcJZn"
   },
   "source": [
    "### Variable settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1NBuZI0cHnHy"
   },
   "source": [
    "#### Data prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CqiyKf1PHtR6"
   },
   "outputs": [],
   "source": [
    "# CIFAR10 data transformation setting\n",
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "# Set the proportion of the original dataset to be available as a whole\n",
    "subset_proportions = [0.01, 0.1, 1]\n",
    "\n",
    "# Proportion of the dataset (after getting the subset) to be used for training\n",
    "train_prop = 0.9\n",
    "\n",
    "# Number of dataset splits\n",
    "n_splits = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eBszD08sH4c4"
   },
   "source": [
    "#### NN settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "d6Z8-H-YH8kQ"
   },
   "outputs": [],
   "source": [
    "# Dropout\n",
    "dropout_rates = [0.1, 0.3, 0.5]\n",
    "\n",
    "# Length-scale\n",
    "length_scale_values = [1e-2]\n",
    "\n",
    "# Model Precision\n",
    "tau_values = [0.1, 0.15, 0.2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0U0QTMyEIVDr"
   },
   "source": [
    "#### Training setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g_pzGq1_cJZp"
   },
   "outputs": [],
   "source": [
    "# Number of epochs\n",
    "n_epoch = 4000\n",
    "\n",
    "# Optimizer learning rate\n",
    "learning_rate = 0.01\n",
    "\n",
    "# Data batch sizes\n",
    "n_training_batch = 512\n",
    "\n",
    "# Number of test predictions (for each data point)\n",
    "prediction_runs = [4, 10, 40, 100, 400, 1000]\n",
    "\n",
    "# Cross Entropy to minimize\n",
    "objective = nn.CrossEntropyLoss()\n",
    "\n",
    "# Test start time\n",
    "test_start_time = datetime.datetime.today().strftime('%Y%m%d%H%M')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1PpzPMI8VFKE"
   },
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1585801,
     "status": "ok",
     "timestamp": 1575025822894,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "m4kavCiTVFKf",
    "outputId": "78d6dac6-74f5-4ea5-d45b-347f4b118049",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for subset_prop, dropout_rate, length_scale, tau in itertools.product(\n",
    "    subset_proportions,\n",
    "    dropout_rates, length_scale_values, tau_values,\n",
    "):\n",
    "\n",
    "    # Reset the random number generator for each method (to produce identical results)\n",
    "    torch.manual_seed(random_seed)\n",
    "    np.random.seed(random_seed)\n",
    "    pyro.set_rng_seed(random_seed)\n",
    "\n",
    "    \"\"\"\n",
    "    Results file storage\n",
    "    \"\"\"\n",
    " \n",
    "    # Create directory to store results for the current test configuration\n",
    "    test_results_path = os.path.join(\n",
    "        './test_results',\n",
    "        'number_of_test_predictions_2',\n",
    "        'CIFAR-10',\n",
    "        test_start_time,\n",
    "        (\n",
    "            str(subset_prop)\n",
    "            + '_' + str(dropout_rate) \n",
    "            + '_' + str(length_scale)\n",
    "            + '_' + str(tau)),\n",
    "    )\n",
    "    \n",
    "    os.makedirs(test_results_path, exist_ok=True)\n",
    "    \n",
    "    test_results_accuracy_mc_path = os.path.join(\n",
    "        test_results_path,\n",
    "        \"accuracy_mc.txt\"\n",
    "    )\n",
    "    \n",
    "    test_results_accuracy_non_mc_path = os.path.join(\n",
    "        test_results_path,\n",
    "        \"accuracy_non_mc.txt\"\n",
    "    )\n",
    "\n",
    "    # Prepare new subset of the original dataset\n",
    "    subset = datasets.CIFAR10(\n",
    "        root='./datasets_files', limit_size=subset_prop, transform=transform, download=True)\n",
    "\n",
    "    # Determine sizes of training and testing set\n",
    "    train_size = int(train_prop * len(subset))\n",
    "    test_size = len(subset) - train_size\n",
    "    \n",
    "    # Print the size of the subset\n",
    "    print(\"subset size = \" + str(subset.data.shape))\n",
    "    print(\"training set size = %d\" % train_size)\n",
    "    print(\"test set size = %d\" % test_size)\n",
    "    \n",
    "    train, test = random_split(subset, lengths=[train_size, test_size])\n",
    "\n",
    "    train_loader = DataLoader(train, batch_size=n_training_batch, pin_memory=use_pin_memory)\n",
    "\n",
    "    # Prepare network\n",
    "    network = models.SimpleCIFAR10(\n",
    "        dropout_rate=dropout_rate,\n",
    "        dropout_type='bernoulli',\n",
    "    )\n",
    "    \n",
    "    # Send the whole model to the selected torch.device\n",
    "    network.to(torch_device)\n",
    "\n",
    "    # Print the network structure\n",
    "    print(network)\n",
    "    \n",
    "    # Model to train mode\n",
    "    network.train()\n",
    "    \n",
    "    # Adam optimizer\n",
    "    # https://pytorch.org/docs/stable/optim.html?highlight=adam#torch.optim.Adam\n",
    "\n",
    "    # NOTE: Need to set L2 regularization from here\n",
    "    reg_strength = utils.reg_strength(dropout_rate, length_scale, train_size, tau)\n",
    "    print('reg_strength = ' + str(reg_strength))\n",
    "    \n",
    "    optimizer = optim.Adam(\n",
    "        network.parameters(),\n",
    "        lr=learning_rate,\n",
    "        weight_decay=reg_strength, # L2 regularization\n",
    "    )\n",
    "\n",
    "    print()\n",
    "\n",
    "    \"\"\"\n",
    "    Training\n",
    "    \"\"\"\n",
    "\n",
    "    print(\n",
    "        \"Starting subset %f, dropout_rate %f, length_scale %f, tau %f\"\n",
    "        % (subset_prop, dropout_rate, length_scale, tau))\n",
    "\n",
    "    # Record training start time (for this split)\n",
    "    tic = time.time()\n",
    "\n",
    "    progress_bar = tqdm.tqdm(range(n_epoch))\n",
    "\n",
    "    for epoch in : # loop over the dataset multiple times\n",
    "        for i, data in enumerate(train_loader):\n",
    "            # get the inputs; data is a list of [inputs, labels]\n",
    "            inputs, targets = data\n",
    "\n",
    "            # Store the batch to torch_device's memory\n",
    "            inputs = inputs.to(torch_device)\n",
    "            targets = targets.to(torch_device)\n",
    "\n",
    "            # zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # forward + backward + optimize\n",
    "            outputs = network(inputs)\n",
    "\n",
    "            loss = objective(outputs, targets)\n",
    "            loss.backward()\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "        progress_bar.set_description(\"[iteration %04d] loss: %.4f\" % (epoch + 1, loss.item()))\n",
    "            \n",
    "    # Record training end time\n",
    "    toc = time.time()\n",
    "\n",
    "    # Report the final loss\n",
    "    print(\"final loss = %f\" % (loss.item()))\n",
    "\n",
    "    \"\"\"\n",
    "    Testing\n",
    "    \"\"\"\n",
    "\n",
    "    # Model to eval mode\n",
    "    network.eval()\n",
    "\n",
    "    for n_predictions in prediction_runs:\n",
    "        print(str(n_predictions) + \" test runs...\")\n",
    "\n",
    "        # Get the test data\n",
    "        test_loader = DataLoader(test, batch_size=n_training_batch, pin_memory=use_pin_memory)\n",
    "\n",
    "        # Record testing start time (for this split)\n",
    "        tic_testing = time.time()\n",
    "\n",
    "        _, mean, metrics = network.predict_dist(test_loader, n_predictions)\n",
    "\n",
    "        # Record testing end time\n",
    "        toc_testing = time.time()\n",
    "\n",
    "        # store additional metrics\n",
    "        if len(metrics) > 0:\n",
    "\n",
    "            for key, value in metrics.items():\n",
    "                print(str(key) + \" = \" + str(value))\n",
    "\n",
    "                if key == 'accuracy_mc':\n",
    "                    with open(test_results_accuracy_mc_path, 'a+') as accuracy_mc_file:\n",
    "                        accuracy_mc_file.write('%d %f \\n' % (n_predictions, value))\n",
    "\n",
    "                elif key == 'accuracy_non_mc':\n",
    "                    with open(test_results_accuracy_non_mc_path, 'a+') as accuracy_non_mc_file:\n",
    "                        accuracy_non_mc_file.write('%d %f \\n' % (n_predictions, value))\n",
    "                print()\n",
    "            \n",
    "    # Report the total training time\n",
    "    print(\"training time = \" + str(toc - tic) + \" seconds\")\n",
    "    \n",
    "    # Report the total testing time\n",
    "    print(\"testing time (last run) = \" + str(toc_testing - tic_testing) + \" seconds\")\n",
    "\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pDxkRM5aVrdf"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "experiment_nn_capacity_2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}