Deep Learning with PyTorch/DL-with-pytorch - 2 [MLP].ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"DL-with-pytorch - 2 [MLP].ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"bIAYbYajk1w9","colab_type":"text"},"cell_type":"markdown","source":["# Multi Layer Perceptrons\n","- CIFAR-10 image classification with MLP\n"]},{"metadata":{"id":"GVU5-yp3N89I","colab_type":"code","colab":{}},"cell_type":"code","source":["!pip3 install torch torchvision"],"execution_count":0,"outputs":[]},{"metadata":{"id":"8yy37hEYOEiQ","colab_type":"code","outputId":"4434d2e8-c5e8-4dce-8316-7f9230dc1b75","executionInfo":{"status":"ok","timestamp":1545794710219,"user_tz":420,"elapsed":1060,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","import torch, torchvision\n","import torch.nn as nn\n","import torch.nn.functional as F\n","torch.__version__"],"execution_count":1,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'1.0.0'"]},"metadata":{"tags":[]},"execution_count":1}]},{"metadata":{"id":"ewrw93tt2BfV","colab_type":"text"},"cell_type":"markdown","source":["## 1. Import & process dataset\n","- CIFAR10 dataset can be downloaded by ```torchvision```\n","  - [torchvision.datasets](https://pytorch.org/docs/stable/torchvision/datasets.html)"]},{"metadata":{"id":"W5anlYa01w3w","colab_type":"code","outputId":"dc43d91b-ad85-423f-fb18-1a0d2af22f25","executionInfo":{"status":"ok","timestamp":1545795192346,"user_tz":420,"elapsed":2530,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":52}},"cell_type":"code","source":["from torchvision import datasets\n","import torchvision.transforms as transforms\n","\n","train_dataset = datasets.CIFAR10(root = \"/\", train = True, download = True, transform = transforms.ToTensor())\n","test_dataset = datasets.CIFAR10(root = \"/\", train = False, download = True, transform = transforms.ToTensor())"],"execution_count":25,"outputs":[{"output_type":"stream","text":["Files already downloaded and verified\n","Files already downloaded and verified\n"],"name":"stdout"}]},{"metadata":{"id":"9nznVMdo5edZ","colab_type":"text"},"cell_type":"markdown","source":["## 2. Creating MLP model and training\n","\n","- Create and train MLP with 1 hidden layer\n","\n","![](http://cs231n.github.io/assets/nn1/neural_net.jpeg)"]},{"metadata":{"id":"YEoNQr10kGUW","colab_type":"code","colab":{}},"cell_type":"code","source":["# create data loaders \n","train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = 128, shuffle = True)\n","test_loader = torch.utils.data.DataLoader(test_dataset, batch_size = 128, shuffle = False)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"AQawpMRPI7jm","colab_type":"code","colab":{}},"cell_type":"code","source":["# create MLP with one hidden layer\n","class net(nn.Module):\n","  def __init__(self, input_dim, hidden_dim, output_dim):\n","    super(net, self).__init__()\n","    self.input_dim = input_dim\n","    self.dense1 = nn.Linear(input_dim, hidden_dim)\n","    self.relu = nn.ReLU()\n","    self.dense2 = nn.Linear(hidden_dim, output_dim)\n","  \n","  def forward(self, x):\n","    x = x.view(x.size(0), self.input_dim)\n","    x = self.dense1(x)\n","    x = self.relu(x)\n","    x = self.dense2(x)\n","    return x"],"execution_count":0,"outputs":[]},{"metadata":{"id":"rP0Gt5E9ajmd","colab_type":"code","colab":{}},"cell_type":"code","source":["# hyperparameters\n","DEVICE = torch.device('cuda')\n","INPUT_DIM = 32 * 32 * 3\n","HIDDEN_DIM = 50\n","OUTPUT_DIM = 10\n","LEARNING_RATE = 1e-3\n","NUM_EPOCHS = 10"],"execution_count":0,"outputs":[]},{"metadata":{"id":"cPBm8qDrSWsi","colab_type":"code","colab":{}},"cell_type":"code","source":["model = net(INPUT_DIM, HIDDEN_DIM, OUTPUT_DIM).to(DEVICE)\n","criterion = nn.CrossEntropyLoss()   # do not need softmax layer when using CEloss criterion\n","optimizer = torch.optim.RMSprop(model.parameters(), lr = LEARNING_RATE)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"SEBtAPYCFeic","colab_type":"code","outputId":"edda354d-30d8-40cb-fef0-9c2a4fd968eb","executionInfo":{"status":"ok","timestamp":1545795315447,"user_tz":420,"elapsed":89814,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":193}},"cell_type":"code","source":["# training for NUM_EPOCHS\n","for i in range(NUM_EPOCHS):\n","  temp_loss = []\n","  for (x, y) in train_loader:\n","    x, y = x.float().to(DEVICE), y.to(DEVICE)\n","    outputs = model(x)\n","    loss = criterion(outputs, y)\n","    temp_loss.append(loss.item())\n","    \n","    optimizer.zero_grad()\n","    loss.backward()\n","    optimizer.step()\n","    \n","  print(\"Loss at {}th epoch: {}\".format(i, np.mean(temp_loss)))"],"execution_count":34,"outputs":[{"output_type":"stream","text":["Loss at 0th epoch: 2.0718233820117646\n","Loss at 1th epoch: 1.8863357893951105\n","Loss at 2th epoch: 1.8327037266758093\n","Loss at 3th epoch: 1.7953610965967788\n","Loss at 4th epoch: 1.764086505336225\n","Loss at 5th epoch: 1.7412422986896448\n","Loss at 6th epoch: 1.7202257174055289\n","Loss at 7th epoch: 1.6997354250124959\n","Loss at 8th epoch: 1.6853232405069845\n","Loss at 9th epoch: 1.6730405244680926\n"],"name":"stdout"}]},{"metadata":{"id":"qpAJUiHm529m","colab_type":"text"},"cell_type":"markdown","source":["## 3. Evaluation\n","- Evaluate the trained MLP model with ROC AUC score \n","  - Store probability of each instance to a list and compare it with true y label"]},{"metadata":{"id":"txXH3dknFpSx","colab_type":"code","outputId":"9330f014-2687-4464-b495-1ba861511e38","executionInfo":{"status":"ok","timestamp":1545795466013,"user_tz":420,"elapsed":2278,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":72}},"cell_type":"code","source":["y_pred, y_true = [], []\n","with torch.no_grad():\n","  for x, y in test_loader:\n","    x, y = x.float().to(DEVICE), y.to(DEVICE)\n","    outputs = F.softmax(model(x)).max(1)[-1]       # predicted label\n","    y_true += list(y.cpu().numpy())                # true label\n","    y_pred += list(outputs.cpu().numpy())   "],"execution_count":39,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n","  \"\"\"\n"],"name":"stderr"}]},{"metadata":{"id":"HV1s3xf5Frkl","colab_type":"code","outputId":"cfe3f879-b673-4d29-ed3b-f1f6b6ee7452","executionInfo":{"status":"ok","timestamp":1545795467960,"user_tz":420,"elapsed":742,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["# evaluation result\n","from sklearn.metrics import accuracy_score\n","accuracy_score(y_true, y_pred)"],"execution_count":40,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.369"]},"metadata":{"tags":[]},"execution_count":40}]}]}