Transformers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "import texar.torch as tx\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from lazy_dataset import LazyDataset\n",
    "from sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score, precision_score, recall_score, roc_auc_score\n",
    "import pytorch_warmup as warmup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Failed to query for notebook name, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable\n",
      "wandb: Currently logged in as: holmeswww (use `wandb login --relogin` to force relogin)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Tracking run with wandb version 0.10.11<br/>\n",
       "                Syncing run <strong style=\"color:#cdcd00\">transformer</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://wandb.ai/holmeswww/ProjectX_transformer\" target=\"_blank\">https://wandb.ai/holmeswww/ProjectX_transformer</a><br/>\n",
       "                Run page: <a href=\"https://wandb.ai/holmeswww/ProjectX_transformer/runs/2vuc6ar8\" target=\"_blank\">https://wandb.ai/holmeswww/ProjectX_transformer/runs/2vuc6ar8</a><br/>\n",
       "                Run data is saved locally in <code>/mnt/Heavy_Duty/deep/peat/wandb/run-20201130_025513-2vuc6ar8</code><br/><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h1>Run(2vuc6ar8)</h1><p></p><iframe src=\"https://wandb.ai/holmeswww/ProjectX_transformer/runs/2vuc6ar8\" style=\"border:none;width:100%;height:400px\"></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7fa81c485790>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import wandb\n",
    "wandb.init(project='ProjectX_transformer', name=\"transformer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(10708)\n",
    "np.random.seed(seed=10708)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "order = ['GSOC', 'CT2019B_XCO2_global_7', 'Tarnocai', 'ERA5', 'CT2019B_XCO2_global_10', 'CT2019B_flux_fuel', 'CT2019B_XCO2_global_6', 'CT2019B_XCO2_global_8', 'CT2019B_XCO2_global_9', 'CWFIS', 'CT2019B_flux_fire', 'VIIRS', 'CT2019B_XCO2_global_4', 'MODIS', 'CT2019B_XCO2_global_3', 'CT2019B_XCO2_global_2', 'CT2019B_XCO2_global_0', 'CT2019B_XCO2_global_5', 'CT2019B_XCO2_global_1']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GSOC ['carbon']\n",
      "CT2019B_XCO2_global_7 ['h', 'e', 'i', 'g', 'h', 't', '_', '7']\n",
      "Tarnocai ['PEATLAND_P', 'TOCC', 'BOG_PCT', 'FEN_PCT', 'SWAMP_PCT', 'MARSH_PCT']\n",
      "ERA5 ['t2m', 'swvl1', 'swvl2', 'swvl3', 'swvl4', 'stl1', 'stl2', 'stl3', 'stl4', 'lai_lv', 'lai_hv', 'tp', 'u10', 'v10']\n",
      "CT2019B_XCO2_global_10 ['h', 'e', 'i', 'g', 'h', 't', '_', '1', '0']\n",
      "CT2019B_flux_fuel ['flux_fuel']\n",
      "CT2019B_XCO2_global_6 ['h', 'e', 'i', 'g', 'h', 't', '_', '6']\n",
      "CT2019B_XCO2_global_8 ['h', 'e', 'i', 'g', 'h', 't', '_', '8']\n",
      "CT2019B_XCO2_global_9 ['h', 'e', 'i', 'g', 'h', 't', '_', '9']\n",
      "CWFIS ['burned_pct_cat', 'burned_cause', 'fire_size_agency', 'fire_size_calc']\n",
      "CT2019B_flux_fire ['flux_fire']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/Heavy_Duty/deep/peat/lazy_dataset.py:30: H5pyDeprecationWarning: dataset.value has been deprecated. Use dataset[()] instead.\n",
      "  data.append((hf[\"data\"], hf[\"dates\"].value.view('<M8[D]'), feats, True))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VIIRS ['bright_ti4', 'confidence', 'frp']\n",
      "CT2019B_XCO2_global_4 ['h', 'e', 'i', 'g', 'h', 't', '_', '4']\n",
      "MODIS ['brightness', 'confidence', 'frp']\n",
      "CT2019B_XCO2_global_3 ['h', 'e', 'i', 'g', 'h', 't', '_', '3']\n",
      "CT2019B_XCO2_global_2 ['h', 'e', 'i', 'g', 'h', 't', '_', '2']\n",
      "CT2019B_XCO2_global_0 ['h', 'e', 'i', 'g', 'h', 't', '_', '0']\n",
      "CT2019B_XCO2_global_5 ['h', 'e', 'i', 'g', 'h', 't', '_', '5']\n",
      "CT2019B_XCO2_global_1 ['h', 'e', 'i', 'g', 'h', 't', '_', '1']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0b1e8d94a1341c7bf87bb365b59693f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='loading aligned index...', max=2538.0, style=ProgressStyl…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dataset = LazyDataset(order = order)\n",
    "idx = TensorDataset(torch.from_numpy(np.arange(len(dataset))))\n",
    "lengths = [int(len(idx)*0.8), int(len(idx)*0.02), len(idx) - int(len(idx)*0.8) - int(len(idx)*0.02)]\n",
    "train, val, test = torch.utils.data.dataset.random_split(idx, lengths)\n",
    "train_loader = DataLoader(train, batch_size=1, shuffle=True)\n",
    "test_loader = DataLoader(test, batch_size=1, shuffle=False)\n",
    "validation_loader = DataLoader(val, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DoubleConv(nn.Module):\n",
    "    \"\"\"(convolution => [BN] => ReLU) * 2\"\"\"\n",
    "\n",
    "    def __init__(self, in_channels, out_channels, mid_channels=None):\n",
    "        super().__init__()\n",
    "        if not mid_channels:\n",
    "            mid_channels = out_channels\n",
    "        self.double_conv = nn.Sequential(\n",
    "            nn.Conv2d(in_channels, mid_channels, kernel_size=3, padding=1),\n",
    "            nn.BatchNorm2d(mid_channels),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(mid_channels, out_channels, kernel_size=3, stride=2, padding=1),\n",
    "            nn.BatchNorm2d(out_channels)\n",
    "        )\n",
    "        self.downsample = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=2),\n",
    "                nn.BatchNorm2d(out_channels)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return F.relu_(self.double_conv(x) + self.downsample(x))\n",
    "\n",
    "class ConvEnc(nn.Module):\n",
    "    \n",
    "    def __init__(self, in_channels, hidden_dim = 256):\n",
    "        super().__init__()\n",
    "        self.first = nn.Sequential(nn.Conv2d(in_channels, hidden_dim, kernel_size=5, padding=2), nn.BatchNorm2d(hidden_dim), nn.ReLU(inplace=True))\n",
    "        self.L = nn.ModuleList([\n",
    "            DoubleConv(hidden_dim, hidden_dim),\n",
    "            DoubleConv(hidden_dim, hidden_dim),\n",
    "            DoubleConv(hidden_dim, hidden_dim)\n",
    "        ])\n",
    "\n",
    "    def forward(self, x):\n",
    "        x=self.first(x)\n",
    "        identity = x\n",
    "        for i in range(len(self.L)):\n",
    "            x = self.L[i](x)\n",
    "        return x\n",
    "\n",
    "class ResBlockGenerator(nn.Module):\n",
    "\n",
    "    def __init__(self, in_channels, out_channels):\n",
    "        super(ResBlockGenerator, self).__init__()\n",
    "\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, 3, 1, padding=1)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, 3, 1, padding=1)\n",
    "        nn.init.kaiming_uniform_(self.conv1.weight.data, mode='fan_in', nonlinearity='relu')\n",
    "        nn.init.kaiming_uniform_(self.conv2.weight.data, mode='fan_in', nonlinearity='relu')\n",
    "\n",
    "        self.model = nn.Sequential(\n",
    "            nn.BatchNorm2d(in_channels),\n",
    "            nn.ReLU(),\n",
    "            nn.Upsample(scale_factor=2),\n",
    "            self.conv1,\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "            nn.ReLU(),\n",
    "            self.conv2\n",
    "            )\n",
    "        self.bypass = nn.Upsample(scale_factor=2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.model(x) + self.bypass(x)\n",
    "\n",
    "class ConvDec(nn.Module):\n",
    "    \n",
    "    def __init__(self, out_channels, hidden_dim = 256):\n",
    "        super().__init__()\n",
    "        self.final = nn.Conv2d(hidden_dim, out_channels, 3, stride=1)\n",
    "        nn.init.xavier_uniform(self.final.weight.data, nn.init.calculate_gain('sigmoid'))\n",
    "#         self.first = nn.Sequential(nn.Conv2d(in_channels, hidden_dim, kernel_size=5, padding=2), nn.BatchNorm2d(hidden_dim), nn.ReLU(inplace=True))\n",
    "        self.model = nn.Sequential(\n",
    "            ResBlockGenerator(hidden_dim, hidden_dim),\n",
    "            ResBlockGenerator(hidden_dim, hidden_dim),\n",
    "            ResBlockGenerator(hidden_dim, hidden_dim),\n",
    "            nn.BatchNorm2d(hidden_dim),\n",
    "            nn.ReLU(),\n",
    "            self.final)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "    \n",
    "class Model(nn.Module):\n",
    "    \n",
    "    def __init__(self, in_channels, hidden_dim = 64):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.conv_enc = ConvEnc(in_channels, hidden_dim = hidden_dim)\n",
    "        self.conv_dec = ConvDec(5, hidden_dim = hidden_dim)\n",
    "        enc_hparam = tx.modules.TransformerEncoder.default_hparams()\n",
    "        enc_hparam['num_blocks']=3\n",
    "        enc_hparam['dim']=hidden_dim\n",
    "        enc_hparam['multihead_attention']['num_units']=hidden_dim\n",
    "        enc_hparam['multihead_attention']['output_dim']=hidden_dim\n",
    "        enc_hparam['poswise_feedforward']={'layers': [{'type': 'Linear', 'kwargs': {'in_features': hidden_dim, 'out_features': hidden_dim*2, 'bias': True}}, {'type': 'ReLU', 'kwargs': {'inplace': True}}, {'type': 'Dropout', 'kwargs': {'p': 0.1}}, {'type': 'Linear', 'kwargs': {'in_features': hidden_dim*2, 'out_features': hidden_dim, 'bias': True}}], 'name': 'ffn'}\n",
    "        self.enc = tx.modules.TransformerEncoder(hparams = enc_hparam)\n",
    "        self.enc.initialize_blocks()\n",
    "\n",
    "    def forward(self, X):\n",
    "        size = X.shape[2]\n",
    "        c = self.conv_enc(X)\n",
    "        shape = c.shape\n",
    "        c = c.view(*c.shape[:2],-1).permute(2,0,1)\n",
    "        e = self.enc(c, torch.full((c.shape[0], ), c.shape[1]))\n",
    "        e = e.permute(1,2,0).reshape(shape)\n",
    "        d = self.conv_dec(e[-1].unsqueeze(0))\n",
    "        d = d[:,:,(d.shape[2]-size)//2:(d.shape[2]-size)//2+size,:]\n",
    "        return d.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:70: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20250\n"
     ]
    }
   ],
   "source": [
    "model = Model(44).cuda()\n",
    "wandb.watch(model)\n",
    "learning_rate = 1e-4\n",
    "opt = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "criterion = torch.nn.BCEWithLogitsLoss(pos_weight=torch.FloatTensor([1/0.0008]), reduction='mean', weight = dataset.mask.view(-1)).cuda()\n",
    "num_steps = len(train_loader) * 10\n",
    "print(num_steps)\n",
    "lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=num_steps, eta_min=3e-5)\n",
    "warmup_scheduler = warmup.UntunedLinearWarmup(opt)\n",
    "class Weighted_MSE(nn.Module):\n",
    "    \n",
    "    def __init__(self, weight):\n",
    "        super().__init__()\n",
    "        self.weight = torch.nn.Parameter(weight.unsqueeze(0), requires_grad=False)\n",
    "\n",
    "    def forward(self, X, target, mask):\n",
    "        return torch.mean(self.weight * mask * (X - target) ** 2)\n",
    "\n",
    "WMSE = Weighted_MSE(dataset.mask).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load(\"model_1.pt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve, roc_curve, confusion_matrix, explained_variance_score, mean_squared_error, mean_absolute_error, r2_score\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subsample(A, B, length=100):\n",
    "    assert(A.shape[0] == B.shape[0])\n",
    "    l = A.shape[0]\n",
    "    if l<length:\n",
    "        return A,B\n",
    "    L = np.ceil(l/length)*length\n",
    "    d = int(L-l)\n",
    "    A,B = np.pad(A, (int(d/2), d-int(d/2)),'edge'), np.pad(B, (int(d/2), d-int(d/2)),'edge')\n",
    "    A,B = np.average(A.reshape(length, -1), axis=1), np.average(B.reshape(length, -1), axis=1)\n",
    "    return A,B\n",
    "\n",
    "def evaluate(loader, prefix=\"val\"):\n",
    "    model.eval()\n",
    "    preds = []\n",
    "    labels = []\n",
    "    reg_y = []\n",
    "    reg_Y = []\n",
    "    with torch.no_grad():\n",
    "        for indexes in tqdm(loader, desc=\"Evaluating...\"):\n",
    "            X, Y = dataset[indexes[0]]\n",
    "            labels.append(Y[0].view(-1).data.numpy()[dataset.mask.view(-1).numpy()==1]/4)\n",
    "            reg_Y.append(Y[1:].view(-1,4).data.numpy()[np.logical_and(dataset.mask.view(-1).numpy()==1, Y[0].view(-1).data.numpy()>0)].reshape(-1))\n",
    "            X = X.cuda()\n",
    "            pred = model(X)\n",
    "            preds.append(torch.sigmoid(pred[0]).view(-1).data.cpu().numpy()[dataset.mask.view(-1).numpy()==1])\n",
    "            reg_y.append((pred[1:]*8.92 + 405).view(-1,4).data.cpu().numpy()[np.logical_and(dataset.mask.view(-1).numpy()==1, Y[0].view(-1).data.numpy()>0)].reshape(-1))\n",
    "\n",
    "    model.train()\n",
    "    preds = np.concatenate(preds)\n",
    "    reg_y = np.concatenate(reg_y)\n",
    "    reg_Y = np.concatenate(reg_Y)\n",
    "    preds_bin = preds>=0.5\n",
    "#     print(Y[0])\n",
    "    labels = np.concatenate(labels)>0\n",
    "    fpr, tpr, threshold = roc_curve(labels, preds)\n",
    "    fpr,tpr = subsample(fpr,tpr)\n",
    "    fig_roc, ax = plt.subplots()\n",
    "    ax.set_title('Receiver Operating Characteristic ({})'.format(prefix))\n",
    "    ax.plot(fpr, tpr)\n",
    "    ax.plot([0, 1], [0, 1], linestyle='--', label='Reference')\n",
    "    ax.legend()\n",
    "    ax.set_xlabel('False Positive Rate')\n",
    "    ax.set_ylabel('True Positive Rate')\n",
    "\n",
    "    precision, recall, threshold = precision_recall_curve(labels, preds)\n",
    "    precision, recall = subsample(precision, recall)\n",
    "    fig_pr, ax = plt.subplots()\n",
    "    ax.set_title('Precision Recall ({})'.format(prefix))\n",
    "    ax.plot(recall, precision)\n",
    "#     ax.plot([0, 1], [1, 0], linestyle='--', label='Reference')\n",
    "    ax.set_xlabel('Recall')\n",
    "    ax.set_ylabel('Precision')\n",
    "    \n",
    "    cf_matrix = confusion_matrix(labels,preds_bin)\n",
    "    fig_confusion, ax = plt.subplots()\n",
    "#     ax.set_title('Confusion Matrix ({})'.format(prefix))\n",
    "    group_names = ['True Neg','False Pos','False Neg','True Pos']\n",
    "    group_counts = [\"{0:0.0f}\".format(value) for value in\n",
    "                    cf_matrix.flatten()]\n",
    "    group_percentages = [\"{0:.2%}\".format(value) for value in\n",
    "                         cf_matrix.flatten()/np.sum(cf_matrix)]\n",
    "    lbl = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "              zip(group_names,group_counts,group_percentages)]\n",
    "    lbl = np.asarray(lbl).reshape(2,2)\n",
    "    sns.heatmap(cf_matrix, annot=lbl, fmt='', cmap='Blues', ax = ax)\n",
    "    return {\n",
    "        prefix+\"_accuracy_score\":accuracy_score(labels,preds_bin),\n",
    "        prefix+\"_balanced_accuracy_score\":balanced_accuracy_score(labels,preds_bin), \n",
    "        prefix+\"_f1_score\":f1_score(labels, preds_bin), \n",
    "        prefix+\"_precision_score\":precision_score(labels, preds_bin), \n",
    "        prefix+\"_recall_score\":recall_score(labels, preds_bin), \n",
    "        prefix+\"_roc_auc_score\":roc_auc_score(labels, preds),\n",
    "        prefix+\"_pr\" : fig_pr,\n",
    "        prefix+\"_roc\" : fig_roc,\n",
    "        prefix+\"_confusion\" : fig_confusion,\n",
    "        prefix+\"_MSE\" : mean_squared_error(reg_Y, reg_y),\n",
    "        prefix+\"_MAE\" : mean_absolute_error(reg_Y, reg_y),\n",
    "        prefix+\"_r2\" : r2_score(reg_Y, reg_y),\n",
    "        prefix+\"_explained_variance_score\" : explained_variance_score(reg_Y, reg_y),\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results = evaluate(validation_loader)\n",
    "# wandb.log(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "929a3a1a844647cba09fb73bfb423ec8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb7b7d88ad0a4e79a2dfff3f3bdb83d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Training...', max=2025.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dfe1a3adc4643f3a7cc5914e4f863e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/conda-bld/pytorch_1591914855613/work/aten/src/ATen/native/TensorFactories.cpp:361: UserWarning: Deprecation warning: In a future PyTorch release torch.full will no longer return tensors of floating dtype by default. Instead, a bool fill_value will return a tensor of torch.bool dtype, and an integral fill_value will return a tensor of torch.long dtype. Set the optional `dtype` or `out` arguments to suppress this warning.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8616d3869ff4b3b98093138ca2c1bad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3575320035e14ea09169eb47085ea5a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b0ebfab45244adead8800f8a919c307",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa435edd7ec546fba95bbba7eed41283",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6b91001b8954bb1bea15a21d83d1450",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=457.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dffa1bf0dd894041ad3011dc44782c78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Training...', max=2025.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f7194d7a91c4ea79f30c02362b7c367",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6d61c738ebb406f91602288999ebf87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a9dbf129fbf47ce8ca34d25ff4f11d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3d5948ddb71424da1bb2bc392a97cfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e6a7adcef3645129582d215e3677b60",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=457.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bbb12f9a73d40c3a37b6b0aff19b4db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Training...', max=2025.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2dbe5e0ad8d4c38b10a4503dc7cd295",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "278e02632dec4030bf94dd8c872a7d51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Evaluating...', max=50.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:410: UserWarning:\n",
      "\n",
      "Bummer! Plotly can currently only draw Line2D objects from matplotlib that are in 'data' coordinates!\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:512: UserWarning:\n",
      "\n",
      "I found a path object that I don't think is part of a bar chart. Ignoring.\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/plotly/matplotlylib/renderer.py:474: UserWarning:\n",
      "\n",
      "Dang! That path collection is out of this world. I totally don't know what to do with it yet! Plotly can only import path collections linked to 'data' coordinates\n",
      "\n",
      "/home/holmes/anaconda3/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:143: UserWarning:\n",
      "\n",
      "The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.train()\n",
    "Eval = 500\n",
    "for i in tqdm(range(100)):\n",
    "    for indexes in tqdm(train_loader, desc=\"Training...\"):\n",
    "        if c % Eval == 0:\n",
    "            results = evaluate(validation_loader)\n",
    "            wandb.log(results)\n",
    "        X, Y = dataset[indexes[0]]\n",
    "        X, Y = X.cuda(), Y.cuda()\n",
    "        pred = model(X)\n",
    "        bceloss = criterion(pred[0].view(-1), Y[0].view(-1)/4)\n",
    "        mseloss = WMSE(pred[1:], (Y[1:]-405)/8.92, (Y[0].unsqueeze(0)/4 + 0.001))\n",
    "        loss = bceloss + 10 * mseloss\n",
    "        opt.zero_grad()\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        lr_scheduler.step(lr_scheduler.last_epoch+1)\n",
    "        warmup_scheduler.dampen()\n",
    "        wandb.log({\"loss\": loss.item(),\"MSE_loss\": mseloss.item(),\"BCE_loss\": bceloss.item(), \"lr\": opt.param_groups[0]['lr']})\n",
    "        c+=1\n",
    "    torch.save(model.state_dict(), os.path.join(wandb.run.dir, 'model_{}.pt'.format(i)))\n",
    "    torch.save(opt.state_dict(), os.path.join(wandb.run.dir, 'opt_{}.pt'.format(i)))\n",
    "    results = evaluate(test_loader, prefix=\"test\")\n",
    "    wandb.log(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}