nbs/exercises/05_wavenet_exercise.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bf09f86b",
   "metadata": {},
   "source": [
    "# Wavenet Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32dd6a19-db09-4a78-9f66-66a416f5cf95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install ray[tune]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c252421",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee0815e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "import random\n",
    "import os\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch import optim\n",
    "import ray\n",
    "from ray import tune\n",
    "from ray.tune import CLIReporter, JupyterNotebookReporter\n",
    "from ray.tune.schedulers import ASHAScheduler\n",
    "from ray.tune.schedulers.pb2 import PB2\n",
    "from ray.air import session\n",
    "from ray.tune.suggest.optuna import OptunaSearch\n",
    "from ray.air import Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7e1df47-e95f-430f-96c0-12e5c7969860",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cpu\"\n",
    "if torch.cuda.is_available():\n",
    "    device = \"cuda:0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ed816f9-a512-42d2-a0dc-f5f0ba0636c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda:0'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29f0b5a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(42);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5404db14",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca34c512",
   "metadata": {},
   "source": [
    "### Setup Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1b7a5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "words = open('names.txt', 'r').read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98d76291",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d27e029e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, block_size=8):\n",
    "    \n",
    "    X, Y = [], []\n",
    "    \n",
    "    random.seed(42)\n",
    "    random.shuffle(words)\n",
    "    \n",
    "    chars = sorted(list(set(''.join(words))))\n",
    "    stoi = {s: i + 1 for i, s in enumerate(chars)}\n",
    "    stoi['.'] = 0\n",
    "    itos = {i: s for s, i in stoi.items()}\n",
    "    vocab_size = len(itos)\n",
    "    \n",
    "    for w in words:\n",
    "        context = [0] * block_size\n",
    "        for ch in w + '.':\n",
    "            ix = stoi[ch]\n",
    "            X.append(context)\n",
    "            Y.append(ix)\n",
    "            context = context[1:] + [ix]\n",
    "    \n",
    "    X = torch.tensor(X).to(device)\n",
    "    Y = torch.tensor(Y).to(device)\n",
    "    return X, Y\n",
    "\n",
    "n1 = int(0.8 * len(words))\n",
    "n2 = int(0.9 * len(words))\n",
    "\n",
    "Xtr, Ytr = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xte, Yte = build_dataset(words[n2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a1f4953",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([182625, 8])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtr.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cabedf8a-25b9-4f90-bac5-fb44546fb38a",
   "metadata": {},
   "source": [
    "### Create Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11fe3309",
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Tanh --- layer\n",
    "class Tanh:\n",
    "    def __call__(self, x):\n",
    "        self.out = torch.tanh(x).to(device)\n",
    "        return self.out\n",
    "    \n",
    "    def parameters(self):\n",
    "        return []\n",
    "    \n",
    "# --- Linear --- layer\n",
    "class Linear:\n",
    "    \n",
    "    def __init__(self, fan_in, fan_out, bias=True):\n",
    "        self.weight = (torch.randn((fan_in, fan_out)) / fan_in ** 0.5).to(device) # note: kaiming init\n",
    "        self.bias = torch.zeros(fan_out).to(device) if bias else None #\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        self.out = x @ self.weight\n",
    "        if self.bias is not None:\n",
    "            self.out += self.bias\n",
    "        return self.out\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [self.weight] + ([] if self.bias is None else [self.bias])\n",
    "\n",
    "\n",
    "# --- Embedding --- layer\n",
    "class Embedding: \n",
    "    def __init__(self, num_embeddings, embedding_dim):\n",
    "        self.weight = torch.randn((num_embeddings, embedding_dim)).to(device)\n",
    "        \n",
    "    def __call__(self, IX):\n",
    "        self.out = self.weight[IX]\n",
    "        return self.out\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [self.weight]\n",
    "\n",
    "# --- Sequential ----\n",
    "class Sequential:\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "        \n",
    "    def __call__(self, x):\n",
    "        B, T, C = x.shape\n",
    "        x = x.view(B, T//self.n, C*self.n)\n",
    "        if x.shape[1] == 1: \n",
    "            x = x.squeeze(1)\n",
    "        self.out = x\n",
    "        return self.out\n",
    "    \n",
    "    def parameters(self):\n",
    "        # get parameters of all layers and stretch them out into one list\n",
    "        return [p for layer in self.layers for p in layer.parameters()]\n",
    "\n",
    "# --- BatchNorm1d ---\n",
    "class BatchNorm1d:\n",
    "    \n",
    "    def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "        self.eps = eps\n",
    "        self.momentum = momentum\n",
    "        self.training = True\n",
    "        # parameters (trained with backprop)\n",
    "        self.gamma = torch.ones(dim).to(device)\n",
    "        self.beta = torch.zeros(dim).to(device)\n",
    "        # buffers (trained with a running `momentum update`)\n",
    "        self.running_mean = torch.zeros(dim).to(device)\n",
    "        self.running_var = torch.ones(dim).to(device)\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        # calculate the forward pass\n",
    "        if self.training:\n",
    "            if x.ndim == 2: dim = 0\n",
    "            elif x.ndim == 3: dim = (0, 1)\n",
    "            xmean = x.mean(dim, keepdim=True)\n",
    "            xvar = x.var(dim, keepdim=True)\n",
    "        else:\n",
    "            # print('In batchnorm call')\n",
    "            xmean = self.running_mean\n",
    "            xvar = self.running_var\n",
    "        \n",
    "        xhat = (x - xmean) / torch.sqrt(xvar + self.eps)\n",
    "        self.out = self.gamma * xhat + self.beta\n",
    "        # update the buffers\n",
    "        if self.training:\n",
    "            with torch.no_grad():\n",
    "                self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * xmean\n",
    "                self.running_var = (1 - self.momentum) * self.running_var + self.momentum * xvar\n",
    "        return self.out\n",
    "    \n",
    "    def parameters(self):\n",
    "        return [self.gamma, self.beta]\n",
    "\n",
    "# --- Flatten Consecutive ---\n",
    "class FlattenConsecutive:\n",
    "    def __init__(self, n):\n",
    "        self.n = n\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "        self.out = x\n",
    "        return self.out\n",
    "    \n",
    "    def parameters(self):\n",
    "        # get parameters of all layers and stretch them out into one list\n",
    "        return [p for layer in self.layers for p in layer.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3f11c1e-d7fb-4adc-bd47-dc79d9a5c7e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76579\n"
     ]
    }
   ],
   "source": [
    "def build_model(n_embd, # the dimensionality of the character embedding vectors\n",
    "                n_hidden, # the number of neurons in the hidden layer of the MLP \n",
    "                last_layer_factor = 0.1 # the factor by to reduce the weights of the last layer\n",
    "               ):\n",
    "    vocab_size = 27\n",
    "    model = Sequential([\n",
    "        Embedding(vocab_size, n_embd),\n",
    "        FlattenConsecutive(2), Linear(n_embd*2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "        FlattenConsecutive(2), Linear(n_hidden*2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "        FlattenConsecutive(2), Linear(n_hidden*2, n_hidden, bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "        Linear(n_hidden, vocab_size),\n",
    "    ])\n",
    "\n",
    "\n",
    "    # parameter init\n",
    "    with torch.no_grad(): model.layers[-1].weight *= last_layer_factor\n",
    "\n",
    "    parameters = model.parameters()\n",
    "    print(\"No of parameters \", sum(p.nelement() for p in parameters))\n",
    "    for p in parameters: p.requires_grad = True\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b57c57b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_model(24, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af66e115",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 8])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0,  0,  0,  0,  0,  0,  0],\n",
       "        [ 0,  0,  0,  0,  0,  0,  0,  6],\n",
       "        [ 0,  0,  0,  0,  0,  0,  0, 12],\n",
       "        [ 0,  0,  0,  2, 18,  9,  5, 12]])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ix = torch.randint(0, Xtr.shape[0], (4,))\n",
    "Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "logits = model(Xb)\n",
    "print(Xb.shape)\n",
    "Xb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8584dfc0-ce59-4211-85c9-b53fcba9fce5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0273, -0.0539,  0.0103,  0.0468,  0.0906, -0.0791,  0.0134, -0.0599,\n",
       "         -0.0407, -0.0617,  0.0134, -0.0230, -0.0095,  0.0769,  0.0231,  0.0755,\n",
       "         -0.0331, -0.0292,  0.0256, -0.1219,  0.0147, -0.0233,  0.0642, -0.0846,\n",
       "         -0.1325,  0.0057,  0.1157],\n",
       "        [ 0.0004,  0.0353, -0.0831,  0.0284, -0.0333,  0.0517, -0.0613, -0.0264,\n",
       "          0.0924, -0.0422, -0.0844, -0.0041,  0.0009,  0.0050,  0.0096,  0.0046,\n",
       "          0.0754, -0.0220, -0.0776,  0.0863, -0.0258,  0.0097,  0.1024,  0.0146,\n",
       "          0.1084, -0.0321, -0.0900],\n",
       "        [-0.0469,  0.0274,  0.0240,  0.0247, -0.0269, -0.0069,  0.1197, -0.0066,\n",
       "         -0.1216,  0.0030, -0.0689, -0.0530,  0.0465, -0.0710, -0.0207, -0.0524,\n",
       "         -0.0485,  0.0697,  0.0468,  0.0067, -0.0114,  0.0317, -0.0701,  0.0484,\n",
       "          0.1649,  0.0206, -0.0751],\n",
       "        [ 0.0238, -0.0383, -0.0129, -0.1187, -0.0069, -0.0030, -0.0763,  0.0728,\n",
       "          0.0896,  0.1063,  0.1267,  0.0320, -0.0092, -0.0483, -0.0477, -0.0461,\n",
       "          0.0270,  0.0346,  0.0168,  0.0283,  0.0219, -0.0034, -0.1050,  0.0165,\n",
       "         -0.1496, -0.0124,  0.0358]], device='cuda:0', grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c421b37-9cf4-4869-a86c-e9cbdfb7fadf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 range(0, 1)\n",
      "1 range(1, 2)\n",
      "2 range(2, 3)\n",
      "3 range(3, 4)\n",
      "4 range(4, 5)\n",
      "5 range(5, 6)\n",
      "6 range(6, 7)\n",
      "7 range(7, 8)\n",
      "8 range(8, 9)\n",
      "9 range(9, 10)\n"
     ]
    }
   ],
   "source": [
    "for i, ix in enumerate(batch(range(10))):\n",
    "    print(i, ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ede064c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch(iterable, n=1):\n",
    "    l = len(iterable)\n",
    "    for ndx in range(0, l, n):\n",
    "        yield iterable[ndx:min(ndx + n, l)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c1c5960",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 range(0, 1)\n",
      "1 range(1, 2)\n",
      "2 range(2, 3)\n",
      "3 range(3, 4)\n",
      "4 range(4, 5)\n",
      "5 range(5, 6)\n",
      "6 range(6, 7)\n",
      "7 range(7, 8)\n",
      "8 range(8, 9)\n",
      "9 range(9, 10)\n"
     ]
    }
   ],
   "source": [
    "for i, ix in enumerate(batch(range(10))):\n",
    "    print(i, ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c640cf2c-f407-4ead-aaf8-1ade48438d54",
   "metadata": {},
   "outputs": [],
   "source": [
    "# same optimization as last time\n",
    "def train_no_tune(config, checkpoint_dir=None):\n",
    "    \n",
    "    n_embd = config['n_embd']\n",
    "    n_hidden = config['n_hidden']\n",
    "    last_layer_factor = config['last_layer_factor']\n",
    "    max_steps = config['max_steps'] \n",
    "    lr = config['lr']\n",
    "    batch_size = config['batch_size']\n",
    "    \n",
    "    model = build_model(n_embd, n_hidden, last_layer_factor)\n",
    "\n",
    "    train_loss = F.cross_entropy(model(Xtr), Ytr)\n",
    "    print('Initial loss ', train_loss)\n",
    "    \n",
    "    lossi = []\n",
    "    \n",
    "    for i in range(max_steps):\n",
    "        running_loss = 0.0\n",
    "        epoch_steps = 0\n",
    "        # minibatch construct\n",
    "        ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n",
    "        Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "\n",
    "\n",
    "        # forward pass\n",
    "        logits = model(Xb)\n",
    "        loss = F.cross_entropy(logits, Yb)\n",
    "\n",
    "        # backward pass\n",
    "        for p in model.parameters():\n",
    "            p.grad = None\n",
    "        loss.backward()\n",
    "\n",
    "        # update: simple SGD\n",
    "        # lr = 0.1 if i < 150_000 else 0.01 # step learning rate decay\n",
    "        for p in model.parameters(): \n",
    "            p.data += -lr * p.grad\n",
    "\n",
    "        # track stats\n",
    "        if i % 10_000 == 0:\n",
    "            print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
    "        lossi.append(loss.log10().item())\n",
    "    \n",
    "#     print(model)\n",
    "        \n",
    "    return model\n",
    "                \n",
    "#     print(train_loss.item(), val_loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daceac7a-c216-4680-b45a-3c7cbb58caba",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "        \"n_embd\": 24,\n",
    "        \"n_hidden\": 128,\n",
    "        \"lr\": 0.1,\n",
    "        \"last_layer_factor\": 0.1,\n",
    "        \"batch_size\": 32,\n",
    "        \"max_steps\": 150_000\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4828e4ad-47e6-4770-8460-ec9e358e1809",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = train_no_tune(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43c67352-0081-47f5-94fa-aaabd02aca8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 7.3465,  3.0281, -0.8438, -0.6315,  0.1254,  2.3173, -5.4998, -3.1930,\n",
       "          1.5278,  1.3920, -0.8801,  0.2029,  0.6476,  1.1801,  3.3868, -0.4111,\n",
       "         -3.7620, -2.9908,  2.5343,  2.5573, -1.7074,  1.2317, -2.6614, -0.9130,\n",
       "         -1.6869, -1.5149, -0.9145],\n",
       "        [ 1.1573,  6.1267, -1.6542, -0.7126, -0.0082,  7.1435, -3.1067, -2.7086,\n",
       "         -0.5152,  5.9013, -1.5665, -2.7343, -0.8459, -2.8573,  0.9811,  5.7301,\n",
       "         -2.6570, -4.4699,  2.2010, -1.5982, -1.9610,  2.9759,  0.5002, -2.5367,\n",
       "         -3.4369,  5.2291, -4.5067],\n",
       "        [ 7.1585,  2.8464,  0.2289, -0.9455,  1.5724,  0.8947, -1.6819,  8.0458,\n",
       "         -3.1600, -1.8369, -0.4268,  0.1184, -0.3447, -0.5721,  2.6477,  0.2432,\n",
       "         -2.7675, -2.0082,  1.5645,  1.6701, -0.3294, -2.4148, -3.6991, -3.9939,\n",
       "         -2.7473,  0.7778, -1.0990],\n",
       "        [-0.6448,  5.7704, -0.8307,  0.5714, -1.6713,  5.6905, -2.8324, -4.6171,\n",
       "          1.4003,  7.0470, -1.9388, -1.7307, -1.1796, -0.9947, -1.5239,  2.1725,\n",
       "         -1.7857, -2.5253,  1.9881,  1.3517, -0.3367, -1.3917, -1.4497, -2.4642,\n",
       "         -1.0517,  4.1390, -1.2129]], device='cuda:0', grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ix = torch.randint(0, Xtr.shape[0], (4,))\n",
    "Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "logits = m(Xb)\n",
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b263806-b786-49ba-8253-1d688b06f08b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.858979344367981, 2.039809226989746)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for layer in m.layers: layer.training = False\n",
    "train_loss = F.cross_entropy(m(Xtr), Ytr).item() \n",
    "val_loss = F.cross_entropy(m(Xdev), Ydev).item()\n",
    "train_loss, val_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ccce81f-912f-4649-8a3b-771598f5d373",
   "metadata": {},
   "source": [
    "### Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a613f489-8b11-422f-970f-fe6212fc7e41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0/ 300000: 3.2818\n",
      "  10000/ 300000: 2.3348\n",
      "  20000/ 300000: 2.0034\n",
      "  30000/ 300000: 1.9822\n",
      "  40000/ 300000: 1.5388\n",
      "  50000/ 300000: 1.9057\n",
      "  60000/ 300000: 1.6040\n",
      "  70000/ 300000: 2.0101\n",
      "  80000/ 300000: 2.2422\n",
      "  90000/ 300000: 1.6573\n",
      " 100000/ 300000: 1.7261\n",
      " 110000/ 300000: 1.8587\n",
      " 120000/ 300000: 1.5202\n",
      " 130000/ 300000: 2.0675\n",
      " 140000/ 300000: 1.9777\n",
      " 150000/ 300000: 2.0351\n",
      " 160000/ 300000: 1.6458\n",
      " 170000/ 300000: 1.8868\n",
      " 180000/ 300000: 1.7516\n",
      " 190000/ 300000: 1.6009\n",
      " 200000/ 300000: 1.3999\n",
      " 210000/ 300000: 1.7495\n",
      " 220000/ 300000: 1.6129\n",
      " 230000/ 300000: 1.7232\n",
      " 240000/ 300000: 2.2450\n",
      " 250000/ 300000: 2.2155\n",
      " 260000/ 300000: 1.5074\n",
      " 270000/ 300000: 1.6431\n",
      " 280000/ 300000: 1.7525\n",
      " 290000/ 300000: 1.8047\n"
     ]
    }
   ],
   "source": [
    "def train(config):\n",
    "    \n",
    "    n_embd = config['n_embd']\n",
    "    n_hidden = config['n_hidden']\n",
    "    last_layer_factor = config['last_layer_factor']\n",
    "    max_steps = config['max_steps'] \n",
    "    lr = config['lr']\n",
    "    batch_size = config['batch_size']\n",
    "    \n",
    "    model = build_model(n_embd, n_hidden, last_layer_factor)\n",
    "\n",
    "    train_loss = F.cross_entropy(model(Xtr), Ytr)\n",
    "    print('Initial loss ', train_loss)\n",
    "    \n",
    "    lossi = []\n",
    "    \n",
    "    # max_steps = 100_00_000\n",
    "    \n",
    "    for i in range(max_steps):\n",
    "        running_loss = 0.0\n",
    "        epoch_steps = 0\n",
    "        # minibatch construct\n",
    "        ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n",
    "        Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "\n",
    "\n",
    "        # forward pass\n",
    "        logits = model(Xb)\n",
    "        loss = F.cross_entropy(logits, Yb)\n",
    "\n",
    "        # backward pass\n",
    "        for p in model.parameters():\n",
    "            p.grad = None\n",
    "        loss.backward()\n",
    "\n",
    "        # update: simple SGD\n",
    "        for p in model.parameters(): \n",
    "            p.data += -lr * p.grad\n",
    "\n",
    "        # track stats\n",
    "        # if i % 10_000 == 0: # print every once in a while\n",
    "        #     # print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
    "            \n",
    "        for layer in model.layers: layer.training = False\n",
    "        train_loss = F.cross_entropy(model(Xtr), Ytr) \n",
    "        val_loss = F.cross_entropy(model(Xdev), Ydev)\n",
    "        session.report({'training_iteration':i, 'train_loss':train_loss.item(), 'val_loss':val_loss.item()})\n",
    "        for layer in model.layers: layer.training = True\n",
    "            \n",
    "        lossi.append(loss.log10().item())\n",
    "    \n",
    "    print('Train Loss ', F.cross_entropy(model(Xtr), Ytr).item())\n",
    "    print('Val Loss ', F.cross_entropy(model(Xdev), Ydev).item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07672054-1013-4e59-9cc5-457f62cf1ad8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tuning():\n",
    "    search_space = {\n",
    "        \"n_embd\": tune.randint(4, 65),\n",
    "        \"n_hidden\": tune.randint(100, 500),\n",
    "        \"lr\": tune.loguniform(1e-4, 1e-1),\n",
    "        \"last_layer_factor\": tune.uniform(0.1, 1.01),\n",
    "        \"batch_size\": tune.choice([16, 32, 64]),\n",
    "        \"max_steps\": tune.randint(100_000, 10_00_000)\n",
    "    }\n",
    "    \n",
    "    initial_params = [{\n",
    "        \"n_embd\": 24, \n",
    "        \"n_hidden\": 128,\n",
    "        \"lr\": 0.1,\n",
    "        \"last_layer_factor\": 0.1,\n",
    "        \"batch_size\": 32,\n",
    "        \"max_steps\": 150_000\n",
    "    }]\n",
    "    \n",
    "    searcher = OptunaSearch(points_to_evaluate=initial_params)\n",
    "    trainable_with_cpu_gpu = tune.with_resources(train, {\"cpu\": 30, \"gpu\": 1})\n",
    "    \n",
    "    tuner = tune.Tuner(\n",
    "        trainable_with_cpu_gpu,\n",
    "        tune_config=tune.TuneConfig(\n",
    "            metric=\"val_loss\",\n",
    "            mode=\"min\",\n",
    "            search_alg=searcher,\n",
    "            num_samples=7,\n",
    "        ),\n",
    "        param_space=search_space,\n",
    "    )\n",
    "    \n",
    "    results = tuner.fit()\n",
    "    \n",
    "    print(\"Best hyperparameters found were: \", results.get_best_result().config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57266dd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-19 17:02:02,578\tINFO worker.py:1553 -- Started a local Ray instance.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"margin-left: 50px;display: flex;flex-direction: row;align-items: center\">\n",
       "        <h3 style=\"color: var(--jp-ui-font-color0)\">Ray</h3>\n",
       "        <svg version=\"1.1\" id=\"ray\" width=\"3em\" viewBox=\"0 0 144.5 144.6\" style=\"margin-left: 3em;margin-right: 3em\">\n",
       "            <g id=\"layer-1\">\n",
       "                <path fill=\"#00a2e9\" class=\"st0\" d=\"M97.3,77.2c-3.8-1.1-6.2,0.9-8.3,5.1c-3.5,6.8-9.9,9.9-17.4,9.6S58,88.1,54.8,81.2c-1.4-3-3-4-6.3-4.1\n",
       "                    c-5.6-0.1-9.9,0.1-13.1,6.4c-3.8,7.6-13.6,10.2-21.8,7.6C5.2,88.4-0.4,80.5,0,71.7c0.1-8.4,5.7-15.8,13.8-18.2\n",
       "                    c8.4-2.6,17.5,0.7,22.3,8c1.3,1.9,1.3,5.2,3.6,5.6c3.9,0.6,8,0.2,12,0.2c1.8,0,1.9-1.6,2.4-2.8c3.5-7.8,9.7-11.8,18-11.9\n",
       "                    c8.2-0.1,14.4,3.9,17.8,11.4c1.3,2.8,2.9,3.6,5.7,3.3c1-0.1,2,0.1,3,0c2.8-0.5,6.4,1.7,8.1-2.7s-2.3-5.5-4.1-7.5\n",
       "                    c-5.1-5.7-10.9-10.8-16.1-16.3C84,38,81.9,37.1,78,38.3C66.7,42,56.2,35.7,53,24.1C50.3,14,57.3,2.8,67.7,0.5\n",
       "                    C78.4-2,89,4.7,91.5,15.3c0.1,0.3,0.1,0.5,0.2,0.8c0.7,3.4,0.7,6.9-0.8,9.8c-1.7,3.2-0.8,5,1.5,7.2c6.7,6.5,13.3,13,19.8,19.7\n",
       "                    c1.8,1.8,3,2.1,5.5,1.2c9.1-3.4,17.9-0.6,23.4,7c4.8,6.9,4.6,16.1-0.4,22.9c-5.4,7.2-14.2,9.9-23.1,6.5c-2.3-0.9-3.5-0.6-5.1,1.1\n",
       "                    c-6.7,6.9-13.6,13.7-20.5,20.4c-1.8,1.8-2.5,3.2-1.4,5.9c3.5,8.7,0.3,18.6-7.7,23.6c-7.9,5-18.2,3.8-24.8-2.9\n",
       "                    c-6.4-6.4-7.4-16.2-2.5-24.3c4.9-7.8,14.5-11,23.1-7.8c3,1.1,4.7,0.5,6.9-1.7C91.7,98.4,98,92.3,104.2,86c1.6-1.6,4.1-2.7,2.6-6.2\n",
       "                    c-1.4-3.3-3.8-2.5-6.2-2.6C99.8,77.2,98.9,77.2,97.3,77.2z M72.1,29.7c5.5,0.1,9.9-4.3,10-9.8c0-0.1,0-0.2,0-0.3\n",
       "                    C81.8,14,77,9.8,71.5,10.2c-5,0.3-9,4.2-9.3,9.2c-0.2,5.5,4,10.1,9.5,10.3C71.8,29.7,72,29.7,72.1,29.7z M72.3,62.3\n",
       "                    c-5.4-0.1-9.9,4.2-10.1,9.7c0,0.2,0,0.3,0,0.5c0.2,5.4,4.5,9.7,9.9,10c5.1,0.1,9.9-4.7,10.1-9.8c0.2-5.5-4-10-9.5-10.3\n",
       "                    C72.6,62.3,72.4,62.3,72.3,62.3z M115,72.5c0.1,5.4,4.5,9.7,9.8,9.9c5.6-0.2,10-4.8,10-10.4c-0.2-5.4-4.6-9.7-10-9.7\n",
       "                    c-5.3-0.1-9.8,4.2-9.9,9.5C115,72.1,115,72.3,115,72.5z M19.5,62.3c-5.4,0.1-9.8,4.4-10,9.8c-0.1,5.1,5.2,10.4,10.2,10.3\n",
       "                    c5.6-0.2,10-4.9,9.8-10.5c-0.1-5.4-4.5-9.7-9.9-9.6C19.6,62.3,19.5,62.3,19.5,62.3z M71.8,134.6c5.9,0.2,10.3-3.9,10.4-9.6\n",
       "                    c0.5-5.5-3.6-10.4-9.1-10.8c-5.5-0.5-10.4,3.6-10.8,9.1c0,0.5,0,0.9,0,1.4c-0.2,5.3,4,9.8,9.3,10\n",
       "                    C71.6,134.6,71.7,134.6,71.8,134.6z\"/>\n",
       "            </g>\n",
       "        </svg>\n",
       "        <table>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left\"><b>Python version:</b></td>\n",
       "                <td style=\"text-align: left\"><b>3.8.10</b></td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left\"><b>Ray version:</b></td>\n",
       "                <td style=\"text-align: left\"><b> 2.3.0</b></td>\n",
       "            </tr>\n",
       "            \n",
       "        </table>\n",
       "    </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "RayContext(dashboard_url='', python_version='3.8.10', ray_version='2.3.0', ray_commit='cf7a56b4b0b648c324722df7c99c168e92ff0b45', address_info={'node_ip_address': '10.19.31.32', 'raylet_ip_address': '10.19.31.32', 'redis_address': None, 'object_store_address': '/tmp/ray/session_2023-03-19_17-02-00_649680_77984/sockets/plasma_store', 'raylet_socket_name': '/tmp/ray/session_2023-03-19_17-02-00_649680_77984/sockets/raylet', 'webui_url': '', 'session_dir': '/tmp/ray/session_2023-03-19_17-02-00_649680_77984', 'metrics_export_port': 57401, 'gcs_address': '10.19.31.32:56740', 'address': '10.19.31.32:56740', 'dashboard_agent_listen_port': 52365, 'node_id': 'acf7b7da5785a42b536fd0c9f331d54fae6c729f27535f0025a46cfe'})"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ray.shutdown(); \n",
    "ray.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e047570c-4130-46cb-bd5f-828e43117796",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/ray/tune/search/optuna/optuna_search.py:683: FutureWarning: IntUniformDistribution has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use :class:`~optuna.distributions.IntDistribution` instead.\n",
      "  return ot.distributions.IntUniformDistribution(\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/ray/tune/search/optuna/optuna_search.py:662: FutureWarning: LogUniformDistribution has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use :class:`~optuna.distributions.FloatDistribution` instead.\n",
      "  return ot.distributions.LogUniformDistribution(\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/ray/tune/search/optuna/optuna_search.py:671: FutureWarning: UniformDistribution has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use :class:`~optuna.distributions.FloatDistribution` instead.\n",
      "  return ot.distributions.UniformDistribution(\n",
      "\u001b[32m[I 2023-03-19 17:02:06,605]\u001b[0m A new study created in memory with name: optuna\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2023-03-19 23:14:45</td></tr>\n",
       "<tr><td>Running for: </td><td>06:12:38.49        </td></tr>\n",
       "<tr><td>Memory:      </td><td>9.4/196.6 GiB      </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using FIFO scheduling algorithm.<br>Resources requested: 30.0/30 CPUs, 1.0/1 GPUs, 0.0/125.77 GiB heap, 0.0/57.89 GiB objects (0.0/1.0 accelerator_type:A100)\n",
       "    </div>\n",
       "    \n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name    </th><th>status    </th><th>loc              </th><th style=\"text-align: right;\">  batch_size</th><th style=\"text-align: right;\">  last_layer_factor</th><th style=\"text-align: right;\">         lr</th><th style=\"text-align: right;\">  max_steps</th><th style=\"text-align: right;\">  n_embd</th><th style=\"text-align: right;\">  n_hidden</th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">  train_loss</th><th style=\"text-align: right;\">  val_loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_f6250653</td><td>RUNNING   </td><td>10.19.31.32:88428</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">           0.99565 </td><td style=\"text-align: right;\">0.000439031</td><td style=\"text-align: right;\">     540060</td><td style=\"text-align: right;\">      24</td><td style=\"text-align: right;\">       456</td><td style=\"text-align: right;\">302615</td><td style=\"text-align: right;\">        19429.5 </td><td style=\"text-align: right;\">     1.85993</td><td style=\"text-align: right;\">   2.00036</td></tr>\n",
       "<tr><td>train_abc822a6</td><td>PENDING   </td><td>                 </td><td style=\"text-align: right;\">          16</td><td style=\"text-align: right;\">           0.127438</td><td style=\"text-align: right;\">0.000668113</td><td style=\"text-align: right;\">     762048</td><td style=\"text-align: right;\">      30</td><td style=\"text-align: right;\">       168</td><td style=\"text-align: right;\">      </td><td style=\"text-align: right;\">                </td><td style=\"text-align: right;\">            </td><td style=\"text-align: right;\">          </td></tr>\n",
       "<tr><td>train_e8e76444</td><td>TERMINATED</td><td>10.19.31.32:88428</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">           0.1     </td><td style=\"text-align: right;\">0.1        </td><td style=\"text-align: right;\">     150000</td><td style=\"text-align: right;\">      24</td><td style=\"text-align: right;\">       128</td><td style=\"text-align: right;\">149999</td><td style=\"text-align: right;\">         2925.94</td><td style=\"text-align: right;\">     1.88007</td><td style=\"text-align: right;\">   2.06663</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "[<matplotlib.lines.Line2D>]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/optuna/distributions.py:799: FutureWarning: IntUniformDistribution(high=64, low=4, step=1) is deprecated and internally converted to IntDistribution(high=64, log=False, low=4, step=1). See https://github.com/optuna/optuna/issues/2941.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/optuna/distributions.py:799: FutureWarning: IntUniformDistribution(high=499, low=100, step=1) is deprecated and internally converted to IntDistribution(high=499, log=False, low=100, step=1). See https://github.com/optuna/optuna/issues/2941.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/optuna/distributions.py:799: FutureWarning: LogUniformDistribution(high=0.1, low=0.0001) is deprecated and internally converted to FloatDistribution(high=0.1, log=True, low=0.0001, step=None). See https://github.com/optuna/optuna/issues/2941.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/optuna/distributions.py:799: FutureWarning: UniformDistribution(high=1.01, low=0.1) is deprecated and internally converted to FloatDistribution(high=1.01, log=False, low=0.1, step=None). See https://github.com/optuna/optuna/issues/2941.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/optuna/distributions.py:799: FutureWarning: IntUniformDistribution(high=999999, low=100000, step=1) is deprecated and internally converted to IntDistribution(high=999999, log=False, low=100000, step=1). See https://github.com/optuna/optuna/issues/2941.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "2023-03-19 17:02:06,901\tWARNING worker.py:1866 -- Warning: The actor ImplicitFunc is very large (14 MiB). Check that its definition is not implicitly capturing a large array or other object in scope. Tip: use ray.put() to put large objects in the Ray object store.\n",
      "\u001b[2m\u001b[36m(pid=88428)\u001b[0m /home/ubuntu/.local/lib/python3.8/site-packages/pandas/core/computation/expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).\n",
      "\u001b[2m\u001b[36m(pid=88428)\u001b[0m   from pandas.core.computation.check import NUMEXPR_INSTALLED\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(train pid=88428)\u001b[0m No of parameters  76579\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name    </th><th>date               </th><th>done  </th><th>episodes_total  </th><th>experiment_id                   </th><th>hostname      </th><th style=\"text-align: right;\">  iterations_since_restore</th><th>node_ip    </th><th style=\"text-align: right;\">  pid</th><th style=\"text-align: right;\">  time_since_restore</th><th style=\"text-align: right;\">  time_this_iter_s</th><th style=\"text-align: right;\">  time_total_s</th><th style=\"text-align: right;\">  timestamp</th><th style=\"text-align: right;\">  timesteps_since_restore</th><th>timesteps_total  </th><th style=\"text-align: right;\">  train_loss</th><th style=\"text-align: right;\">  training_iteration</th><th>trial_id  </th><th style=\"text-align: right;\">  val_loss</th><th style=\"text-align: right;\">  warmup_time</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>train_e8e76444</td><td>2023-03-19_17-50-55</td><td>True  </td><td>                </td><td>352e72ef24b0485790725bb268154829</td><td>129-146-49-170</td><td style=\"text-align: right;\">                    150000</td><td>10.19.31.32</td><td style=\"text-align: right;\">88428</td><td style=\"text-align: right;\">             2925.94</td><td style=\"text-align: right;\">         0.0204446</td><td style=\"text-align: right;\">       2925.94</td><td style=\"text-align: right;\"> 1679248255</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">     1.88007</td><td style=\"text-align: right;\">              149999</td><td>e8e76444  </td><td style=\"text-align: right;\">   2.06663</td><td style=\"text-align: right;\">   0.00304294</td></tr>\n",
       "<tr><td>train_f6250653</td><td>2023-03-19_23-14-47</td><td>False </td><td>                </td><td>352e72ef24b0485790725bb268154829</td><td>129-146-49-170</td><td style=\"text-align: right;\">                    302648</td><td>10.19.31.32</td><td style=\"text-align: right;\">88428</td><td style=\"text-align: right;\">            19431.6 </td><td style=\"text-align: right;\">         0.0662513</td><td style=\"text-align: right;\">      19431.6 </td><td style=\"text-align: right;\"> 1679267687</td><td style=\"text-align: right;\">                        0</td><td>                 </td><td style=\"text-align: right;\">     1.85784</td><td style=\"text-align: right;\">              302647</td><td>f6250653  </td><td style=\"text-align: right;\">   1.99851</td><td style=\"text-align: right;\">   0.00304294</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[2m\u001b[36m(train pid=88428)\u001b[0m Initial loss  tensor(3.2965, device='cuda:0', grad_fn=<NllLossBackward0>)\n",
      "\u001b[2m\u001b[36m(train pid=88428)\u001b[0m Train Loss  1.873051643371582\n",
      "\u001b[2m\u001b[36m(train pid=88428)\u001b[0m Val Loss  2.0602710247039795\n",
      "\u001b[2m\u001b[36m(train pid=88428)\u001b[0m No of parameters  869355\n",
      "\u001b[2m\u001b[36m(train pid=88428)\u001b[0m Initial loss  \n",
      "\u001b[2m\u001b[36m(train pid=88428)\u001b[0m tensor(3.4301, device='cuda:0', grad_fn=<NllLossBackward0>)\n"
     ]
    }
   ],
   "source": [
    "tuning()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c16900d",
   "metadata": {},
   "source": [
    "### Analyze the Ray Tune trials"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd7fbd3f",
   "metadata": {},
   "source": [
    "All the trials' information did not show up above therefore doing the analysis in the folowing cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc52a0b4-b6d2-43a8-829d-2698cbf4b020",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading results from ray_results/train_2023-03-19_17-02-06...\n"
     ]
    }
   ],
   "source": [
    "experiment_path = f\"ray_results/train_2023-03-19_17-02-06\"\n",
    "print(f\"Loading results from {experiment_path}...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dafd551-45d8-414d-aaee-0f8d6adb7516",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/ray/tune/tuner.py:230: UserWarning: Passing in the experiment's `trainable` will be a required argument to `Tuner.restore` starting from version 2.5. Please specify the trainable to avoid this warning.\n",
      "  warnings.warn(warning_message)\n",
      "2023-03-21 14:13:11,149\tINFO experiment_analysis.py:789 -- No `self.trials`. Drawing logdirs from checkpoint file. This may result in some information that is out of sync, as checkpointing is periodic.\n"
     ]
    }
   ],
   "source": [
    "restored_tuner = tune.Tuner.restore(experiment_path)\n",
    "result_grid = restored_tuner.get_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7595d042-2889-4db8-9ffc-726ef4bfeb5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>training_iteration</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>870783</td>\n",
       "      <td>2.044881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>423444</td>\n",
       "      <td>2.018260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>762047</td>\n",
       "      <td>2.015049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>149999</td>\n",
       "      <td>2.066634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>199144</td>\n",
       "      <td>2.051399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>289118</td>\n",
       "      <td>2.046853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>540059</td>\n",
       "      <td>1.989679</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   training_iteration  val_loss\n",
       "0              870783  2.044881\n",
       "1              423444  2.018260\n",
       "2              762047  2.015049\n",
       "3              149999  2.066634\n",
       "4              199144  2.051399\n",
       "5              289118  2.046853\n",
       "6              540059  1.989679"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_df = result_grid.get_dataframe()\n",
    "results_df[[\"training_iteration\", \"val_loss\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ceb9d1f-2e33-4cf8-b870-6c314b98beff",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_result: Result = result_grid.get_best_result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "589ab543-d204-4ad3-811e-3a862e225708",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_embd': 24,\n",
       " 'n_hidden': 456,\n",
       " 'lr': 0.0004390312584437158,\n",
       " 'last_layer_factor': 0.9956496873025195,\n",
       " 'batch_size': 64,\n",
       " 'max_steps': 540060}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_result.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56e8c26e-ed70-445f-8f8a-495ca00f554d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 1.7052044868469238\n",
      "val 2.2605273723602295\n"
     ]
    }
   ],
   "source": [
    "best_result.checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "876e8226-bf1f-4d1c-9360-2aab91273b27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'training_iteration': 540059,\n",
       " 'train_loss': 1.7585725784301758,\n",
       " 'val_loss': 1.9896790981292725,\n",
       " 'time_this_iter_s': 0.06922125816345215,\n",
       " 'done': True,\n",
       " 'timesteps_total': None,\n",
       " 'episodes_total': None,\n",
       " 'trial_id': 'f6250653',\n",
       " 'experiment_id': '352e72ef24b0485790725bb268154829',\n",
       " 'date': '2023-03-20_03-45-17',\n",
       " 'timestamp': 1679283917,\n",
       " 'time_total_s': 35661.531294584274,\n",
       " 'pid': 88428,\n",
       " 'hostname': '129-146-49-170',\n",
       " 'node_ip': '10.19.31.32',\n",
       " 'config': {'n_embd': 24,\n",
       "  'n_hidden': 456,\n",
       "  'lr': 0.0004390312584437158,\n",
       "  'last_layer_factor': 0.9956496873025195,\n",
       "  'batch_size': 64,\n",
       "  'max_steps': 540060},\n",
       " 'time_since_restore': 35661.531294584274,\n",
       " 'timesteps_since_restore': 0,\n",
       " 'iterations_since_restore': 540060,\n",
       " 'warmup_time': 0.003042936325073242,\n",
       " 'experiment_tag': '2_batch_size=64,last_layer_factor=0.9956,lr=0.0004,max_steps=540060,n_embd=24,n_hidden=456'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_result.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cc95b1f-4bfd-470e-973e-685400497960",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='training_iteration'>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAh2UlEQVR4nO3de3wU9b3/8dcnFxIw3IncIXhFBASNCEdrLV4r1dPzU4+tl1qPLd5a9fQm2taqbT1aTz2nrVd+2mp90Faqtlq1IlW0UisSkIuAIiJgEExAkYsCuXzOHzMJm2Wz2YTdbGbzfj4eeTA7+92ZzxfXd4bvzHzH3B0REYm+vGwXICIi6aFAFxHJEQp0EZEcoUAXEckRCnQRkRyhQBcRyREFLTUws2Lg70BR2P5Rd/9RM23PAh4Fjnb3imTb7devn5eVlbW6YBGRzmzBggWb3L000XstBjqwC5js7tvNrBCYa2Z/dfdXYxuZWXfgamBeKkWVlZVRUZE080VEJI6ZrW3uvRaHXDywPXxZGP4kuhvpx8BtwM62FCkiIvsmpTF0M8s3s0VAFTDb3efFvX8kMNTdn05/iSIikoqUAt3d69x9HDAEmGBmoxveM7M84A7g2y1tx8ymmlmFmVVUV1e3sWQREUnEWjuXi5ndAHzi7v8dvu4JvAM0DMsMAD4Ezkx2YrS8vNw1hi7S+dTU1FBZWcnOnRqdTaa4uJghQ4ZQWFjYZL2ZLXD38kSfSeUql1Kgxt23mFlX4GSCsXIA3P1joF9M+xeB77R0lYuIdE6VlZV0796dsrIyzCzb5XRI7s7mzZuprKxkxIgRKX8ulSGXgcAcM1sCzCcYQ3/KzG42szPbWK+IdFI7d+6kb9++CvMkzIy+ffu2+l8xLR6hu/sSYHyC9Tc00/6EVlUgIp2Owrxlbfk7itydorV19by0UidURUTiRS7Qf/nCKi769WvMfXtTtksRkU6gpKSk2ffWrFnD6NGjm32/vUUu0Ndu3gFA9XadIRcRiRW5QBcR2RfTpk3jrrvuanx944038pOf/IQTTzyRI488kjFjxvDEE0+0ers7d+7k4osvZsyYMYwfP545c+YAsGzZMiZMmMC4ceMYO3Ysb7/9Njt27GDKlCkcccQRjB49mkceeSQtfUtlLhcRkYy46S/LWP7+1rRuc9SgHvzojMObff/cc8/lmmuu4corrwRg5syZzJo1i6uuuooePXqwadMmJk6cyJlnntmqE5N33XUXZsbSpUt58803OeWUU1i5ciX33nsvV199Neeffz67d++mrq6OZ555hkGDBvH008HN9R9//PG+dToUuSP0hr9ePdtaRNpi/PjxVFVV8f7777N48WJ69+7NgAEDuP766xk7diwnnXQS69ev54MPPmjVdufOncsFF1wAwMiRIxk+fDgrV65k0qRJ3HLLLdx2222sXbuWrl27MmbMGGbPns21117Lyy+/TM+ePdPSNx2hi0jWJDuSzqRzzjmHRx99lI0bN3LuuecyY8YMqqurWbBgAYWFhZSVlaXtTtbzzjuPY445hqeffprTTz+d++67j8mTJ7Nw4UKeeeYZfvCDH3DiiSdyww0JrwRvFQW6iHQ65557Ll//+tfZtGkTL730EjNnzmT//fensLCQOXPmsHZtszPUNuszn/kMM2bMYPLkyaxcuZJ169Zx6KGHsnr1ag444ACuuuoq1q1bx5IlSxg5ciR9+vThggsuoFevXtx///1p6VfkAl03JIjIvjr88MPZtm0bgwcPZuDAgZx//vmcccYZjBkzhvLyckaOHNnqbV5xxRVcfvnljBkzhoKCAh588EGKioqYOXMmDz/8MIWFhY1DO/Pnz+e73/0ueXl5FBYWcs8996SlX62enCtd2jo5159fX881jyziscsncdTwPhmoTEQyacWKFRx22GHZLiMSEv1dJZucK3InRUuKgn9UFOZHrnQRkYyK4JBLtisQkc5m6dKlXHjhhU3WFRUVMW9eSk/cbDeRC/QGumxRRNrLmDFjWLRoUbbLaFHkxi0ajtCV5yLRla1zd1HSlr+j6AU6GnMRibLi4mI2b96sUE+i4QEXxcXFrfpchIdc9GUQiaIhQ4ZQWVmJniucXMMj6FojeoGuIReRSCssLGzVY9UkdREcchERkUQiF+gNNOIiItJU5AJ9z63/SnQRkVgtBrqZFZvZa2a22MyWmdlNCdp8y8yWm9kSM3vezIZnplwNuYiINCeVI/RdwGR3PwIYB5xmZhPj2rwOlLv7WOBR4GdprTIBDbmIiDTVYqB7YHv4sjD88bg2c9z9k/Dlq0DrrrVpBd36LyKSWEpj6GaWb2aLgCpgtrsnm8DgEuCvaagtKR2gi4g0lVKgu3udu48jOPKeYGajE7UzswuAcuD2Zt6famYVZlbR1psKGu4U1ZCLiEhTrbrKxd23AHOA0+LfM7OTgO8DZ7r7rmY+P93dy929vLS0tA3lashFRKQ5qVzlUmpmvcLlrsDJwJtxbcYD9xGEeVUG6tyLbv0XEWkqlVv/BwIPmVk+wS+Ame7+lJndDFS4+5MEQywlwB/D68TXufuZmShYV6GLiCTWYqC7+xJgfIL1N8Qsn5TmupqnIRcRkYQid6doA424iIg0FblAX7Z+KwCLK7dktxARkQ4mcoE+alAPAA4b2CPLlYiIdCyRC/TC/KDkfF2/KCLSROQCfc8zRTWILiISK3KBntcQ6MpzEZEmIhfoDdct1ivRRUSaiFyg5+mZoiIiCUUu0BueWKRb/0VEmopeoId/Ks9FRJqKXKDnWcMYepYLERHpYCIX6I2XLeoQXUSkiegGenbLEBHpcKIX6OikqIhIIpEL9LywYuW5iEhTkQt0QydFRUQSiV6gay4XEZGEIhfoDXeK6ghdRKSpyAU6OikqIpJQ5AI9T9Ogi4gk1GKgm1mxmb1mZovNbJmZ3ZSgTZGZPWJmq8xsnpmVZaRa9szlotkWRUSaSuUIfRcw2d2PAMYBp5nZxLg2lwAfuftBwP8At6W1yhiaD11EJLEWA90D28OXheFPfJz+K/BQuPwocKJZZp4Rp8sWRUQSS2kM3czyzWwRUAXMdvd5cU0GA+8BuHst8DHQN411xtQS/KmToiIiTaUU6O5e5+7jgCHABDMb3ZadmdlUM6sws4rq6uq2bEJzuYiINKNVV7m4+xZgDnBa3FvrgaEAZlYA9AQ2J/j8dHcvd/fy0tLSNhWsB1yIiCSWylUupWbWK1zuCpwMvBnX7EngonD5bOAFz1Di6qSoiEhiBSm0GQg8ZGb5BL8AZrr7U2Z2M1Dh7k8CDwAPm9kq4EPgS5kqWCdFRUQSazHQ3X0JMD7B+htilncC56S3tMTyNJeLiEhCkbtTFM3lIiKSUOQCvWHIRYPoIiJNRS7Q83TZoohIQpEL9E921wFwwxPLslyJiEjHErlA3/JJTbZLEBHpkCIX6N2LU7nSUkSk84lcoHfrkp/tEkREOqTIBXqxAl1EJKHIBXrXQgW6iEgikQv0Aj2DTkQkocgFeoaemyEiEnmRC3QREUlMgS4ikiMU6CIiOSKygT5+WK9slyAi0qFEMtD3717EyAHds12GiEiHEslAL8zPo6ZO8y2KiMSKZKDn5xl1esKFiEgTkQz0gnyjpq4+22WIiHQokZy6cHX1Dj7asTvbZYiIdCgtHqGb2VAzm2Nmy81smZldnaBNTzP7i5ktDttcnJly9/hI86KLiDSRyhF6LfBtd19oZt2BBWY2292Xx7S5Elju7meYWSnwlpnNcPeMHEb371FE725dMrFpEZHIajHQ3X0DsCFc3mZmK4DBQGygO9DdgolWSoAPCX4RZMSwPt0oyIvk8L+ISMa0KhXNrAwYD8yLe+tO4DDgfWApcLW773XW0symmlmFmVVUV1e3rWKgIC+P2nqdFBURiZVyoJtZCfAYcI27b417+1RgETAIGAfcaWY94rfh7tPdvdzdy0tLS9tc9D9Xb2b+mo/a/HkRkVyUUqCbWSFBmM9w98cTNLkYeNwDq4B3gZHpK1NERFqSylUuBjwArHD3O5pptg44MWzfHzgUWJ2uIkVEpGWpXOVyLHAhsNTMFoXrrgeGAbj7vcCPgQfNbClgwLXuvin95QamjBnIy2+3fQxeRCQXpXKVy1yCkE7W5n3glHQV1ZKnl25or12JiESGrv0TEckRkQz0S44bke0SREQ6nEgGekFeMAJUrxkXRUQaRTLQ7/t7cAHNph27slyJiEjHEclA/+bkgwDIt6TnakVEOpVIBvrDr64FYHHlluwWIiLSgUQy0LeEU+f+fWXGLnUXEYmcSAZ6g+rtGkMXEWkQyUDPD69yeXqJbjASEWkQyUDXA6JFRPYWyUA/t3woACVFkXwkqohIRkQy0McN6wXA9l0ZeyiSiEjkRDLQh/bulu0SREQ6nEgG+uDeXQHYr0t+lisREek4Ihnog3oVA7Bjd12WKxER6TgiGegFeZEsW0QkoyKZjHmawkVEZC+RDHTTpFwiInuJZKCLiMjeWgx0MxtqZnPMbLmZLTOzq5tpd4KZLQrbvJT+UkVEJJlUbrWsBb7t7gvNrDuwwMxmu/vyhgZm1gu4GzjN3deZ2f6ZKVdERJrT4hG6u29w94Xh8jZgBTA4rtl5wOPuvi5sV5XuQpPU1167EhHp0Fo1hm5mZcB4YF7cW4cAvc3sRTNbYGZfSVN9LaqpU6CLiEArAt3MSoDHgGvcfWvc2wXAUcAU4FTgh2Z2SIJtTDWzCjOrqK6u3oey91ixIb4UEZHOKaVAN7NCgjCf4e6PJ2hSCcxy9x3uvgn4O3BEfCN3n+7u5e5eXlpaui91N3rolTVp2Y6ISNSlcpWLAQ8AK9z9jmaaPQEcZ2YFZtYNOIZgrD3jhvXVRF0iIpDaEfqxwIXA5PCyxEVmdrqZXWZmlwG4+wrgWWAJ8Bpwv7u/kbGqgbIwyP/5zuZM7kZEJDJavGzR3ecCLd6a6e63A7eno6hUdOsSlD7v3Q/ba5ciIh1aZO8UvfjYsmyXICLSoUQ20I89qF+2SxAR6VAiG+h9S7pkuwQRkQ4lsoFeVKCnFYmIxIpsoIuISFMKdBGRHJETgV5bV5/tEkREsi4nAv2fq3VzkYhITgT6hQ+8lu0SRESyLicCXUREIh7oXzp6aLZLEBHpMCId6BdOGp7tEkREOoxIB/rhg3pmuwQRkQ4j0oEea/uu2myXICKSVTkT6Eve25LtEkREsipnAv2i3+jSRRHp3HIm0GvqPNsliIhkVeQD/Wdnjc12CSIiHULkA/3U0QOyXYKISIcQ+UDv2bWwcdldwy4i0nm1GOhmNtTM5pjZcjNbZmZXJ2l7tJnVmtnZ6S0zNX+sqMzGbkVEOoRUjtBrgW+7+yhgInClmY2Kb2Rm+cBtwHPpLTF133tsSbZ2LSKSdS0GurtvcPeF4fI2YAUwOEHTbwKPAVVprVBERFLSqjF0MysDxgPz4tYPBv4NuCdtlbXCby4+Ohu7FRHpUFIOdDMrITgCv8bdt8a9/b/Ate6e9NFBZjbVzCrMrKK6urrVxTbnsweXNi6vrt6etu2KiERJSoFuZoUEYT7D3R9P0KQc+IOZrQHOBu42sy/GN3L36e5e7u7lpaWl8W+3WV6eNS5P/vlLaduuiEiUFLTUwMwMeABY4e53JGrj7iNi2j8IPOXuf05TjSIikoJUjtCPBS4EJpvZovDndDO7zMwuy3B9KdPDLkSks2vxCN3d5wLWUruY9l/dl4La6tazxvKH+e8B8Mb6jxk9WHOli0jnEvk7RRP5wq/mZrsEEZF2l5OBLiLSGeVUoM///kmNy5rXRUQ6m5wK9NLuRY3Llz68IIuViIi0v5wK9FjPLf8g2yWIiLSrnAv0759+WOPy7tqkN66KiOSUnAv0rx9/QOPyIT/4axYrERFpXzkX6CIinVVOBvr0C49qXF647qMsViIi0n5yMtBPOXzPc0b/392vZLESEZH2k5OBHq9q685slyAiknE5G+jv/tfpjcsTbnk+i5WIiLSPnA30YNbfPT7+tCZLlYiItI+cDXSAd27Zc5R+xE1Ze3a1iEi7yOlAz89repR+7aNLslSJiEjm5XSgA6z66ecblx+peE+TdolIzsr5QC/Ib9rFEdc9k6VKREQyK+cDHWDNrVOavF6xYWuWKhERyZxOEegAv/vaMY3Ln//Fy2zdqateRCS3dJpA/5eD+jV5PfZGXfUiIrmlxUA3s6FmNsfMlpvZMjO7OkGb881siZktNbNXzOyIzJS7b+KHXo677YUsVSIikn6pHKHXAt9291HAROBKMxsV1+Zd4LPuPgb4MTA9vWWmT+wdpJUffcplerKRiOSIFgPd3Te4+8JweRuwAhgc1+YVd2+Y1vBVYEi6C00XM+MPUyc2vn522UbKpj2dxYpERNKjVWPoZlYGjAfmJWl2CdChnywx8YC+fPVfypqse+iVNbpGXUQiLeVAN7MS4DHgGndPeN2fmX2OINCvbeb9qWZWYWYV1dXVbak3bW4883COHNar8fWPnlyma9RFJNJSCnQzKyQI8xnu/ngzbcYC9wP/6u6bE7Vx9+nuXu7u5aWlpW2tOW0ev+JY+u7Xpcm6smlPs31XbZYqEhFpu1SucjHgAWCFu9/RTJthwOPAhe6+Mr0lZtaCH57M5w5t+stl9I9msaRyS3YKEhFpI2tp3NjMjgNeBpYC9eHq64FhAO5+r5ndD5wFrA3fr3X38mTbLS8v94qKin0oPb3e3bSDz/33i03WPXb5JI4a3ic7BYmIJGBmC5rL1xYDPVM6WqADuHvCcfR3/+v0veZXFxHJhmSB3mnuFE2FmTW5Tr3BiOue4ddz381CRSIiqVOgxzEz1tw6hSljBjZZf/NTyymb9jQn3/ESdfW6vFFEOh4FejPuOv/IJnOpN3i7ajsHXv8MWz7ZnYWqRESap0BPoiA/jzW3TuGOf997appxN8+mbNrTVG3dmYXKRET2ppOirXD3i6v42bNvJXzvgH778cJ3TmjfgkSk09FVLmn2tYfm87cVVQnf69Yln6U3nrrX80xFRNJBgZ4hD7+6lh/++Y1m3591zfEcOqB7O1YkIrlOgZ5h1dt2cfRP/5a0zW//YwLHH5L96Q5EJNoU6O2kuRuT4v3ua8cw6cC+ullJRFpNgZ4Fb6z/mC/8am6L7cr6duO5//wsXQp0wZGItEyBnmVPLXmfb/zu9ZTa3nP+kQzoWcz4Yb0zXJWIRJECvQN5btlGprbisXdTjz+Ac44awsH9dXJVRBToHdbOmjpG/vDZVn9uypiBTPv8SIb26ZaBqkSkI1OgR8SOXbUc/qNZbf78y9/7HKXdi9i+q5Z+JUVprExEOgoFekS5O9/54xIeW1jZ5m1MGTuQn59zBEUFebqqRiQHKNBzSG1dPQd9P33P4H74kgl85mBdHy8SFQr0TuClldVc9OvX0ra9fzmwLzecMYpD9u9OnqYxEOkwFOid1LadNVz68AJeeSfhM7v32aWfPYBLjz+Q3t0KNZwj0k4U6LKX2rp6LnhgHq+u/jBj+xg3tBeL3tsCwN++dTz9exSzX5cCHfGL7AMFurRa9bZd/Py5t/jD/PfabZ/FhXkU5OXx5yuPZXCvruyqraNXty7ttn+RKNinQDezocBvgf6AA9Pd/RdxbQz4BXA68AnwVXdfmGy7CvTocnfq6p1V1duZ/tJqHn99fbZLAuCL4wZx3MGl1NTVM3pQTwDGDOm5V7stn+xm+YatlA/voykXJHL2NdAHAgPdfaGZdQcWAF909+UxbU4HvkkQ6McAv3D3Y5JtV4Ge+xq+W+99+CkzK97jzjmrslxR8w7pX0KeGW9u3MbPzhrLjt21nDyqP/1KirjvpdWcddRghvTecyNXbV09Bfl5jX3UOQRpL2kdcjGzJ4A73X12zLr7gBfd/ffh67eAE9x9Q3PbUaBLrIbv4ZZPapj+8moeW1BJ1bZdWa4qM44/pJSCPGN3bT0H9y/hN/9YA0CewXEHl/LdUw6lT0kXamrrGdCzmJ01dfTsGpx4rq93lqz/mHFDezVub1dtHflmFOTnsez9jxnRbz+6dSloss8Ptu6kpKiA/YqarpfWcXfe+mAbIwf0yFoNaQt0MysD/g6MdvetMeufAm5197nh6+eBa9292cRWoEu61NbVs6u2HoAbn1xGbb2zuHILq6t3ZLkySaRfSRc2bU//Q9a/PGEYv39tXePr7556KLfPCh4Z+Y3PHcSzyzay/qNP+bSmjosmDefU0QOY+/Ym7n7xnYTbO2JIT74x+WDufnEVV00+mIf+uYYX36pOS63/mDaZwb26tumzaQl0MysBXgJ+6u6Px72XUqCb2VRgKsCwYcOOWrt2bWv7IpJ27s6HO3YzY946zj5qCJc+vICCfOP1dVuyXZrksDW3TmnT55IFekr//jKzQuAxYEZ8mIfWA0NjXg8J1zXh7tOB6RAcoaeyb5FMMzP6lhRx1YkHA/CXbx63z9t0d8ys8U8IJmMrKsjjiUXvU9ZvP7oXF7B5+26G9enGB1t3UlyYz3sffsLXfqt/uUrbpHJS1ICHgA/d/Zpm2kwBvsGek6K/dPcJybarIReR7NldW0+9O8WF+Xu9t31XLes2f8LIAd3ZXVdPYX4eNXX1dMnPa7yHoKauHnfoUpBHfb2Tl2fsrKmjausuirvkUVJUwKJ1W7h/7rucN2EYXQryuOkvyzhsYA+eX1HFr748npLiAnp1K8Qdenfrwo7dtXQvKqAgP4/a+nr+tryKSQf2ZfbyjWzavpvLP3sgVdt2MX/Nh6zZFAynDezVlVNG9eevb2zglmfeBOCo4b3Z+mkNb1dtb9XfySmj+vPc8g9abHf4oB6cf8xwrv/TUgBu+bcx9OhawP/MXokDq6t3cMYRg3hu2Uamf6WcwwZ2p99+Razf8ik7a+o4sLRkn+7F2NerXI4DXgaWAvXh6uuBYQDufm8Y+ncCpxFctnhxsvFzUKCLiLTFPg25hOPiSX+dePBb4cq2lSciIumguypERHKEAl1EJEco0EVEcoQCXUQkRyjQRURyhAJdRCRHKNBFRHJE1h5wYWbVQFsnc+kHbEpjOR1RrvdR/Yu+XO9jR+3fcHdP+GT3rAX6vjCziubulMoVud5H9S/6cr2PUeyfhlxERHKEAl1EJEdENdCnZ7uAdpDrfVT/oi/X+xi5/kVyDF1ERPYW1SN0ERGJE7lAN7PTzOwtM1tlZtOyXU88M/u1mVWZ2Rsx6/qY2Wwzezv8s3e43szsl2FflpjZkTGfuShs/7aZXRSz/igzWxp+5pfhXPTN7iMD/RtqZnPMbLmZLTOzq3Opj2ZWbGavmdnisH83hetHmNm8sKZHzKxLuL4ofL0qfL8sZlvXhevfMrNTY9Yn/A43t49MMbN8M3s9fIRkTvXRzNaE36FFZlYRrsuJ72hS7h6ZHyAfeAc4AOgCLAZGZbuuuBqPB44E3ohZ9zNgWrg8DbgtXD4d+CvBfPMTgXnh+j7A6vDP3uFy7/C918K2Fn7288n2kYH+DQSODJe7AyuBUbnSx3CfJeFyITAvrGUm8KVw/b3A5eHyFcC94fKXgEfC5VHh97MIGBF+b/OTfYeb20cGv6vfAn4HPJVs/1HsI7AG6Be3Lie+o0n73Z47S8N/pEnArJjX1wHXZbuuBHWW0TTQ3wIGhssDgbfC5fuAL8e3A74M3Bez/r5w3UDgzZj1je2a20c79PUJ4ORc7CPQDVhI8FjFTUBB/PcQmAVMCpcLwnYW/91saNfcdzj8TMJ9ZKhvQ4DngcnAU8n2H8U+kjjQc+47Gv8TtSGXwcB7Ma8rw3UdXX933xAubwT6h8vN9SfZ+soE65PtI2PCf3qPJziKzZk+hkMRi4AqYDbB0eYWd69NUFNjP8L3Pwb60vp+902yj0z4X+B77HmsZLL9R7GPDjxnZgvMbGq4Lme+o81p8RF0kl7u7maW0UuL2mMfZlYCPAZc4+5bwyHEdtt/Jvfh7nXAODPrBfwJGJmJ/WSLmX0BqHL3BWZ2QpbLyZTj3H29me0PzDazN2PfjPp3tDlRO0JfDwyNeT0kXNfRfWBmAwHCP6vC9c31J9n6IQnWJ9tH2plZIUGYz3D3x1vYfyT7CODuW4A5BEMDvcys4QAotqbGfoTv9wQ20/p+b06yj3Q7FjjTzNYAfyAYdvlFkv1Hro/uvj78s4rgl/IEcvA7Gi9qgT4fODg8U96F4ATNk1muKRVPAg1nyC8iGHduWP+V8Cz7RODj8J9rs4BTzKx3eJb8FIKxxg3AVjObGJ5V/0rcthLtI63C/T4ArHD3O3Ktj2ZWGh6ZY2ZdCc4PrCAI9rOb6V9DTWcDL3gwgPok8KXwCpERwMEEJ9ISfofDzzS3j7Ry9+vcfYi7l4X7f8Hdz8+VPprZfmbWvWGZ4Lv1BjnyHU2qPQfs0/FDcEZ6JcG45vezXU+C+n4PbABqCMbWLiEYO3weeBv4G9AnbGvAXWFflgLlMdv5D2BV+HNxzPpygi/nO8Cd7Lk5LOE+MtC/4wjGJ5cAi8Kf03Olj8BY4PWwf28AN4TrDyAIq1XAH4GicH1x+HpV+P4BMdv6ftiHtwivgkj2HW5uHxn+vp7AnqtccqKP4T4Whz/LGvafK9/RZD+6U1REJEdEbchFRESaoUAXEckRCnQRkRyhQBcRyREKdBGRHKFAFxHJEQp0aTdm1svMrmjD555puNknSZubzeykNheXeJuvhH+Wmdl5ad729Yn2JbIvdB26tJtwMq+n3H103PoC3zNhU4cTznfyHXf/Qis+k7RPZrbd3UvSUJ5IIx2hS3u6FTjQgocOzDezl83sSWA5gJn9OZwdb1nMDHkNDyvoFx4przCz/x+2eS68PR8ze9DMzo5pf5OZLbTgIQQjw/WlFjx0YJmZ3W9ma82sX3PFmtn2mLo/E9b9nxbMxnh72IclZnZp2P6EVPpkZrcCXcPtzYjdV3j7+e1m9kZY+7kx237RzB41szfNbEZ427nIHu15W6p+OvcPMfPEE9xyvgMYEfN+w63YXQluq+4bvl4D9As/XwuMC9fPBC4Ilx8Ezo5p/81w+Qrg/nD5TsL5u4HTCKYw6Jek3u0xtT4Vs34q8INwuQioIHjAQ2v6tL2ZfZ1FMGVvPsHUq+sI5tU+gWDa2iEEB2L/JJhRMOv/XfXTcX50hC7Z9Jq7vxvz+iozWwy8SjDL3cEJPvOuuy8KlxcQhHwijydocxzB7IK4+7PAR22s+xSCyZwWEcwF3zem1rb0KdZxwO/dvc7dPwBeAo6O2Xalu9cTzKFTlngT0llpPnTJph0NC+E49UkET8b5xMxeJJgUKt6umOU6giPfRHbFtEn399wI/gUwq8nKoA9t6VOq4vuu/3+lCR2hS3vaRvAc0kR6Ah+FwTeS4HmN6fYP4N8BzOwUgudEpiK+7lnA5RbMC4+ZHRJO0xovWZ9qGj4f52Xg3HCcvpTgGbWvpVindHL6DS/txt03m9k/zOwN4FPgg5i3nwUuM7MVBFOxvpqBEm4Cfm9mFxKMQW8kCOuWLAHqwqGTBwkeBlEGLAxPTFYDX0zwuWR9mg4sMbOFHsxF3uBPBA/UWEwwxv89d9/YcGJXJBldtiidhpkVAXXuXmtmk4B73H1clssSSRsdoUtnMgyYaWZ5wG7g61muRyStdIQunZqZNTxhJt6J7r65vesR2RcKdBGRHKGrXEREcoQCXUQkRyjQRURyhAJdRCRHKNBFRHLE/wGggyevvr8ppwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_result.metrics_dataframe.plot(\"training_iteration\", \"val_loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7132ef94-280d-451c-b582-dcebfb2b78bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Mean Test Accuracy')"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEXCAYAAACgUUN5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABGwklEQVR4nO3deXxU5dXA8d+ZTFYSSMJuAgYJsokiBQFXFBHRqlVxrRV9tVZr3dpadyvt26rV6mtrrXVpBetSRXCt4ILUFSggsmtQtkDYQiAkZJ057x/3Jpkkk2SSzGRhzvfzmU/ufe5z733uzM2cudt5RFUxxhgTvTzt3QBjjDHtywKBMcZEOQsExhgT5SwQGGNMlLNAYIwxUc4CgTHGRDkLBMY0QkR+KCLvhbtuZyQiRSJyWASWO1hElovIfhG5MdzLD7K+CSKSGzC+UURODcNyQ3p/RCRLRFREvK1dZ7hYIHC5O0O5iPSoU/6l+6FltVO7BoiIX0T+2h7r74xE5En3n7LI/UwrAsbfbc6yVPUFVT0t3HWbK/DLSkSuEJFPI7GegPUtEJGrA8tUNVlVv4vA6n4FfKSqKar6p3At1H2fVEQuauH87wbsNxXuvlQ1/mTd+hF8fyLOAkFtG4BLqkZEZASQ1H7NAeByoAC4SETi23LFIhLTlusLF1W91v2nTAZ+D/yralxVp1TV60i/yNpSB9zuQ4HVLZmxiW2ZBuzB+R9qNlWdErAfvQD8IWA/ujbENnQKFghqe57aO800YGZgBRGJF5GHRWSziOxwf30mutPSRORtEdklIgXucGbAvAtE5Lci8pl7GPxe3SOQOusStz13AxXAWXWmn+MeUheKyLcicrpbni4i/xCRbW47XnfL6/2SdH8xZbvDz4nIX0Xk3yJSDJwsIme6R0WFIrJFRO6rM//xIvK5iOx1p18hImPc9yYmoN55IvJVkG0cKyLb69Q9V0RWuMPHiMgSd/07ROSRht6vULi/rG9zl18sIl4Rud19//aLyBoROTegfq33zH2/rhWRHHeb/+J+Ts2tGyMifxSR3SKyQUR+JiGcLhCRocCTwHj3l+let7yx/XKCiOS6270d+Edj+6qI/A44AXjcXcfjAdtTta90E5GZ7vybRORuEfEEvg9uewrc7ZtSd1vcuvOBkwPWdXgIy/5MRB4VkXzgvgaWeyhwEnANMFlE+jT2vjaX+15cLyI5QE6Q96fR/5s6y7pCRL5z978NIvLDcLY1JKpqLyfNxkbgVOBrYCgQA+Ti/FpRIMut9yjwJpAOpABvAfe707oD5+McRaQArwKvB6xjAfAtcDiQ6I4/0EibTgDKgDTgz8BbAdOOAfYBk3ACegYwxJ32DvAvd75Y4CS3/Arg0zrrUCDbHX7OXeZx7jITgAnACHf8SGAH8AO3/qHAfpyjqFh3+0e609YAUwLWMwf4RQPb+S0wKWD8VeB2d/gL4EfucDIwrpmf633AP+t8zsuBfkCiW3YBcIi7jRcBxUDfYO+Z+369DaQC/YFdwOktqHut+x5lup/TB259b2P7ZyOfY2P75QSgEngQiMfZ90LZV69uZF+ZCbzhzpsFfANcFdC+CuDHOP9H1wHbAGlg22qtK4RlVwI3AN6qzzDIMu8BFrvDKwnY99z3IzfYe9vIfvQc8L913ov33fc7Mcj7M4GG/2+yqj5roAtQCAx2p/UFhrfld5+qWiCouzPg/Pq+Hzjd/aC97oeWBQjOl8TAgPnGAxsaWOZIoCBgfAFwd8D4T4G5jbTpmap/Tnc9FUAvd/xvwKNB5ukL+IG0INOuoOlAMLOJ9+n/qtYL3AHMaaDebcAL7nA6cAD3yzVI3f8F/u4Op7jv8aHu+MfAdKBHCz/X+6gfCP6niXmWA+cEe8/c9+v4gPFXqAlazak7H/hJwLRTaWEgaGq/xPlSKgcSGtnmYPtq0ECA8+VeDgwLmPYTYEFA+9YHTEty5+3TwLqr1xXisjeH8LnnADcH7KdfBUybQHgCwSkN/S818X+TRe1AsBcnKAcNam3xslND9T0PXIqzw82sM60nzk691D3U3wvMdcsRkSQR+Zt7OFuI8yWWKrXPtW8PGD6A8yu3Hvew/gKcc5Oo6hfAZrdt4Pyi/TbIrP2APapaENLW1relTjvGishH7mH6PpxfslWnsxpqA8A/gbNEpAtwIfCJquY1UPdF4DxxroGcByxT1U3utKtwjqDWich/ReT7LdyuQHW38XJxTrFVfaZHULONwYT0GTZR95A67ajVpmZqdL907VLV0qqREPfVhvTAOQLcFFC2CeeotEr1dqvqAXewsfepOctu9L0SkeOAAcDLbtGLwAgRGRnC+pujwXY08X9TTVWLcY5CrwXyROQdERkS5nY2yQJBHe4X0AbgDGB2ncm7gRKcQ7dU99VNnYtJAL8ABgNjVbUrcKJbLi1oyrlAV+AJcc6hb8f5Z5jmTt8CDAwy3xYgXURSg0wrJuDidwPnTbXO+Is4pxz6qWo3nPPTVdvTUBtQ1a04p3XOA36EE2CDUtU1OP/sU3AC3YsB03JU9RKgF86pjVlucGmN6m10zyU/DfwM6K6qqcAqWvaZNUcezmmhKv2aMW/dz6ip/TLYPE3tq3Xr111fBc6pwSr9ga3N2IbWLLuxtoHzPyLAcvf/ZlFAeTg11o7G/m9qL0R1nqpOwjmaX4ezP7YpCwTBXYVz2FccWKiqfpwP6VER6QUgIhkiMtmtkoLzD7lXRNKBX7eiDdOAv+OcZxzpvo4DjhLnbqZngStFZKKIeNx2DHF/db+LE0DSRCRWRKr+yb8ChovISBFJoIELbXWk4BxhlIrIMdQckYBztHKqiFwozkXX7nV+dc3EuTVwBPWDal0vAjfhfCG9WlUoIpeJSE/3vd/rFvtDaHeouuD8Q+9y13clzhFBpL0C3OR+bqk4p9JCtQPIFJE4CGm/DKapfXUHEPSeeFX1ue3/nYikuMH05zhHga3S2mW7+/WFOBeJRwa8bgAulba7w6ex/5tqItJbnJs+uuBcDywivPt3SCwQBKGq36rqkgYm3wasBxa6h9Qf4PyyAuc8YCLOr5qFOIfnzSYiGcBE4P9UdXvAa6m7zGmquhi4Euci4T7gP9T8ivoRzq+qdcBO4GZ3u74BfuO2OQcI5V70nwK/EZH9wL04/6S4y9uMc+T0C5zb9JYDRwXMO8dt05yA0wMNeQnnLo/5qro7oPx0YLWIFAGPAReraglUP8BzQgjb0CD3aOSPOEcvO3CC1metWWaIngbeA1YAXwL/xrkI6gth3vk4t1tuF5Gq96qx/TKY/6PxffUxYKp710+we/tvwDnC/A5nP3oR54dLOLRm2T/ACXAzA/933Pm9OPtTW2jw/6YOD06g24bzP3QSzsX1NiXuxQtjIkJEvsW5KPpBe7elIxPn9sonVfXQJisbE2Z2RGAiRkTOxzntMr+929LRiEiiiJzhnlLLwDk1M6e922Wikx0RmIgQkQXAMJxnAOa1c3M6HBFJwjmdNwTnVMY7wE2qWtiuDTNRyQKBMcZEOTs1ZIwxUa7TJUvq0aOHZmVltXczjDGmU1m6dOluVe0ZbFqnCwRZWVksWdLQnZ3GGGOCEZFNDU2zU0PGGBPlLBAYY0yUs0BgjDFRrtNdIzDGHPwqKirIzc2ltLS06cqmloSEBDIzM4mNjQ15HgsExpgOJzc3l5SUFLKyshCJdCLYg4eqkp+fT25uLgMGDAh5Pjs1ZIzpcEpLS+nevbsFgWYSEbp3797sIykLBMaYDsmCQMu05H2LmkDg8/nZvCa/vZthjDEdTtQEgld/v4S3/vQVqz8JRydKxpiDXXJyKD1rBrdnzx4mTZrEoEGDmDRpEgUFwXuOnTFjBoMGDWLQoEHMmDGjuvyuu+6iX79+9drw8ccfM2rUKLxeL7NmzWpx++qKmkCwZ1sRAPv32F0IxpiWqaysDKneAw88wMSJE8nJyWHixIk88MAD9ers2bOH6dOns2jRIhYvXsz06dOrA8ZZZ53F4sWL683Tv39/nnvuOS69NGiHZy0WNYFAYpzzZuq3bKvGmNAtWLCAE044gbPPPpthw4aFNM8bb7zBtGlOF8nTpk3j9ddfr1dn3rx5TJo0ifT0dNLS0pg0aRJz5zodxY0bN46+ffvWmycrK4sjjzwSjye8X91Rc/uoRwQ/ir/NewM1xrTG9LdWs2ZbeLtpGHZIV3591vCQ6y9btoxVq1ZV35J5wgknsH///nr1Hn74YU499VR27NhR/UXep08fduzYUa/u1q1b6devX/V4ZmYmW7e2z6nriAUCtxPpj4F4dz2zVDVoZ+5uT1azgDGN9BXcuva4AVQrKyKxeGPMQeyYY46pdV/+J598EvK8ItLh74CK5BFBGXCKqhaJSCzwqYi8q6oLAyuJSApwE7Aogm3BQyXgwX9gXyRXY4wJs+b8co+ULl261Bpv6oigd+/e5OXl0bdvX/Ly8ujVq1e9uhkZGSxYsKB6PDc3lwkTJoS76SGJWCBQp+uzInc01n0FO0H/W+BB4NZItQWg6pSa2qkhY0wrNXVEcPbZZzNjxgxuv/12ZsyYwTnnnFOvzuTJk7nzzjurLxC/99573H///RFpb1MierFYRGJEZDmwE3hfVRfVmT4K6Keq7zSxnGtEZImILNm1a1cL2+L89dvFYmNMhN1+++28//77DBo0iA8++IDbb78dgCVLlnD11VcDkJ6ezj333MOYMWMYM2YM9957L+np6QD86le/IjMzkwMHDpCZmcl9990HwH//+18yMzN59dVX+clPfsLw4eE5WmqTPotFJBWYA9ygqqvcMg8wH7hCVTe6nZ3/sqlrBKNHj9aWdEwz45cfUFTkYejRCZzyk2ObPb8xpu2sXbuWoUOHtnczOq1g75+ILFXV0cHqt8nto6q6F/gIOD2gOAU4AlggIhuBccCbIhK0oa1VdbHY7hoyxpjaIhYIRKSneySAiCQCk4B1VdNVdZ+q9lDVLFXNAhYCZ0fqriGPnRoyxpigInlE0Bf4SERWAP/FuUbwtoj8RkTOjuB6g6q+fdQCgTHG1BLJu4ZWAEcHKb+3gfoTItUWgHQRTkqN5etyCwTGGBMoalJM9NEYABLL2rkhxhjTwURNIKDqGoEdEBhjTC1REwiqH/Bug9tljTGdX3umoV66dCkjRowgOzubG2+8karb/F999VWGDx+Ox+OhJbfRNyRqAoHPfSPLKy0QGGNapq3SUF933XU8/fTT5OTkkJOTU52V9IgjjmD27NmceOKJ4dsooigQ+H1OPwS+igPt3BJjTGfS1mmo8/LyKCwsZNy4cYgIl19+efX8Q4cOZfDgweHatGpRk4Za/RXgAb/Pso8a06m8eztsXxneZfYZAVPq/0pvSFumod66dSuZmZn1yiMpagKBD+eR4kp7tNgY00yWhvogsaN8F/283SmutK4qjelUmvHLPVLaMg11RkYGubm5tcozMjLCtzFBRE0g8Pt97t92bogxptOLZBrq9PR0unbtysKFCxk7diwzZ87khhtuiMh2VImai8XV7KYhY0yEtTYN9RNPPMHVV19NdnY2AwcOZMqUKQDMmTOHzMxMvvjiC84880wmT54clva2SRrqcGppGurZP/srxyQfwSclW7nksYsj0DJjTLhYGurW6ZBpqDsGrfXHGGOMI3oCgXvVXrVjX703xpi2FjWBQNzbRy0QGGNMbVETCIwxxgQXPYHADgSMMSao6AkE1ReLLSIYY0ygKAwE7dsKY0znYGmow0BEEkRksYh8JSKrRWR6kDo/F5E1IrJCRD4UkUMj2B7ALhYbY1rO0lA3XxlwiqoeBYwETheRcXXqfAmMVtUjgVnAHyLXHDsUMMY0n6WhbgV1jmWK3NFY96V16nwUMLoQuCxS7alhRwTGdCYPLn6QdXvWhXWZQ9KHcNsxt4Vc39JQt4KIxABLgWzgL6q6qJHqVwHvNrCca4BrAPr379/Cxtg1AmNMy1ga6lZQVR8wUkRSgTkicoSqrqpbT0QuA0YDJzWwnKeAp8DJNdSStkTRVXFjDirN+eUeKZaGOgxUda+IfAScDtQKBCJyKnAXcJKqlkWsDR07IBtjOhFLQx0iEenpHgkgIonAJGBdnTpHA38DzlbVnZFqS6112jUCY0yEWRrqqgWLHAnMAGJwAs4rqvobEfkNsERV3xSRD4ARQJ4722ZVPbux5bY0DfWbtzzGqPhR/GfvLn745HnNnt8Y03YsDXXrNDcNdSTvGloBHB2k/N6A4VMjtf567EDAGGOCipprqDVxwG4bMsaYQFETCOyIwBhjgoueQGBHAsYYE1TUBIKqBzrswMAYY2qLmkBQ/WSxMcaYWqImENiRgDGmOdoiDfXpp59Oamoq3//+92uVb9iwgbFjx5Kdnc1FF11EeXk5AGVlZVx00UVkZ2czduxYNm7c2OI2BrJAYIwxIQpnGmqAW2+9leeff75e+W233cYtt9zC+vXrSUtL49lnnwXg2WefJS0tjfXr13PLLbdw223hSb8RNYHAIoExpiUilYYaYOLEiaSkpNQqU1Xmz5/P1KlT680fuNypU6fy4YcfEo6Hgtsk11CHED0hz5iDyvbf/56yteFNQx0/dAh97rwz5PqRSEPdkPz8fFJTU/F6na/nwDTUgamrvV4v3bp1Iz8/nx49eoS8/GCiJhBYjiFjTEtZGuqDRIVW2ukhYzqh5vxyj5RIpKFuSPfu3dm7dy+VlZV4vd5aaagzMjLYsmULmZmZVFZWsm/fPrp37966jSOEQCAiZwHvqKq/1WtrRypO8zt4YDbGdALhSEPdEBHh5JNPZtasWVx88cW15q9a7vjx45k1axannHJKWI42QjlzfhGQIyJ/EJEhrV5jO4n1xLZ3E4wxUSKUNNTgHFlccMEFfPjhh2RmZjJv3jwAHnzwQR555BGys7PJz8/nqquuAuCqq64iPz+f7OxsHnnkkQbvRmqukNJQi0hX4BLgSpxcDf8AXlLV+sdGEdbSNNTv3fU4w3xH8cm+nVzy1/Mj0DJjTLhYGurWaW4a6pDupVHVQmAW8DLQFzgXWCYike02JyLsCWNjjAnUZCAQkbNFZA6wAIgFjlHVKcBRwC8i27ww8liuIWOMCSaUu4bOBx5V1Y8DC1X1gIhcFZlmhV9N0jkLBcYYEyiUQHAfNV1JVvU/3FtVN6rqh5FqWPhZADDGmGBCuUbwKhB466jPLWuUiCSIyGIR+UpEVovI9CB14kXkXyKyXkQWiUhWyC1vLndLLRwYY0xtoQQCr6qWV424w3EhzFcGnKKqRwEjgdNFZFydOlcBBaqaDTwKPBhSq1tAxHJMGGNMMKF8O+4SkbOrRkTkHGB3UzOpo8gdjXVfdW/ZOQeY4Q7PAiZKpJ7F9jir7uiPehtjOoa2SEM9Y8YMBg0axKBBg5gxY0Z1+dKlSxkxYgTZ2dnceOON1YnlXn31VYYPH47H46Elt9E3JJRAcC1wp4hsFpEtwG3AT0JZuIjEiMhyYCfwvqouqlMlA9gCoKqVwD6g3vPSInKNiCwRkSW7du0KZdXBGuMO2O2jxpiWCWca6j179jB9+nQWLVrE4sWLmT59enXAuO6663j66afJyckhJyeHuXPnAnDEEUcwe/ZsTjzxxPBtFCEEAlX9VlXHAcOAoap6rKquD2XhqupT1ZFAJnCMiBzRkkaq6lOqOlpVR/fs2bMli6hmxwPGmOaIVBrqefPmMWnSJNLT00lLS2PSpEnMnTuXvLw8CgsLGTduHCLC5ZdfXj3/0KFDGTx4cLg2rVpISedE5ExgOJBQdWpFVX8T6kpUda+IfAScDqwKmLQV6AfkiogX6Abkh7rc5qi6RmCBwJjO5ZNXvmH3lqKmKzZDj37JnHDh4SHXj0Qa6sCU0lCTbnrr1q1kZmbWK4+kUJLOPQkkAScDzwBTgcUhzNcTqHCDQCIwifoXg98EpgFfuMudr+HoZSFog6oWa6HAGNM8loYajlXVI0VkhapOF5E/Au+GMF9fYIaIxOCcgnpFVd8Wkd8AS1T1TeBZ4HkRWQ/sAS5u4XY0ST0d+4MwxgTXnF/ukRKJNNQZGRksWLCgejw3N5cJEyaQkZFBbm5urfKqNNSREkogKHX/HhCRQ3BO3fRtaiZVXQEcHaT83oDhUuCC0JraOmIpJowxYRKONNSTJ0/mzjvvrL5A/N5773H//feTnp5O165dWbhwIWPHjmXmzJnccENk07qFctfQWyKSCjwELAM2Ai9GsE2R0cEPzYwxB49Q0lCnp6dzzz33MGbMGMaMGcO9995Leno6AE888QRXX3012dnZDBw4kClTpgAwZ84cMjMz+eKLLzjzzDOZPHlyWNrbaBpqca6wjlPVz93xeCBBVfeFZe0t0NI01B8+9CyD8w/nv/t3ce5fzotAy4wx4WJpqFsnrGmo3V7J/hIwXtaeQaA1pDrFhB0ZGGNMoFBODX0oIudH7InfttK5W2+MMRETSiD4CU6SuTIRKRSR/SJSGOF2RYzYk8XGGFNLk3cNqWpKWzSk7dihgTHGBArlgbKgSS3qdlTTWVgYMMaY2kJ5juDWgOEE4BhgKXBKRFoUIeqxU0LGGBNMKEnnzgp4TQKOAILnVO3Q7IEyY0zo2isN9YEDBzjzzDMZMmQIw4cPr34GIdBrr72GiIQtFXVLemvJBTrvDb4iVPpDSyVrjDGB2ioN9S9/+UvWrVvHl19+yWeffca779Zk9dm/fz+PPfYYY8eODc9GEUIgEJE/i8if3NfjwCc4Txh3Mm7HNKoUlHbCAxpjTLto6zTUSUlJnHzyyQDExcUxatSoWrmH7rnnHm677TYSEhJav3GuUK4RBB57VAIvqepnYWtBW3Efg0jwxPDp1k85d9C57dwgY0woPnruKXZu+i6sy+x16GGcfMU1IddvyzTUgfbu3ctbb73FTTfdVN2OLVu2cOaZZ/LQQw+F3P6mhBIIZgGlquqD6l7HklT1QNha0Qa8PmdTh3RJY1bh0nZujTGmM2mPNNSVlZVccskl3HjjjRx22GH4/X5+/vOf89xzzzV7WU0JJRB8CJwKVPUMkQi8Bxwb9tZEUKW3AoDc0iLKfGXt3BpjTKia88s9UtoyDXWVa665hkGDBnHzzTcDzrWBVatWVdfZvn07Z599Nm+++SajRwdNIRSyUAJBQkAn9KhqkYgktWqt7UDdjmn2VpYzoNuAJmobY0zDIpmGGuDuu+9m3759PPPMM9X1u3Xrxu7du6vHJ0yYwMMPP9zqIACh3TVULCKjqkZE5HtASavX3NYCuqpcu2dt+7bFGHNQa00a6tzcXH73u9+xZs0aRo0axciRI2sFhEhoNA01gIiMAV4GtuF8j/YBLlLVdjnR3tI01O//5RmGbhnMl6Ub+cO4p/jk4tDP8Rlj2paloW6d5qahDiXX0H9FZAgw2C36WlUrWt3SduKXMvaW7W3vZhhjTIcRynME1wNdVHWVqq4CkkXkpyHM109EPhKRNSKyWkRuClKnm4i8JSJfuXWubNlmhMC9aJ9UWf+ijTHGRLNQrhH8WFX3Vo2oagHw4xDmqwR+oarDgHHA9SJS92mM64E1qnoUMAH4o4jEhdLw5qq6fat3XGIkFm+MMZ1WKIEgJrBTGhGJAZr8slbVPFVd5g7vB9YCGXWrASnu8pOBPTgBJGLSY8P3NJ4xxhwMQrl9dC7wLxH5mzv+E7csZCKSBRwNLKoz6XHgTZwL0Sk4F6H9Qea/BrgGoH///s1ZdcBCoMJfxsbSznfDkzHGRFIoRwS3AfOB69zXh9ROTd0oEUkGXgNuVtW6PZtNBpYDhwAjgcdFpGvdZajqU6o6WlVH9+zZM9RV124Hgqpan8XGGFNHKGmo/ar6pKpOVdWpwBrgz6EsXERicYLAC6o6O0iVK4HZ6lgPbACGhN785vHjx2NxwBgTgvZKQw2wdOlSRowYQXZ2NjfeeCNVt/mHutzmCikNtYgcLSJ/EJGNwG+AdSHMI8CzwFpVfaSBapuBiW793ji3qIY3u1QAVZ8dERhjWqyt0lBfd911PP300+Tk5JCTk8PcuXNDXm5LNBgIRORwEfm1iKzDOQLYgvMA2smqGsoRwXHAj4BTRGS5+zpDRK4VkWvdOr8FjhWRlTinnG5T1d0NLbBVBBK9KfSN63TZMYwx7ait01Dn5eVRWFjIuHHjEBEuv/zy6vlDWW5LNHaxeB1O3wPfd0/bICK3hLpgVf2UJjoEU9VtwGmhLrNVqtJQxzibXFpZSoLX7iAypqPb+9a3lG8rDusy4w7pQupZA0Ou35ZpqLdu3UpmZma9ciCk5bZEY4HgPOBi4CMRmYuTZqLTn1cprCwH4JuCbziy55Ht3BpjTGfQHmmo23K5DQYCVX0deF1EugDnADcDvUTkr8AcVX0vLC1oIyJCQdkOSnEuAC3dsdQCgTGdQHN+uUdKW6ahzsjIqNUjWW5uLhkZziNYoSy3JUK5a6hYVV9U1bOATOBLnFtKOx1B8Ijg8Xt4bNlj7d0cY0wn9cknn7B8+fJ6r1NPPRWoSUMNNJqG+r333qOgoICCggLee+89Jk+eTN++fenatSsLFy5EVZk5c2b1/KEstyWa1Xm9qha49/RPDMva21hqfC96xyWRWJFCj8Qe7d0cY8xBqjVpqAGeeOIJrr76arKzsxk4cCBTpkxpdLmt1WQa6o6mpWmo5z/9HId/6xxi3tjj7+T0XMLKaSvD3TxjTBhYGurWaW4a6mYdEXRmgc8PeP0RyWtnjDGdUihpqB8MpazDC7i47tGoiX/GGNOkUL4RJwUpmxLuhkRcQCAoitvbbs0wxoSms5227iha8r419mTxde4Tv4NFZEXAawOwohXtbBeBOU0PKcxuv4YYY5qUkJBAfn6+BYNmUlXy8/NJSGjew7KNPVD2IvAucD8QeGl6v6ruaX4T29eKrfs43B1OLkttz6YYY5qQmZlJbm4uu3btau+mdDoJCQm1nkwORWMPlO0D9onI3cB2VS0TkQnAkSIyM7DXss5gQ34xuXxDZpfDSSnr3t7NMcY0IjY2ttaTvCayQrlG8BrgE5Fs4CmgH87RQqdSWu4j1hMPQK/iFnZuY4wxB6FQAoFfVStxcg/9WVVvBfpGtlnhd0RGGr0TD61Vtq9sXzu1xhhjOo5QAkGFiFwCXA687ZbFRq5JkeGNqRmuuoEorzivXdpijDEdSSiB4EpgPPA7Vd0gIgOA5yPbrPCL8XjIL90GQKwbCdbmr23HFhljTMcQStK5NThJ5pa54xtUtdM9UOaNEbonHAJA31hns2flzGrPJhljTIcQypPFZ+F0MD/XHR8pIm9GuF1hFyOBm+rcm7xiV6d7HMIYY8IulFND9wHHAHsBVHU5cFjEWhQhMQFbmhFrKSaMMaZKSBeL3WcKAvmD1gwgIv1E5CMRWSMiq0XkpgbqTXD7M14tIv8JpdEt4Y2puVrcZOONMSaKNJZi4jx3cLWIXArEiMggEfkz8HkIy64EfqGqw4BxwPUiUqvnZxFJBZ4AzlbV4cAFLdiGkMR4hNzibwAotUhgjDHVGjsiuNv9ewMwHCgDXgIKcbqtbJSq5qlq1QXm/cBaIKNOtUuB2aq62a23szmNbw5vjJDkTQHg0Hg7NWSMMVUayzUEgKoeAO5yXy0iIlnA0cCiOpMOB2JFZAGQAjymqjODzH8NcA1A//4teyq4sKSCcn9Zi+Y1xpiDWWOBYIiINHhbjaqG1PO7iCTjpKm4WVULg6z/e8BEIBH4QkQWquo3ddb1FE56C0aPHt2ydIQi7CjZQJ/ELHeh1EpNbYwx0aqxQLABOKs1CxeRWJwg8IKqzg5SJRfIV9VioFhEPgaOAr4JUrdVEmJjKK6siUMJlV0ojS1m54Gd9ErqFe7VGWNMp9HYyfJyVd3U0KupBYuIAM8Ca1X1kQaqvQEcLyJeEUkCxuJcSwi7jNREdpVuqR73i3PFeP7m+ZFYnTHGdBqNBYLPWrns44AfAae4t4cuF5EzRORaEbkWQFXX4jyotgJYDDyjqqtaud6gYmM8lPqKq8cTK5IB+N2i30VidcYY02k01h/Bz1qzYFX9lBDOwqvqQ8BDrVlXKGJiajelZ1F/9iVapxfGGBM191GK1n54YMCeEe3UEmOM6ViiJhDkrl9Ta3zgnqPbqSXGGNOxNPkcAYCIHAtkBdYPdr9/RxbjiZqYZ4wxzdJkIBCR54GBOBlIfW6xAp0qEHg8DV+uWL5zOSN7jWy7xhhjTAcSyhHBaGCYqrbsQa4OwiMNB4KPtnxkgcAYE7VCOV+yCugT6YZEnDQ8+vdVf2/TphhjTEcSyhFBD2CNiCzGSTwHgKqeHbFWRYDUiQSpMZZfwhhjILRAcF+kG9EWMrMOY9Wympx3Pb0WCIwxBkLLPhqxzmLaUmp6j1rj2Ql2F5ExxkBofRaPE5H/ikiRiJSLiE9E6mYR7TQKKvYDECvC3/Ii1v2BMcZ0GqH8LH4cuATIwUkVfTXwl0g2KhIK9xYA8N2+L6rLhgV0VbZ85/K2bpIxxnQIIZ0fUdX1QIyq+lT1H8DpkW1W+HncB8q2HVhfXRbr61I9/Nev/trmbTLGmI4glIvFB0QkDlguIn8A8uiEqSk8buf1Jb6i6rIiXw+qboT6fFso3TAbY8zBJ5Qv9B+59X4GFAP9gPMj2ahIqDoiUGqei5u15w/t1RxjjOkwQrlraJOIJAJ9VXV6G7QpIqSBXEND4tJYV+5cP1BVpJEnkI0x5mAUyl1DZ+HkGZrrjo8UkTcj3K6wK/lyeb2yBIFncmq6ZZ63cV4btsgYYzqGUE4N3QccA+wFUNXlwICItShCtLy8XlmvWKGbv+ZU0a0f39qWTTLGmA4hlEBQoar76pR1ugR0RRVl9cq6ezvdNW9jjAm7UL4JV4vIpUCMiAwSkT8DTd5iIyL9ROQjEVkjIqtF5KZG6o4RkUoRmdqMtjeLxxNTr6x/XP3NL/fVP3IwxpiDWSiB4AZgOM59li8BhcDNIcxXCfxCVYcB44DrRWRY3UoiEgM8CLwXYptbpOr2UYCUM7Kqh8v9Cdw77t7q8bs/vTuSzTDGmA6nyUCgqgdU9S5VHaOqo93h0hDmy1PVZe7wfmAtkBGk6g3Aa0BE8z2kdkmpHvYfnlY9vLR4Khd4a/IQvbvx3Ug2wxhjOpwGbx9t6s6g5qShFpEs4GhgUZ3yDOBc4GRgTCPzXwNcA9C/f/9QV1tLYnxCzfK6xlUPLys+n/HP/wAGtGy5xhjT2TX2HMF4YAvO6aBF1OvaJTQikozzi/9mVa2brO7/gNtU1d/Y/fuq+hTwFMDo0aNbdKFavDWnhjbsLqanO1xV2iuxFztLnIOSL7Z9wfhDxrdkNcYY0+k0dmqoD3AncATwGDAJ2K2q/wk1NbWIxOIEgRdUdXaQKqOBl0VkIzAVeEJEfhB680MnpTVns5ZuKqgeHtvFCQXzptY8Q3DN+9dEognGGNMhNRgI3ARzc1V1Gs7F3vXAAhH5WSgLFucn/rPAWlV9pIF1DFDVLFXNAmYBP1XV15u5DSEpWvBx9bAv4NmBnrHOW+D1VUZitcYY0+E1erFYROJF5Dzgn8D1wJ+AOSEu+zicPEWniMhy93WGiFwrIte2qtUtkHbuudXDu/bXf6aAvK9qja7ctTLSTTLGmA6hsYvFM3FOC/0bmK6qq5qzYFX9lGZcV1DVK5qz/OaShPjq4XlrtvOrn41h5+PLAfCpl5i/n8aKX+/lyJlHAnDpvy9l5TQLBsaYg19jRwSXAYOAm4DPRaTQfe3vjD2Uebw1MW/LnhLiMmtuJ31yx6sA9RLOzd88v20aZ4wx7aixawQeVU1xX10DXimq2rUtGxkOFbm5DU5Lj6kJAPeMu6d6+KaPGnwY2hhjDhpRk2xH4uIanHZCinu0sOo1Lhx8Ya1pPr8vks0yxph2F0WBIL7R6YWVPWHW/9QrH/n8SEoqSyLVLGOMaXdRFAhqjggOK95AaYWPnr8cXV32/O6nqofrXiSe8K8JEW+fMca0l6gJBJ6kxOrhM3fO5e7XVxHfo6asW9UjxjtWA3D9yOurpx2oPIBqp8u8bYwxIYmaQBA/eHCt8WUBTxcDTEiJdQb+eiwA1x5V+1GHqttKjTHmYBM1gSD2kENqjX+3u7henYX7L601/vKZL9ca/3DTh+FvmDHGtLOoCQTenj1rF7inenrdeHR10dLiC5yB+7oBMLzH8Fqz3LzgZlbtbtZzdcYY0+FFTSCgzsNi4woWAxB3SHJ12eSuXio1tla9pZctrTV+yTuXUOGviFAjjTGm7UVNIKj71PDg4hz8/toXgBM8wmf7r3RGVjhPG8fFxDH/gtpPGI96fpQFA2PMQSNqAkFdXSv388SC9QD0uPqI6vJvSqY4A7Ovri7rmVTntBJOMLCHzYwxB4OoDQQAD7/3DT6/kpBd03XllG6x7Pd1d0YeHFBdHiwB3cjnR0a6icYYE3FRHQgAbnzpy3plM3c94wyU7Km+qAzBg8GYf45hd8nuiLXPGGMiLaoDgUd9vLMyD4A+t9V0mXxslxiWFbn9F0xPrTXPistX1Bov9ZVy8isns69sX0TbaowxkRJVgSClpHaHNCmV+6uHvWk1ndv3jPXwRdHlbC8f5BQ8NrJ6mojw1eW1O7EBOP7l45ny2pTwNtgYY9pAVAWCfntqd6Nwee5LTNr5AUs27gGg189GVk8bkuDhtT1/cEYKNsDWZdXTPOIJepootyiXETNGMHfD3PA33hhjIiRigUBE+onIRyKyRkRWi0i95P4i8kMRWSEiK0XkcxE5KlLtAUgurX/L55DiHKY++QVArc5qBic4yYf2VfZ2Cp4+Gcr215q3oR7Mbv34VkbMGEFheafrv8cYE4UieURQCfxCVYcB44DrRWRYnTobgJNUdQTwW+ApIqhHUfB00ifkf1o9nP7DodXD56TG8s/dT+JTNyPd/ZlQp5P7ldNWctqhpwVd7nEvHcdN861zG2NMxxaxQKCqeaq6zB3eD6wFMurU+VxVq7K/LQQyI9WexowsXEnW7e/wzY79JI3oUWtagsCTO2bVFPy2O1TWvtbwxwl/5IOpHwRd9vwt8xkxYwTfFHwT9nYbY0w4tMk1AhHJAo4GFjVS7Srg3bZoTzCj9n7J5fe/gqrS945jqssnd3NSTswufrym8v/2ggN7as3fu0vvRju7P//N8xkxYwQ5BTnhbbgxxrRSxAOBiCQDrwE3q2rQk+YicjJOILitgenXiMgSEVmya9euFrclceTIBqcdV7CQC/Lm8Of564npVrs3s3NSY8nbn8F87q8p/MMAePvn9ZazctpKll22rF55lfPePI+z5pzFW9++1ez2G2NMJEgkO1wRkVjgbWCeqj7SQJ0jgTnAFFVt8vzJ6NGjdcmSJS1qT/EXX/D1NT/mo2FZDdb5Z8bFPPLDsUwYlc3WO2quHeRX+vm0yEkpcX2fc2vP9Ou99ZLaAeQV5XHaa8GvH1TpldiLuefPJTYmttF6xhjTGiKyVFVHB5sWybuGBHgWWNtIEOgPzAZ+FEoQaK3EI48ksaLx/ECXbX2ZZX+4hTP+9Cm9f/696vLuXg/pMc6X/V+2z6HcX/PcAdNToaK03rL6Jvdl5bSVPDrh0QbXt7NkJ6P+OYq/r/o757x+jj2lbIxpc5E8NXQc8CPgFBFZ7r7OEJFrRaSq+697ge7AE+70lv3UD5GnS5eQ636Xu4u8GEg5qeb69QkpXqp+9z+98yVm5T9Qk4Hid72dfgzqXDsAOPXQU1k5bSW/HP3LBtf36NJH+W7fd5z8yslM/2I6uw7sosJnGU6NMZEX0VNDkdCaU0MAa4cM5d9HDWyy3vs9TmZdyhDeuP44uv+l9kXgt/dWEHhccXrqgwxMWFhT8IMnYeQlDS673FfO9/75vQan13Xv+Hv5wcAf2OkjY0yLNXZqKCoDwfZuXViW1afJui9mXEh+XHdu2PBXLhpQ+zp23WAAMDn1IbITPg8o+D2Mv77B5fvVzw/e+AEb9m0Iuf3LfrSMWI8FBGNM81ggCLDvrbfZduutIR0V1FU3GAC8sbf26ZuTuj7J8MR5ta8dDzoNzn8WEro2uGxV5aR/nURBWUGDdeq6bcxtPLPyGX5//O/JSMng0K6HhjyvMSa6WCAIUFlQQM74YylIiueLQc17fm1R6mgeSZtYr7xuMPBXbmd8l0cZnRbk+vftmyGhW6PrOVBxgEmzJjU7RcUNR9/ABYdfwJNfPcl1R11HakJqs+Y3xhy8LBDUsXbIUBR4twVHBRD8yOCj/NUUxhwOQGmBc5NUQprznMFF3W8hOSafBE9ArqL0w+DG+n0hBPPp1k+57oPrmt3OHok9OPaQY8lIzuCnI3/a7PmNMQcPCwR1HFi2jE2X/pC9ifF8fnjLslpkJh3Ocb3PrVf+xt6KeoEg0Ild/8aIpDrZSU+6HSbcHvRZhLpy9+cyZXbr012nJ6Tz/tT3iYuJa/WyjDEdnwWCINYOcZLLteRaQRUPHi4YcGu98gV5L7OjdFOtQOCv3Ip4UhFPFwbGf06cp5hxyS+QFON0aONT4YUNIznpkss5dPI08MTUW67f58NXWUFsfM0zDD6/jxvm38AnWz8Jqc1JJTGUxPvQRm4cfuCEBzjt0NPsLiVjDiIWCIKoCgQH4rwsGNq6i6zH9DiDASkj6pV/uK+QPYVzAR/+ys1AHAlpP6tX79z0uyirKGH25ky6xpZx9cDFNQcHv9oASekAvPq/d7N55XJ+8a+3G2xLua+cVbtXMW3utHrTvJXCZe/1JyeziM+OzA95+7rFd+P5Kc/jEQ+fbv2U07NOp1t8N7web8jLMMa0LwsEQZR9+y3fnfl9APK7JLAoO6OJORonCBcO+FXQaR9se578sm0AxKfe4tR3v+nVX0xlyeeoFuOv+A7xpHJk7wEc1eVNeng3sSg/E1UY33MLf1x7QvUyL5r+IJlDhofcvhfWvMDbLz/B6K/TKIv18dKk3JZualCp8ansLdvL74//PRP7T2Tz/s3kFeUxJH0IfZP7hnVdxpjms0DQgKqjAoCKGA/vHzGg1ctMi+vDaRn1f40DrNjzH9buWxh0WqCYuOHEdpkM1Fx4ntDrWxbsrDmNNbTrDpK8FYz6xd/pmj2q3jJeuOvnDD1+Atmjx/Hdsv+y/dtvWP2fD6unVx1VlFaWMu7FcfjUeSrCWyn4YhQVSCn2UpxQiUcFUaiIbXxf6b4vjoKUcvxBTjtlb+nCiWnj6HnS9xhVkMmgY45l1YL36Tn+KPomH0JMkFNhxpjwsUDQAPX7WTes5lf1/oQ4PhncLyzLzkgaxPG9z2tw+sfbXyWv5LuwrAvgsOR8jui2g0RvBeWjfsycOYsB6JKWTnFB/bQXP33mRTatXM63SxZx5o234vf78Pv8PHbZuRw58XRWfFhzQbvc6yeu0vl2f2d8Hru7lZNQHsOxK9P5eORuDt2ehAqcsMLpy2HOiVspj/VTEu/nh/P6Eetr+ILEp0fuZn1mMQCxnlgq/BX8ZeJfSItPY2j3oVTtnyW+EpJjk/FIVPWuatqZ36+I1BzBd2YWCBrhLynh66NrflG35PmCxhyWchRjepzeaJ3ZGx+lQsvDts62VBHjb/SLPhTPnbGpRfMNTR9Kl9guLNmxhMuGXsavxvyKFbtXMCx9GCW+ErrGOQ/wVfgrUFW7Q8o0S3FZJcN/PY8eyXF8ccdEYmNC389VFZ9f8TZjnsYs21zAnGVb+fVZw1q8TAsETSjftIlvJ9d8WRcmxOH1+VkwLHxP6sZILFOz6t9OWtfc3GfZVxFdGUjfHbudHd3Lmq4YBhnJGdx37H2M6T2G13Je4/C0wxnZayQAZb4yYj2xLNm+hGP6HtP4gkyH9dcF3zJpWG+yeyUDsGXPATLTEnl1aS4bdxdz06mDiPfWPhWpqgy+Zy7Txh/KHVOG8vryrYzqn8aEhxcAcPXxA/jl5MF4PcI7K/MYnZVOn64JxHiEorJKPAJej4enP/mOq08YwKyludw1ZxUL75hIn24JrMzdx+vLt3Lr5MGszSvEI8JR/VIByNtXgiD06ebcDejzK8s2F5BfVM7k4b0ZcMe/q9v58AVHMfV7LfuhaoEgBCWrV7Px/Km1yorjYinzxrBwUOsuJNc1uvtkBnYdGVLd+XkvsKs0vBd2O5rtaaXMHb+jvZvRoCuPuJIrhl9BWnwa8zbO4+heR9MrqRcllSUkxSa1d/NCkltwgN1F5Yx0v3w6ki++zSc53ktWjyRiYzx8/M0uUhJiGT+we726H6zZwcyFm/j4m1388rTDmTSsD326JdAt0bnV+foXl/HOijzSkmJZds8kvv/nT1m9rZD7zxvBHbOd5JGThvXmdz84grMe/5Q/XzKKGZ9vZPHGPeza7/wYOaxnF77bVRyWbbvvrGE88+kGcguC95feXHefOZSrTzisRfNaIAiRb+9evhk3vl55a55CbsqUjKvpGld/hw9mc9E6Fu56E0URPCj+iLSpPUx+5AUO69mFpLjat6TOX5fH9vK1fH/IaF7PeZ2HljzUTi1svR+P+DHbircxts9Yzh1U8zDilsItdI3vSrf42qlHKnx+Ply7k8nDe9c7R711bwkZqYkhrXfD7mJOdn/ZfnnPJFISvNw6awUFB8p54oejKCqrpFdKAqrKz178kutPzmbYIc5ptZJyH7uLyvhmx36mv7WGbXtLqPQrSXExPH35aLomxJIQ62H6W2soLq/klZ+M5+0V2+iVksC81dv59VnDKSyp4Ojfvs9bPzseEZi/bieFJRV879A0rnuh4d787j5zKP/7zlqe+OEo1uUV8qf560Pa3oPdxgfObNF8FgiayVdUxDejx9QqK/PGUJgQx+YeXdnRLTki6x3abTxHpp/YrHk+3PZPdpdtjUh72tKfB1zHKUN68fcrnPd9y54DLN1UwM3/Wg44O/+LizaT3iWOScN64/MrX+Xu5aXFm7nzjKEUlVYy4eEFiHcfC249nitfeoXd8a+ingOM6jWKZTsb/sLpDMrzT+TC0YeyaU8h3eJSeOvTgTxx6SieXLCVFVv3cMzRSzghayh/nO0Ek/NGZfDuyu2UNNERk+l8LBDQNoGgSsXWrayfeGqtMr9ASayXooQ4PKqklJSzJb0rOX3Tw77+Q5KyOaH3+S2a9/Odb7KleG2YWxQZcZ5Kbhj8Ra2yMaVPkCIHEJR9mozgZxepQMvu3vjh2P68sGhzvfJjstJZvDEfiSlCfclIbAFdBj6ESOf6v2itkq0X4oktxFd6CN6UlagvmYqC8Xi7Lseb9B2l289BK7uBVII6F93jes6lcv8I/KWBp07VfdndXZFigYC2DQQA/tJSfPsKie3dq9ZzBw35eHA/0otKKInzsqtr6D2iheKw5CMZ07PleYb2lG1nft6LxEgM8TFJ7K+of1tpezi1Tw5HpW0Py7I+9Q1nq/ZkvGc1D1deSJKU8bLvFAA8+EnhAPto7RGdH4nLR31JeJPXktDndcRTCUDl/iF4U9a1cvkHl8qibGISc5GYUkpyL8WTsJ3y3ROISdyCeIvwxG9HK9LwlWTi7boCb9J3gFC68wz8pYcQk7gFX2lfPLF78Jf3Ao3u1CcWCGj7QBCoYvt2YlJTEY+Hwrlz8fboweb/uap6es9bbmHXozX9E398eD+KEuMYtWE7Xr8fvwhLDgvfU7ZeieX8EO5ECtX6wmUsy/+wza89XJr1JX0Ti9p0nY0p1nie9p3J6Z7/stw/kHXan5X+AWzVHhwgnkK60NIjkxp+kEo8cfmoLxHET3L2H1CVqDsaiaTyPeOI7bYc1RjK80/CE5dPxZ5jScr6K+X5J6K+JCSmBF/JofgOZOF8rh37mYFOFQhEpB8wE+iNc6z4lKo+VqeOAI8BZwAHgCtUtdGTue0ZCBpSvGgxMSnJJAwbBoBv3z48KSmIp/bhcWVBAZtnv8aeP/2J3cmJdC8qIbnM6ctAgZI4L0w8mRXfrGZPcu0LgV1KyylOCO0+eK/EcX7WLa3fsCBy9i1lc/Fa8su2uRetBaXxfciHhzf7nEluYiaDi77h0AObOaR0Gy9kXkKP8t3kJQQPjoKfIbKFMZ51/CZ2RiQ2p9WOK32MrfRs72aApwwnk6AfNBZPgnOnmTdlLSLlxHX/rFZ19SUiMeG5k+VgdGDz/+ArPrxd2/DHC47iF69+xVlHHcJbXzkpahb8cgJZPVp2pqG9AkFfoK+qLhORFGAp8ANVXRNQ5wzgBpxAMBZ4TFXHNrbcjhgIWsNXVIwnIR5iYtCyMjwJzr3EBWtWk5DZj8SuXTmwNZe4pC5409Kq58tds4q0QzIoLSpCPMI/brm2elpSt1QGHXMsA47+Hj36HUq3Xn0oLa9k9szPOHF92527/WPZFjZ5hHxvN3LFOc6IBdrm0TlFUI71rOaFuPsjtpaflN/MPH+UPHMg5YAHpJKYxM14U1YRl+Y8wa6VSYj3QPu2L8xO8D7H1O9lcsqQ3tw1ZyUvLNrMXy4dRddEL0lxMWT3TCE5wcviDXs4ZkA6qkqlX0mIdZ5RKK3wsXpbIX/6MIfuXeLol57EyP6pnDy4F9v3laIofbomsPdABUnxMewpLufVJbnccEp2RJ5k7hCnhkTkDeBxVX0/oOxvwAJVfckd/xqYoKp5DS3nYAsE4eL3+9i0YjmHHjkSTzPz9pSXlLDgjfn0L+5Hcs6+CLWw+fbg50YOkILgB84klm34eY1y6v6WTQECuv2pvlTZ2pNc8ZRTRizjPWvYrumksZ/Z8fdVT88qfbGVa2i+qk+3o98PNLJfKsu37A067fYpQ+gSF0POziI8Ilx8TD8O75WCCFRqJUs37CM+NoZeXROY+cVG/vafHHqnl/OjE5LI6u1n7prv+E/+0xFre8/Ensy/cH6tMlXt1Kkm2j0QiEgW8DFwhKoWBpS/DTygqp+64x8Ct6lqg9/0Fgjah/oVLfdRvGQH+94OX46kaPUZFRyDl7ep4FzieNxTxil+L0PSkig8OYPU2bXf4+XiY7VWon27cFleZcjr2ds9nv43HM1XW/ZyxCHd+GDtDk4b1gefKmlJsYgIEx76iMlH9OGOKW5q9vJK4mI8eGM87Cws5dnPNnDLqYdX/9KtUjcPT6XPj18hztt57xgqKi/C6/GS4E1ounIn066BQESSgf8Av1PV2XWmhRQIROQa4BqA/v37f2/TppblpjFtx19aSf7MNZR913GOMKJV5gMnNF3JHPQaCwQR7VlERGKB14AX6gYB11YgMN1npltWi6o+BTwFzhFBBJpqwsyT4KXnNUc2ax71Kfj9+Et9eBJiKFm3h4LXcohJjqNybxmxPRLwV/jx5ZdGqNXGRKeIBQL3jqBngbWq+kgD1d4EfiYiL+NcLN7X2PUBc3CTGIGYGGLcUxBJI3qSNCL8d+T4isqJSXbuwFJVyjcWEtMtHjwCPj8xqQngAS2pRGI9IELpt3uJSYnjwPKdFH28ldhDulCxzclHk3R0L+L6p+BJjoNKPzFp8ZSszqdy5wE8KXHEZSRzYPkuKncdIOnoXhR9ti1s2+LtmYivsBwtq7li0Ov6kfhLKlG/knB4WiNzG+OI5F1DxwOfACupuWZ3J9AfQFWfdIPF48DpOLePXtnY9QGwawTGGNMS7XJqyD3v3+gldnWi0PWRaoMxxpimdd7L+8YYY8LCAoExxkQ5CwTGGBPlLBAYY0yUs0BgjDFRzgKBMcZEOQsExhgT5TpdxzQisgtoabKhHsDuMDbnYGHvS3D2vgRn70twHf19OVRVgz6q3+kCQWuIyJKGnqyLZva+BGfvS3D2vgTXmd8XOzVkjDFRzgKBMcZEuWgLBE+1dwM6KHtfgrP3JTh7X4LrtO9LVF0jMMYYU1+0HREYY4ypwwKBMcZEuagJBCJyuoh8LSLrReT29m5POIhIPxH5SETWiMhqEbnJLU8XkfdFJMf9m+aWi4j8yX0PVojIqIBlTXPr54jItIDy74nISneeP7mdCTW4jo5ERGJE5Eu3b2xEZICILHK35V8iEueWx7vj693pWQHLuMMt/1pEJgeUB92fGlpHRyEiqSIyS0TWichaERlv+wuIyC3u/9AqEXlJRBKian9R1YP+BcQA3wKHAXHAV8Cw9m5XGLarLzDKHU4BvgGGAX8AbnfLbwcedIfPAN7F6TBoHLDILU8HvnP/prnDae60xW5dceed4pYHXUdHegE/B14E3nbHXwEudoefBK5zh38KPOkOXwz8yx0e5u4r8cAAdx+KaWx/amgdHeUFzACudofjgNRo31+ADGADkBjwGV4RTftLu38IbfRBjwfmBYzfAdzR3u2KwHa+AUwCvgb6umV9ga/d4b8BlwTU/9qdfgnwt4Dyv7llfYF1AeXV9RpaR0d5AZnAh8ApwNvuF9NuwFt3nwDmAePdYa9bT+ruJ1X1GtqfGltHR3gB3dwvPKlTHtX7C04g2IIT2Lzu/jI5mvaXaDk1VPVBV8l1yw4a7uHp0cAioLeq5rmTtgO93eGG3ofGynODlNPIOjqK/wN+RU1/2d2Bvapa6Y4Hbkv19rvT97n1m/t+NbaOjmAAsAv4h3vK7BkR6UKU7y+quhV4GNgM5OF8/kuJov0lWgLBQU1EkoHXgJtVtTBwmjo/NSJ6j3BbrKM5ROT7wE5VXdrebelgvMAo4K+qejRQjHOaplqU7i9pwDk4gfIQoAtwers2qo1FSyDYCvQLGM90yzo9EYnFCQIvqOpst3iHiPR1p/cFdrrlDb0PjZVnBilvbB0dwXHA2SKyEXgZ5/TQY0CqiHjdOoHbUr397vRuQD7Nf7/yG1lHR5AL5KrqInd8Fk5giPb95VRgg6ruUtUKYDbOPhQ1+0u0BIL/AoPcK/RxOBd43mznNrWae0fGs8BaVX0kYNKbQNWdHNNwrh1UlV/u3g0yDtjnHq7PA04TkTT319FpOOcq84BCERnnruvyOssKto52p6p3qGqmqmbhfNbzVfWHwEfAVLda3felalumuvXVLb/YvUtkADAI52Jo0P3JnaehdbQ7Vd0ObBGRwW7RRGANUb6/4JwSGiciSW67q96X6Nlf2vtCTVu9cO6A+Abn6v1d7d2eMG3T8TiH2CuA5e7rDJxzjx8COcAHQLpbX4C/uO/BSmB0wLL+B1jvvq4MKB8NrHLneZyap9GDrqOjvYAJ1Nw1dBjOP+Z64FUg3i1PcMfXu9MPC5j/Lnfbv8a9A6ax/amhdXSUFzASWOLuM6/j3PUT9fsLMB1Y57b9eZw7f6Jmf7EUE8YYE+Wi5dSQMcaYBlggMMaYKGeBwBhjopwFAmOMiXIWCIwxJspZIDDGmChngcB0eG7q5J+2YL5/i0hqE3V+IyKntrhxwZf5ufs3S0QuDfOy7wy2LmNaw54jMB2em1DvbVU9ok65V2sSdnU4IjIB+KWqfr8Z8zS6TSJSpKrJYWieMdXsiMB0Bg8AA0VkuYj8V0Q+EZE3cdIAICKvi8hSt2ORa6pmEpGNItLD/WW+VkSeduu8JyKJbp3nRGRqQP3pIrJMnM5VhrjlPcXpTGW1m7Fzk4j0aKixIlIU0O4T3HbfIk5HOQ+527BCRH7i1p8QyjaJyANAoru8FwLX5aaBeEicjlVWishFActeIDWd0bzgplEwpkZ7P9ptL3s19QKygFXu8AScrJkDAqZXpURIxEkR0N0d3wj0cOevBEa65a8Al7nDzwFTA+rf4A7/FHjGHX4cN888TlZKBXo00t6igLa+HVB+DXC3OxyPk+phQDO3qaiBdZ0PvI/TCUpvnPw5fd1l78NJaOYBvgCOb+/P1F4d62VHBKYzWqyqGwLGbxSRr4CFOFkeBwWZZ4OqLneHl+IEh2BmB6lzPE4WU1R1LlDQwnafhpPEbTlOvxHdA9rakm0KdDzwkqr6VHUH8B9gTMCyc1XVj5OPKiv4Iky08jZdxZgOp7hqwD0PfypOj1EHRGQBTlKwusoChn04v7SDKQuoE+7/D8E54phXq9DZhpZsU6jqbrv935ta7IjAdAb7cfpkDqYbUOB+YQ7B6S833D4DLgQQkdNwMnaGom675wHXidOHBCJyuDg9hNXV2DZVVM1fxyfARe51iJ7AiThZLY1pkv0yMB2equaLyGcisgooAXYETJ4LXCsia3FS/y6MQBOmAy+JyI9wzrFvx/mSb8oKwOee4nkOp3OcLGCZe8F2F/CDIPM1tk1PAStEZJk6fSxUmYPT5+1XONcwfqWq26sueBvTGLt91JgmiEg84FPVShEZj9PV48h2bpYxYWNHBMY0rT/wioh4gHLgx+3cHmPCyo4IjGkBEanqcauuiaqa39btMaY1LBAYY0yUs7uGjDEmylkgMMaYKGeBwBhjopwFAmOMiXL/D7UT44JTHNJIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = None\n",
    "for result in result_grid:\n",
    "    label = f\"lr={result.config['lr']:.3f}\"\n",
    "    if ax is None:\n",
    "        ax = result.metrics_dataframe.plot(\"training_iteration\", \"val_loss\", label=label)\n",
    "    else:\n",
    "        result.metrics_dataframe.plot(\"training_iteration\", \"val_loss\", ax=ax, label=label)\n",
    "ax.set_title(\"Mean Accuracy vs. Training Iteration for All Trials\")\n",
    "ax.set_ylabel(\"Mean Test Accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e7ceebc-3958-4ed0-b578-505caffe3fb9",
   "metadata": {},
   "source": [
    "### Train and Test with Best Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48751c95-d5f4-4d44-8eac-5ebcd8bba252",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'training_iteration': 540059,\n",
       " 'train_loss': 1.7585725784301758,\n",
       " 'val_loss': 1.9896790981292725,\n",
       " 'time_this_iter_s': 0.06922125816345215,\n",
       " 'done': True,\n",
       " 'timesteps_total': None,\n",
       " 'episodes_total': None,\n",
       " 'trial_id': 'f6250653',\n",
       " 'experiment_id': '352e72ef24b0485790725bb268154829',\n",
       " 'date': '2023-03-20_03-45-17',\n",
       " 'timestamp': 1679283917,\n",
       " 'time_total_s': 35661.531294584274,\n",
       " 'pid': 88428,\n",
       " 'hostname': '129-146-49-170',\n",
       " 'node_ip': '10.19.31.32',\n",
       " 'config': {'n_embd': 24,\n",
       "  'n_hidden': 456,\n",
       "  'lr': 0.0004390312584437158,\n",
       "  'last_layer_factor': 0.9956496873025195,\n",
       "  'batch_size': 64,\n",
       "  'max_steps': 540060},\n",
       " 'time_since_restore': 35661.531294584274,\n",
       " 'timesteps_since_restore': 0,\n",
       " 'iterations_since_restore': 540060,\n",
       " 'warmup_time': 0.003042936325073242,\n",
       " 'experiment_tag': '2_batch_size=64,last_layer_factor=0.9956,lr=0.0004,max_steps=540060,n_embd=24,n_hidden=456'}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_result.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fae2d87b",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {'n_embd': 24,\n",
    "  'n_hidden': 456,\n",
    "  'lr': 0.0004390312584437158,\n",
    "  'last_layer_factor': 0.9956496873025195,\n",
    "  'batch_size': 64,\n",
    "  'max_steps': 540060}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2b4949f-186b-4dd0-810f-a7ada055f932",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No of parameters  869355\n",
      "Initial loss  tensor(3.4630, device='cuda:0', grad_fn=<NllLossBackward>)\n",
      "      0/ 540060: 3.4067\n",
      "  10000/ 540060: 2.4061\n",
      "  20000/ 540060: 2.1215\n",
      "  30000/ 540060: 2.2546\n",
      "  40000/ 540060: 2.2071\n",
      "  50000/ 540060: 2.1789\n",
      "  60000/ 540060: 2.5076\n",
      "  70000/ 540060: 2.0653\n",
      "  80000/ 540060: 1.9867\n",
      "  90000/ 540060: 1.9995\n",
      " 100000/ 540060: 2.2169\n",
      " 110000/ 540060: 2.2312\n",
      " 120000/ 540060: 2.0663\n",
      " 130000/ 540060: 1.8910\n",
      " 140000/ 540060: 1.9463\n",
      " 150000/ 540060: 2.0409\n",
      " 160000/ 540060: 1.8872\n",
      " 170000/ 540060: 2.1610\n",
      " 180000/ 540060: 1.8664\n",
      " 190000/ 540060: 2.0235\n",
      " 200000/ 540060: 1.8428\n",
      " 210000/ 540060: 1.8524\n",
      " 220000/ 540060: 1.9312\n",
      " 230000/ 540060: 1.8659\n",
      " 240000/ 540060: 1.7783\n",
      " 250000/ 540060: 1.8855\n",
      " 260000/ 540060: 2.0965\n",
      " 270000/ 540060: 1.8449\n",
      " 280000/ 540060: 2.2505\n",
      " 290000/ 540060: 2.0819\n",
      " 300000/ 540060: 1.9650\n",
      " 310000/ 540060: 1.8367\n",
      " 320000/ 540060: 1.9641\n",
      " 330000/ 540060: 2.1167\n",
      " 340000/ 540060: 1.8066\n",
      " 350000/ 540060: 1.8251\n",
      " 360000/ 540060: 1.5952\n",
      " 370000/ 540060: 2.0327\n",
      " 380000/ 540060: 1.8862\n",
      " 390000/ 540060: 1.8184\n",
      " 400000/ 540060: 1.8827\n",
      " 410000/ 540060: 1.8935\n",
      " 420000/ 540060: 2.1733\n",
      " 430000/ 540060: 1.8394\n",
      " 440000/ 540060: 2.0016\n",
      " 450000/ 540060: 2.0970\n",
      " 460000/ 540060: 1.6659\n",
      " 470000/ 540060: 1.5410\n",
      " 480000/ 540060: 1.9260\n",
      " 490000/ 540060: 1.6596\n",
      " 500000/ 540060: 1.7006\n",
      " 510000/ 540060: 1.7158\n",
      " 520000/ 540060: 1.8447\n",
      " 530000/ 540060: 1.9106\n",
      " 540000/ 540060: 1.7005\n"
     ]
    }
   ],
   "source": [
    "model = train_no_tune(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6b0fed4-7419-4996-ade2-18e35bbf9bf2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss  tensor(1.7622, device='cuda:0')\n",
      "Val loss  tensor(1.9855, device='cuda:0')\n",
      "Test loss  tensor(1.9815, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "for layer in model.layers: layer.training = False\n",
    "\n",
    "with torch.no_grad():\n",
    "    train_loss = F.cross_entropy(model(Xtr), Ytr)\n",
    "    val_loss = F.cross_entropy(model(Xdev), Ydev)\n",
    "    test_loss = F.cross_entropy(model(Xte), Yte)\n",
    "    print('Train loss ', train_loss)\n",
    "    print('Val loss ', val_loss)\n",
    "    print('Test loss ', test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da963f96",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "zhnn",
   "language": "python",
   "name": "zhnn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}