tutorial_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6d45fad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt # for making figures\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4b8c662c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read in all the words\n",
    "words = open('names.txt', 'r').read().splitlines()\n",
    "words[:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "721b682a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32033"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c970e916",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n"
     ]
    }
   ],
   "source": [
    "# build the vocabulary of characters and mappings to/from integers\n",
    "chars = sorted(list(set(''.join(words))))\n",
    "stoi = {s:i+1 for i,s in enumerate(chars)}\n",
    "stoi['.'] = 0\n",
    "itos = {i:s for s,i in stoi.items()}\n",
    "print(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f9082420",
   "metadata": {},
   "outputs": [],
   "source": [
    "block_size = 3\n",
    "X,Y = [], []\n",
    "\n",
    "for w in words:\n",
    "    #print(w)\n",
    "    contenxt = [0] * block_size\n",
    "    for ch in w + '.':\n",
    "        ix = stoi[ch]\n",
    "        X.append(contenxt)\n",
    "        Y.append(ix)\n",
    "        #print(''.join(itos[i] for i in contenxt), '-->', itos[ix])\n",
    "\n",
    "        contenxt = contenxt[1:] + [ix] # crop -remove first + append \n",
    "        \n",
    "X = torch.tensor(X)\n",
    "Y = torch.tensor(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e7af9da5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([228146, 3]), torch.int64, torch.Size([228146]), torch.int64)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, X.dtype, Y.shape, Y.dtype "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c07179b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "C = torch.rand((27, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "bd3d3aee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.4170, 0.9382])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "34b8ce28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.4170, 0.9382])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.one_hot(torch.tensor(5), num_classes=27).float() @ C\n",
    "\n",
    "# 27, @ 27, 2\n",
    "# Like a first layer \n",
    "# int @ flot -> nope "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "c7c2872b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4170, 0.9382],\n",
       "        [0.4673, 0.6043],\n",
       "        [0.5565, 0.7477],\n",
       "        [0.5565, 0.7477],\n",
       "        [0.5565, 0.7477]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C[torch.tensor([5, 6, 7])]\n",
    "C[torch.tensor([5, 6, 7, 7,  7])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "8d02be76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 3, 2])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C[X].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ff32aff2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[13, 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "00ae9f29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.4108, 0.1122])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C[X][13,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "fa4e20b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.4108, 0.1122])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "204722d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 3, 2])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb = C[X]\n",
    "emb.shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "00c26192",
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = torch.randn((6, 100)) # 3 * 2 \n",
    "b1 = torch.rand(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "82f94b5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 6])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb[:, 0, :].shape # First word 32, 2\n",
    "torch.cat([emb[:, 0, :], emb[:, 1, :], emb[:, 1, :]], dim=1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "d18b2d1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 6])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat(torch.unbind(emb, 1), 1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "66111949",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0\n",
      " 1\n",
      " 2\n",
      " 3\n",
      " 4\n",
      " 5\n",
      " 6\n",
      " 7\n",
      " 8\n",
      " 9\n",
      " 10\n",
      " 11\n",
      " 12\n",
      " 13\n",
      " 14\n",
      " 15\n",
      " 16\n",
      " 17\n",
      "[torch.storage.TypedStorage(dtype=torch.int64, device=cpu) of size 18]\n"
     ]
    }
   ],
   "source": [
    "a = torch.arange(18)\n",
    "a.view(2, 9) # 3, 3, 2\n",
    "print(a.storage()) # storage offset, strides, shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "47c027c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 3, 2])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb.view(32, 6).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ff0953b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 100])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = torch.tanh(emb.view(-1, 6) @ W1 + b1)\n",
    "h.shape\n",
    "# b1 -> \"1\", 100 ok "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "606606e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 27])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W2 = torch.rand((100, 27)) # 27 chars \n",
    "b2 = torch.rand(27)\n",
    "logits = h @ W2 + b2\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "a98ccdfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 27])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts = logits.exp()\n",
    "prob = counts / counts.sum(1, keepdims=True)\n",
    "prob.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "a5c092d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 5, 13, 13,  1,  0, 15, 12,  9, 22,  9,  1,  0,  1, 22,  1,  0,  9, 19,\n",
       "         1,  2,  5, 12, 12,  1,  0, 19, 15, 16,  8,  9,  1,  0])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "08c0cdad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3.8403e-02, 7.0592e-03, 1.7599e-03, 2.6821e-02, 6.5413e-03, 1.5030e-01,\n",
       "        1.6646e-03, 2.3824e-03, 2.0726e-02, 2.5658e-03, 3.4209e-04, 1.3199e-02,\n",
       "        6.1501e-04, 2.6742e-02, 2.5813e-02, 2.0209e-03, 8.4313e-04, 1.1306e-02,\n",
       "        1.6984e-02, 1.0087e-03, 2.6100e-01, 3.1355e-02, 9.9775e-03, 3.6842e-02,\n",
       "        5.9788e-03, 2.4443e-02, 1.5153e-01, 7.9555e-05, 6.9071e-03, 9.2070e-03,\n",
       "        4.8175e-03, 1.0235e-02])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob[torch.arange(32), Y] # Prob of network "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "cf5e4eba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(4.8609)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = -prob[torch.arange(32), Y].log().mean()\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "eefa5fcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
    "C = torch.randn((27, 2), generator=g)\n",
    "W1 = torch.randn((6, 100), generator=g)\n",
    "b1 = torch.randn(100, generator=g)\n",
    "W2 = torch.randn((100, 27), generator=g)\n",
    "b2 = torch.randn(27, generator=g)\n",
    "parameters = [C, W1, b1, W2, b2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e8c8b380",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3481"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(p.nelement() for p in parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7c229399",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "shape mismatch: indexing tensors could not be broadcast together with shapes [32], [228146]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_17092/84710896.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mcounts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogits\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcounts\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mcounts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprob\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Fushed kernel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: shape mismatch: indexing tensors could not be broadcast together with shapes [32], [228146]"
     ]
    }
   ],
   "source": [
    "emb = C[X] # 32, 3, 2\n",
    "h = torch.tanh((emb.view(-1, 6) @ W1) + b1) # 32, 100\n",
    "logits = (h @ W2) + b2 # 32, 27 \n",
    "counts = logits.exp()\n",
    "prob = counts / counts.sum(1, keepdims=True)\n",
    "loss = -(prob[torch.arange(32), Y].log()).mean()\n",
    "loss = F.cross_entropy(logits, Y)  # Fushed kernel \n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "24dd888d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# e^100 -> inf -> ops \n",
    "# e^-100 -> close to zero fine \n",
    "# cross_entropy fix this \n",
    "# - 100\n",
    "for p in parameters:\n",
    "    p.requires_grad = True "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1483bb2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "lre = torch.linspace(-3, 0, 1000)\n",
    "lrs = 10**lre # linearly to the exponents \n",
    "#lre at the end 1 \n",
    "# Step LR "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "aa9b8a6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.9937652349472046\n"
     ]
    }
   ],
   "source": [
    "lri = []\n",
    "lossi = []\n",
    "for i in range(100000): # overfit 32 ;p of 5 words \n",
    "    \n",
    "    ix = torch.randint(0, X.shape[0], (32,))\n",
    "    emb = C[X[ix]] # 32, 3, 2\n",
    "    h = torch.tanh((emb.view(-1, 6) @ W1) + b1) # 32, 100\n",
    "    logits = (h @ W2) + b2 # 32, 27 \n",
    "    for p in parameters:\n",
    "        p.grad = None \n",
    "    \n",
    "    loss = F.cross_entropy(logits, Y[ix])\n",
    "    loss.backward()\n",
    "    #lr = lrs[i]\n",
    "    lr = 0.1\n",
    "    lr = 0.01\n",
    "    for p in parameters:\n",
    "        p.data += -lr * p.grad\n",
    "\n",
    "    #lri.append(lre[i])\n",
    "    #lossi.append(loss.item())\n",
    "print(loss.item())\n",
    "#plt.plot(lri, lossi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "b73d67b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.return_types.max(\n",
       "values=tensor([13.0398, 17.0848, 20.3021, 20.1482, 16.2372, 13.0398, 15.5342, 13.7703,\n",
       "        15.5154, 17.8000, 15.3920, 20.4114, 13.0398, 16.4957, 16.5941, 19.4904,\n",
       "        13.0398, 16.1475, 14.4814, 16.4696, 18.1447, 15.4064, 10.3858, 10.2541,\n",
       "        15.1862, 13.0398, 15.7258, 16.5125, 12.4300, 15.8490, 18.5210, 15.3460],\n",
       "       grad_fn=<MaxBackward0>),\n",
       "indices=tensor([19, 13, 13,  1,  0, 19, 12,  9, 22,  9,  1,  0, 19, 22,  1,  0, 19, 19,\n",
       "         1,  2,  5, 12, 12,  1,  0, 19, 15, 16,  8,  9,  1,  0]))"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits.max(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "cc18f438",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 5, 13, 13,  1,  0, 15, 12,  9, 22,  9,  1,  0,  1, 22,  1,  0,  9, 19,\n",
       "         1,  2,  5, 12, 12,  1,  0, 19, 15, 16,  8,  9,  1,  0])"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y # ... -> e, o, s, i etc first meh \n",
    "# Loss no zero "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "93895ad2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([151608, 170234,  40998, 105770,  37434,  86300, 100430, 186107,  50646,\n",
       "         93757, 221642, 141549,  37466,  39001, 224388, 150211, 150755, 139803,\n",
       "        213677, 175277,   3432, 204899, 199025, 103530,  55743,  97003, 214838,\n",
       "        209506,  85838, 189624, 163389, 127198])"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.randint(0, X.shape[0], (32,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "8895e7ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.4496, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb = C[X] # 32, 3, 2\n",
    "h = torch.tanh((emb.view(-1, 6) @ W1) + b1) # 32, 100\n",
    "logits = (h @ W2) + b2 # 32, 27 \n",
    "loss = F.cross_entropy(logits, Y)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "6520f1a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182551, 3]) torch.Size([182551])\n",
      "torch.Size([22783, 3]) torch.Size([22783])\n",
      "torch.Size([22812, 3]) torch.Size([22812])\n"
     ]
    }
   ],
   "source": [
    "# train, dev/val, test \n",
    "# 80, 10, 10\n",
    "\n",
    "block_size = 3 # context length: how many characters do we take to predict the next one?\n",
    "\n",
    "def build_dataset(words):  \n",
    "  X, Y = [], []\n",
    "  for w in words:\n",
    "\n",
    "    #print(w)\n",
    "    context = [0] * block_size\n",
    "    for ch in w + '.':\n",
    "      ix = stoi[ch]\n",
    "      X.append(context)\n",
    "      Y.append(ix)\n",
    "      #print(''.join(itos[i] for i in context), '--->', itos[ix])\n",
    "      context = context[1:] + [ix] # crop and append\n",
    "\n",
    "  X = torch.tensor(X)\n",
    "  Y = torch.tensor(Y)\n",
    "  print(X.shape, Y.shape)\n",
    "  return X, Y\n",
    "\n",
    "import random\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "\n",
    "Xtr, Ytr = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xte, Yte = build_dataset(words[n2:])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "80243cdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6612"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
    "C = torch.randn((27, 5), generator=g) # Beat karpathy less dims here ;p (kidding)\n",
    "W1 = torch.randn((15, 150), generator=g)\n",
    "b1 = torch.randn(150, generator=g)\n",
    "W2 = torch.randn((150, 27), generator=g)\n",
    "b2 = torch.randn(27, generator=g)\n",
    "parameters = [C, W1, b1, W2, b2]\n",
    "for p in parameters:\n",
    "    p.requires_grad = True \n",
    "sum(p.nelement() for p in parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "51f2c2be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN LOSS: 1.850675106048584\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f86d018ac10>]"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNJUlEQVR4nO3deVxU5f4H8M+wKwIuCIIiornjihu4pShqapktll6zssVK0+zefnmtNKuLt8Vs07K6mi1qpa2umBuKpiIqrrmDCiKogBvbnN8fysTALOfMnJnzzPB5v168XjKcOfMcz8yZ73me7/N9dJIkSSAiIiIShIfWDSAiIiKqiMEJERERCYXBCREREQmFwQkREREJhcEJERERCYXBCREREQmFwQkREREJhcEJERERCcVL6wbIodfrcf78eQQEBECn02ndHCIiIpJBkiQUFhYiPDwcHh7y+0NcIjg5f/48IiIitG4GERER2SAzMxONGjWSvb1LBCcBAQEAbh1cYGCgxq0hIiIiOQoKChAREWH4HpfLJYKT8qGcwMBABidEREQuRmlKBhNiiYiISCgMToiIiEgoDE6IiIhIKAxOiIiISCgMToiIiEgoDE6IiIhIKAxOiIiISCgMToiIiEgoDE6IiIhIKAxOiIiISCgMToiIiEgoDE6IiIhIKAxOSFhZ+Tfw6eYTyL9eonVTiIjIiVxiVWKqnh74dDvOXr6BPWcuY8EjXbRuDhEROQl7TkhYZy/fAAAkH8vVuCVERORMDE6IiIhIKAxOiIiISCgMToiIiEgoDE6IiIhIKAxOiIiISCgMToiIiEgoDE6IiIhIKAxOqNqQJAm/7TuP4zlXtW4KERFZwOCEhCdBUmU/6w/nYNKSNAyYs1mV/REROUvBzRLMXf8XTlysHjdXDE6o2tiXeUXrJhAR2WTmrwcxd/2xanNzxeCEiIhIcGkZVwAAkjodycJjcEJERERCYXBCwtNBp3UTiIjIiRicEBERkVAYnBAREZFQGJwQERGRUBicEBERCa66Zd4xOCEiIiKhMDihakNX3W49iIhcFIMTEp5a5euJiMg1MDghl3P+yg3MSfoLOYU3tW4KERE5gJfWDSBS6h9f/ImTudeQcjwXPz4Tp3VziIgcr5oNS7PnhFzOydxrAIDdZy5r3BIiInIEBidEREQkFAYnJDyurUNEVL0wOCEil/TfNUfw5dZTWjeDiBzApuBk3rx5iIqKgp+fH2JiYpCcnGx2202bNkGn01X5OXLkiM2NJrIF+1/EcO7KDVwvLrVrH8dzCjF/0wm88fshlVpF5DhFpWUoLdNr3QyXojg4WbZsGaZMmYLp06cjLS0NvXv3xpAhQ5CRkWHxeUePHkVWVpbhp3nz5jY3msgmrMKmuRMXr6Ln7A3oOXuDXfu5WlSmUoucp4RfTg4jSRKKSsV8TxSVlqHzrCT0fWeTXftxxNVrT8ZlDP9oK3advuSAvdtHcXAyZ84cjB8/Hk888QRat26NuXPnIiIiAvPnz7f4vJCQEDRo0MDw4+npaXOjSXs3issw67dD2HEyT+umkAvZeCQHAHD5eonGLXEMvV7CzF8PYnnqWaPH1x3MRvPpq7Fkp+WbOLLN80v3ouUra5B56brWTani5MVruFZchnNXbmjdlCoe+HQ70s/l44FPt2vdlCoUBSfFxcVITU1FQkKC0eMJCQlISUmx+NxOnTohLCwM8fHx2Lhxo8Vti4qKUFBQYPRDYvl08wn8b9spPLRgh8Nfy1qF2F/3nceIT7YJ+eG3V/rZfNz1QTK2Hc/Vuikkw6a/crAo5TRe/GGf0eNPfZ0KAJi2Il2LZrm93/adBwB88+cZjVuiDb1ewsmLVyFJyqppl+nFrb6tKDjJzc1FWVkZQkNDjR4PDQ1Fdna2yeeEhYVhwYIFWL58OVasWIGWLVsiPj4eW7ZsMfs6iYmJCAoKMvxEREQoaSY5wZm8ayYfz7x0XfEHxF7PL0nD3swrmPHLAae+rjOM/d+fOJRVgDFf/Kl1U4Rw/soNHDiX7/T3mFyXrrlnjxBpT2dhWHrW74fQ/73N+HjDcSe2yLFsSoit/J8kSZLZ/7iWLVviySefROfOnREbG4t58+Zh6NChePfdd83uf9q0acjPzzf8ZGZm2tJMcrKvUk6j99sbMfPXg5q8fsFN+5Is1XTi4lVMWZqG4zmFdu0n/wa/7CqKm70Bwz7aigwBu++JtLIo5TQA4L2kv7RtiIoUBSfBwcHw9PSs0kuSk5NTpTfFkh49euDYsWNm/+7r64vAwECjHxLf7NW3ZmB9tV37rlVJknDwfD6KS7VJQhz9+Q78vPc8HvzM8cNe1dGh8xzqpeql8u3/mbxreOKrXUg9I14yqxoUBSc+Pj6IiYlBUlKS0eNJSUmIi5O/xklaWhrCwsKUvDRVcDznqmZfuq7imx1nMPTDrXhy8W5NXv9CQREA4NK1Yk1e31FulpSZPKYD5/Lxc9o5q8+31DVtyg+7M/Hdn+IlkS7blYHV6VlaN4OqsWe+2YP1h3Nw33zxklnVoHjhv6lTp2Ls2LHo0qULYmNjsWDBAmRkZGDChAkAbg3JnDt3DosXLwYAzJ07F02aNEHbtm1RXFyMb775BsuXL8fy5cvVPRKNSJKEjEvX0bhuTcUX3op+2XsOaRlX8NqwNvDwML+fNQeyMOGbPYiJrIPlXPTObELXwm2nAQCb/7roxNa4v56zNyDvWjF2TR+A+gG+hseHfbQVABAS4Iu4O4JVea2bJWX414/7AQCD2oaiXi1fK89wjrOXr+P/lt9KbD09e6jGraHqKvOyew9tKg5ORo0ahby8PMyaNQtZWVmIjo7GqlWrEBkZCQDIysoyqnlSXFyMf/7znzh37hxq1KiBtm3bYuXKlbjrrrvUOwoNvb32KOZvOoEXBrTA5AG2126ZvHQvAKB7VF0MaWe+V+m7nbfyb1I1XvTOnkBMTe8rGGMVo8WuLe92r8nOU5cwtH3V9+lfFwpVC04qBp7Xi8tQr8LftEyHvaLSNOhrRaXILriJZvVrqbI/tVwsLEJwLR9hPuP2On/lBtYcyMaDXSNQy1fxV57LKi3Tw8vTdYvA29TyZ599FqdPn0ZRURFSU1PRp08fw98WLVqETZs2GX5/6aWXcPz4cdy4cQOXLl1CcnKy2wQmADB/0wkAwPvr1UlEunTdvYYBHG3eJvGz0/UCT9dzVaLO1lGi7zsbEf/eZqRliLO69sYjOej61npMWpKmdVNUM+KTbZj1+6Eqifpleglzkv7C1mPuN03/i+STaP7Kauw85br5KK4bVlG14eoL//2yz3ouhq0kSbJaFru0TI+NR3M480cwuVdv3YisP3xB45b8rTzY/31/Fq4Xl+Luj7dijovPAMkpvJX/lXzMeIj3l73n8OEfx/CPLx07TV+LDqg3Vx6GJAEvL9/v/BdXCYMTcqhtx3PxbTUtjFRu/9l8h+372W/3oNMbSRYDj083n8BjC3fhYTsL5l0ouGn4t7XCeOaodZ0utaM3KqfwpslS8kWlZbj7462Y9RvX6wGApTszsf9sPj78w/zMSuFYeFtcKCjCowt3GnoyzU1Hz7x0HfvPXrGrGY4oblYlyJHxEq7cv8jghKq4WVKGX/aeU2WmyZgv/sT0nw6Y7brOv1GCxdtPI/dqkd2v5Siv/3YQT3+9W8ihhNUHslF4sxSrLMwc+en2LJpDWfZNv+3+nz/ser6ayhOelTp0vgDd3voD93y8rcrf1hzIxv6z+fjfNssrHR/Ntq92jatwxlpA567cwN7MK3btw9K6MEmHjHulNh29iP3nLN8s9H57I+7+eBsy8pQnnN4sKcPxnKtoP3Mt5qo01F+u0IY6TqdyTRfLdAUMTizQ6yW7V08tN3f9X/jvGusrMZeWaf8F+M7ao5i8dC9GfWZ+iprSO+DzV26afPzF7/fhtV8O4tGFO80+19a7dLUs3HYaaw9ewIFz9tfWyCm4icsKgj5XHdC6dK0Y3+w4g/wbJfgq5TTu/nirENOqf95rPlCTe7dbuTS9JaVleuRZCbyvFZUJGfg6Q8/ZGzDik204nnPV5n1YWhdm9urDVR4r08sLug5nK/u8p565jFavrsGAOZtxrbgMc9er2+OUlW/6Guquqk/qshlXi0rNZnA/8Nl2pJ65jJ3T4xES4Gfza9ws+fuN+lhcE5y5dB0f/nEMM4a3xR0hxpn6M349iAA/L4zs3MjkvpzxZVV+F37MjguGXOXj7Wp88RtxwH9UqcyLmjlXi0rR7Xbvgy1TUM9duQEPHRAWVMOudjjDU4t3Y/eZy9hwJAcbbi/299GGY4ioU9PmfZ64aP/7cfUB0z1MN4rLMOMX9Ssb3zsvBelW7tQXpZzGqvQsrHy+t9H07OrkwLn8KtfCii4WFmHj0RwMbx+OGj5iLhr7n1VVAyGyXbXuOZmT9BeiZ6zFuoOm1wUqn6679qB9CWv6CndFxWV6PPDpdiQfy8VTZgqETf1e/p2ZZmR8+VfsUvx+dybyBV6Jds66ow4v9nXWzroEPWdvQGziBiEW67J2o7/79menPDABgJsl9gV33++2bxmLExevIvOS6cUh528+gcIi+3tJK38srAUm5XIKi/DZ5hN2v767euDTFLz04368tcpyPlBW/k30e3cTvtxqeWjOnblLL1y1Dk7KE71maLQWTHaBc7vpikrLnPp6Fauzbv7rIp79LtWu/UmSZDHx01q8ZC5r/tD5Any44Tj+/ZPxirF6vSREIFCZK1cH1rJ0xrnL5letzuRaPUI7fTv/45sdGVi68++biMo3PL/uO49Tudfwxu/iJDVXDNCdcYP22373qFxcrYOT6uTr7afR8pU1WGuml8gcvV7CH4cvIKdQeSBVORt+2/E8xfso98bvhxA1bRU6vL5O3hMqxBTlU23NTUkuvFn1giFJEu6dtw0D5mxW3FY5Pt5wDOkOnMWTf6MEy1PPmjw2azIvXVd0EZV7p7ZkZ4bsngRA20CGxPXyir9vIvq/t8lhr6NWB8Tba44a/j0n6aiFLZUx17zKScCuisGJA5y8eBVjvtiBlOOWi/tcLy7DqvQsi19Ser2EP0/mocCGL5mKXr09nj7pO2XFlVakncP4r3bjznc2KXpeTsFN1e7wb5bozXbTyqli+dx3exS/piQB+87mOyzb/d11f2H4x1tV299v+84b/T7xuz148Yd9eLHSEOEXySct7icr/wZ6v70RHWZZDgIrXhgrDpVYm+K7Yo/jar5o4Uh2Ad5aeQhXHFA80ZZeO71ekh0sfrn1FNZX+iKztabQXxcK8eL3+zBgzmY0eXklftnrnPOcJ0CStSWVZzzlOri97hTQMzhxgGe/3YNtx/Mw+gvrxX2e/XYPxnxhvv7ED6mZGLVgB+79pOrUR+DWXf9Ti3dX+XJSy8ajt7okrxcbDwlZu4jNFag2wtqDF3CzxLlDWvbS6yVcVZADkXIiD5euFSPp0AWUlumRfLvq5bpKXz5vrrSctLfPhmmdFROFXbHk+fe7M7HbxpVdB89NxufJp/DKzwdUbdOq9Cy0eW2NortgvV7C8I+3YsS8FKsByoo9Z/HG74fwhIWFMU9elB+Y3/3xVizfc9Yw62by0r2IS/xD9nDZ0QvqT88u00s4oeAYAPWrOTefvlrV/dnCVStUMziB8u671DOXsGLPWbN/v1iorGZHgYX56z+n3Qo6Tly8hpIyfZXI+JONJ7Du0AVVy00r/XrJvVqEN38/hBsVAgBrVUudzVxwUnizRPZEZblfvOUBnRzmepfGLdyJ6BlrYeq6Ym5q9fCPbq3CvCjltOzXF4mtdTVOXryKNQeyFCcCnrh4FbtPX8JLP+43myhbmbm8rYPn1Z1t9uy3e1BUqjfkbb32ywEM+SDZYpCdU1iEg+cLsC/zitXkXjlJ98sUJCCbSnY+n38Tb66Ul/tRvgyImj6woc7Ij6nmr+uuauvxXFwtKsWSnRlWp7WLhMGJDe6bvx1Tv9+Hr7efdurrmsrmv3TN+M0mSRJSTuQ6pKbEjeIy7Dp9qUok/uL3+/CFaNnxMuKIw1kFaDdzHZ77Vvmwjzk3isvw2MJdsrdv8cpq7DFRoC7Zwnof5m6Ezl259QW75oCyvCKtSJKE3KtFSD52EYPnbkHz6ab/L6zp/95mTPhmj+IVqEd8ss2QaCnXJxsct5aTpdBq8fYzOJxVoDhnTC69XsJTi3djp4WCZuZYGr6tGG+W37Do9RImfJ3q8LL4n242PYRZHsOaimX/OFKxp0r9HofcwiJ8vuWkU2v+XC8uw8vL92PainQ88j/z9aREw+AEt2bN2FIN8NVKdRHKP6SOGgeVM6X5t/1ZGP35n4i3kihmy13quIU78cCn2/HFVuMP/T47Sz2r6Uh2gVGZ9XK/m8hgX3i7Eqia5+uGDcNHI+elKNo+esZabFH4RSyiV385gC5vrsfYL3fiyO2qq0r/LyqytkxA5a8aWypufrfTsdPNrdl/Nh/Hc9QfAvnz1KUqQ4ByyUm63ngkB81fWY2lOzOw7UQu1hzMdq2y+Cr589QlvLXqsCo3RKZ6Cs11Hq6+fcOidg+fIzE4uW3iEvvfLM8vSUPnN5KqPK5W1rcEyWqHQHmC22Uzsy2Ky/RYtO0Umk9fjR8U1o0oX+Fy6a5M/OWAMWJ7nb18A4PnJpsss65GToAjh6qUlvB+fJH83hlRfbPD9Be9M6dKrzFTlM2c8sX6tPLl1lMYMGeLyb/ZU0m52MHDsE99vfvWQnQr0lFkpd5NcakeG45cwNWiUuRfL0HfdzY6tG1a2H7S8szFbcdzrSbjKznbIpZEsIbByW1nLdRAkJuDseZgttXuOqUJg0ouOJuO5uBXGYmxM28vbPavH21fsVLJlNCKKs46OmxhrRdbxkbLhzUAmPzkKqnzYuo03W+hTPb+s1dwOveaydf4ZscZdHlzPR608Pzy/4u8q0UaJ7DZl9D6zY4zdgcXpirxXisuxY4Ttk9FN2f9Yfn5Qa5EyVk8dL4A4wTq7n9n7RE8vmg3Hl+0C4tSTuOMDb3aaiou1WPT0RxcLy6FJEmKlp+wxcHz+RjzxZ/o9+4mVfY341d1k7WdpdqXr3cl1npgHlWQ66CVPw5fwL2dbpXmH/JBstHfjl0oRPPQAHy9/XSVITOlTpq465hXIelux8k8u9cxulpUio1HctA6LAB3m1hIrlxJmSRrYcPUM5dx3/wU9G1R3652WfPp5hNoGRqAfq1CcLOkDPvP5qNz49rw8vSQXSJekm6tO+Xt6WEUB57KvYbPrUxXtt6+k5g6sIXRY498uRM5ChPNncmRVTnt7bGTJAmZl24gom4NkzdH/1SwVpAzLNt1q0d356lLht5aLb258hAWbz+D+FYhCAn0w5KdGfhyXBfEtw51yOsdlLGUh5IA6UKBuJ8bS9hz4mBNXl5ZpdaEWrLy5c0wUEqr6aAD37/VXW1vYCLHQwt2YEWafbUYXli2F5OWpFkMTJRYfDvBWmlip1KzVx/BY7eHhSZ+twcPfrYdH9we/39nrbwiUVeLStHmtbUm7+5225BUWdHGI1V7M+QGJofOF2D05zvwzY4zVf42f5PyZNaCmyV49ecDdh+TEpWDkY1Hzb8fzM3eqbjQ5jtrj6LPOxsN57gyvWDlzi3NXnSmH1PP4kZxGb6+/V7640gOltzOOXp5RbpNhSkrOmCi97ny8Mv5K6av8bbUbir3towFaEXA4MQJ1tiQYV9UWoaSMj1uVBifvVlSZhQ4xCZusLiPhdtOoaRMj/0WElYnL02rMrZ5zswHQglzdVBeWLYPORbK9iu9U3JUGGUuZ6fisFl5DYrKNWBspfQ7wtQXsCmW8gnKhzXmbzphcTrnt38av9aeM1cAWB4OtZctBcHu+jAZKSfy8MrPB6oMG+44Kf+9NXXZXoz4ZBte+mE/vt5xxuKQXkVqxPWVp4KbS15/7ZcDaPXqGoxftAslZXqj98+guX/npZT3GJpbJddUL6NW5Bazm7PuqM29VZZW9634nlt/OMfsVOiLhUXo9tYfdi0JMuyjrTia/Xfu3vtJf6HNa2twrELC8wgz9a1SLAxxbj1m+eZm3qYTuFZUKvwaPAxOBNVpVhJ6/XeDUSb8DYVfgq//dggTv9tjccXfX/aex90qViqVY9qKdLN/e/AzeV8CjmYugex5FevJVCYnX6hcqV4ym+RbvuheOTk1PEr1Ev5r4Y5q+k/Gr7XcQp0fe+kl29Y0qjw19aMNts8GWZF2Dnszr9h0Y2Gr8u8KOT1npWV6LN7+9x390p0ZVapIj5y3rUoAa6ocgZwcoeVOqv8ht0bUhxuO408bh3yU1ISyNnX7+12ZGPe/nYprW5WrWPzvgz+OoahUj8+T/y7LUN5jaGoYx1xsYe7GqqK2M9ZavA6LgMGJoK4Xl+FCQZHdPQNrD16wWvpe6ZRKJZUjTXHk3ba9Ll0rdsnMdkdJXH0YY7+0XOlY7Ruwg+cLMHDOZruHG9Yfyqnwb/vXGzlkYRrm6bzrWJXuvAXXkistjfHqLwcxeK5xDteejCtVAtjE1bZ16b/4wz5cvlZc5W77ZkmZxSUL1h++gDd+P6T6ewSwHsg4egYScOv/ffNfFzHLgQsNbvnrIjqZmAVqr6W77Fvl29EYnFSQeek63ll7xOYo2BEqll+29fMtp4T0PDPj8Y4YNlFzdWRra7ko1fmNJIz+fIdd0zJtoUWWj6UVnst9tvmkxYJwjnIy95oqw4vArRwYS2Xa5RplpVfv2Uq1Kz4wM4yihhINVqbu9EYShnyQjDN5t65JRaVl6PD6OvT6r+Xh5S+3nlL9cwrYN+X8erG6eS2/7TuvuGe7IktDLNWxHgzA4MTIffNT8MnGE+j61nrhyq8DpnMb5FzA5VwW3l5zFOev3MAOK/PvTVH6xXo677rNpcqdwdbuYnt8+6fzi3uN/tz8mk7KOCaQ+8jOaqzlAeY+lVZ/tlYSvrL3zZRPV+OL8RsN3i8AcCS7EH1vLwKakXcdRaV6lCic9SbC8kv/+OLPKu/akjI9tppYrFVucw9ZKI1gTmmZhFXpWS5T1dmZOJW4goozAiqWY1fzw5RtIRnLmvwbJVW+1K8WqdcLMeyjrVXqtDiq2JpIvVMisLVujD1ErxZp72KW5Tejbziwy70yOUM7C7acxJQBLaxuZ4nWFYI/2XhcVs+bKbZUUVbbnowr2JNxxfD7zZIyfPjHMUWLbaph5m8HHTLk5Q4YnJixrlIilCRJqkyxtXdRtsql2W1ZRdYcUwXk/rogr+5FZXLqeojsr2zLx21PkOlutmow7CNHqV5y+rRJOUXd5q4/hsZ1a1Z5vLynx9HDaGrM0pA75dyUid85LqncVvM2Hre7Po8t5JwKc9Po1R56Li3Tw8tTnMEUcVoisOk/HUC3//yBXSrUOrjm5MgckJdbYA9T2eF/mKhV4Ur+YSUJtEdi1RL57kZu71Z5xWERzXPAardqMLUq8MJtp01u+9462wMBU1q+skbV/bmDHScvKR6ecpYMMzmD9haRrMyRM/BsweDktsq9BpW/cC8WFqlS4lmLueUrTSx6R7bbZmJc2h11fWu9Tc+zVEeClDth5+y4ypwxi8VWSjqnF8us8+Ou1E4yrli4TwQMTswwtejS9eIyu9dVsFTtUQ5XLUVc2SY7/x+0tNKJU0Zd0ZFs8RaFdBWuPhzqTGoOad80M4NQkrSrmO1s5ioIa4XBiUKv/qLtIkqOHqJxln//JHYBICItVNdpo4A2vcrl9qs0o4vUw+BEId7Z0M92rslDZE5RibhDLo4WN3uD0AUaybkYnBAppNZaOkT0t6z8m0KurF6dKkarXZzOHgxOiIgEUcaiF0LJszPH0NXkFopzvAxOFFKysikRkRI/OmmBPSLRMTghIiIioTA4ISIiIqcveGoJgxMiIiISCoMTIiIiEgqDEyIiIhJqtXgGJ0RERISf94pTYJLBCREREQmFwQkREREJhcEJERERCYXBCREREQmFwQkREREJhcEJERERCbXiOoMTIiIiwoo9nEpMREREZBKDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhKKTcHJvHnzEBUVBT8/P8TExCA5OVnW87Zt2wYvLy907NjRlpclIiKiakBxcLJs2TJMmTIF06dPR1paGnr37o0hQ4YgIyPD4vPy8/PxyCOPID4+3ubGEhERkftTHJzMmTMH48ePxxNPPIHWrVtj7ty5iIiIwPz58y0+7+mnn8bo0aMRGxtrc2OJiIjI/SkKToqLi5GamoqEhASjxxMSEpCSkmL2eQsXLsSJEycwY8YMWa9TVFSEgoICox8iIiKqHhQFJ7m5uSgrK0NoaKjR46GhocjOzjb5nGPHjuHll1/Gt99+Cy8vL1mvk5iYiKCgIMNPRESEkmYSERGRC7MpIVan0xn9LklSlccAoKysDKNHj8brr7+OFi1ayN7/tGnTkJ+fb/jJzMy0pZlERETkguR1ZdwWHBwMT0/PKr0kOTk5VXpTAKCwsBC7d+9GWloaJk6cCADQ6/WQJAleXl5Yt24d+vfvX+V5vr6+8PX1VdI0IiIichOKek58fHwQExODpKQko8eTkpIQFxdXZfvAwECkp6dj7969hp8JEyagZcuW2Lt3L7p3725f64mIiMjtKOo5AYCpU6di7Nix6NKlC2JjY7FgwQJkZGRgwoQJAG4NyZw7dw6LFy+Gh4cHoqOjjZ4fEhICPz+/Ko8TERERATYEJ6NGjUJeXh5mzZqFrKwsREdHY9WqVYiMjAQAZGVlWa15QkRERGSOTpIkSetGWFNQUICgoCDk5+cjMDBQtf02eXmlavsiIiJydadnD1V1f7Z+f3NtHSIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhKKTcHJvHnzEBUVBT8/P8TExCA5Odnstlu3bkXPnj1Rr1491KhRA61atcL7779vc4OJiIjIvXkpfcKyZcswZcoUzJs3Dz179sRnn32GIUOG4NChQ2jcuHGV7f39/TFx4kS0b98e/v7+2Lp1K55++mn4+/vjqaeeUuUgiIiIyH3oJEmSlDyhe/fu6Ny5M+bPn294rHXr1hgxYgQSExNl7WPkyJHw9/fH119/LWv7goICBAUFIT8/H4GBgUqaa1GTl1eqti8iIiJXd3r2UFX3Z+v3t6JhneLiYqSmpiIhIcHo8YSEBKSkpMjaR1paGlJSUtC3b18lL01ERETVhKJhndzcXJSVlSE0NNTo8dDQUGRnZ1t8bqNGjXDx4kWUlpZi5syZeOKJJ8xuW1RUhKKiIsPvBQUFSppJRERELsymhFidTmf0uyRJVR6rLDk5Gbt378ann36KuXPnYsmSJWa3TUxMRFBQkOEnIiLClmYSERGRC1LUcxIcHAxPT88qvSQ5OTlVelMqi4qKAgC0a9cOFy5cwMyZM/Hwww+b3HbatGmYOnWq4feCggIGKERERNWEop4THx8fxMTEICkpyejxpKQkxMXFyd6PJElGwzaV+fr6IjAw0OiHiIiIqgfFU4mnTp2KsWPHokuXLoiNjcWCBQuQkZGBCRMmALjV63Hu3DksXrwYAPDJJ5+gcePGaNWqFYBbdU/effddTJo0ScXDICIiInehODgZNWoU8vLyMGvWLGRlZSE6OhqrVq1CZGQkACArKwsZGRmG7fV6PaZNm4ZTp07By8sLzZo1w+zZs/H000+rdxRERETkNhTXOdEC65wQERE5nkvWOSEiIiJyNAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidERESEu9o10LoJBgxOiIiICDGRdbVuggGDEyIiIhIKgxMiIiISSrUOToJr+WrdBCKiaiskgNdgMq1aBydRwTW1boJL6t08WOsmEJEbkLRuAAmrWgcnRJUNaB2qdROIiDTRrL6/1k0wYHBCVMH9MQ3RsHYNrZtBROR0fVvU17oJBgxOBDKyU0NNXtfHi2+Div45qAUAwN/HU+OWEBE5R2igL3Q6ndbNMKjW30p3hNTSuglG5ozqqMnrPtk7SpPXleO1YW2c/pr3dmqE7dP6Y/Z97Z3+2kREVM2Dk9BAP6PfB7ZxvXyDR+Oa2L0PPy9xewjCgvysb+SQ162B7lHiFCQiItfzr0EttW6CbDqI02sC2BiczJs3D1FRUfDz80NMTAySk5PNbrtixQoMHDgQ9evXR2BgIGJjY7F27VqbG+xI88d0xuT45lo3Q5EWoQFOf83mIc5/TS1wqjmRY0luNF1naLswjOoSoeg5nRvXdkxj3IDi4GTZsmWYMmUKpk+fjrS0NPTu3RtDhgxBRkaGye23bNmCgQMHYtWqVUhNTUW/fv0wfPhwpKWl2d14tXl5eqCpQNnKla16vneVxyQLk/Ei6jomsXNqQgs82TsKloYn749p5JDXJrJmXGykxb8H1/Kxa/98b5Mpn4zpjP/eL38oeETHcCx9Khbzx3RG7ZreDmyZPAKlmwCwITiZM2cOxo8fjyeeeAKtW7fG3LlzERERgfnz55vcfu7cuXjppZfQtWtXNG/eHP/5z3/QvHlz/Pbbb3Y3Xk22LHh0KvEuPBDTSJWhFTnq+vvg5SGtZG//f4Plb6tELV8vTB/aBh0a1Ta7zbsPdJC9v5jIOiq0ynY+nqY/Bh4eOrwxIhr1/P/+MqsvSNGokZ21SZ52BS9a7Uq37yoc3yrErufb4uDrg5z+miJ6uJuyngmRvT+qI3y8PDCkXRj2vDJQ6+YIR1FwUlxcjNTUVCQkJBg9npCQgJSUFFn70Ov1KCwsRN265sfzi4qKUFBQYPTjCBXzGeaNiVH8fJ1Oh3ce6ICZd7dVs1kmzbqnLRoE+WFC32aq71vrntXlz8RhSLR2q2EemmX+wj+2RySe7tvU8Pvgttqv2vnBQx2ROLKdU1/TWm+ELZoG+2P3KwNU36+jbwBr+no5+BWMNQ32h7+TX1NUHqLd3lvRqI753uuKM2NEOCzRJogoCk5yc3NRVlaG0FDjxNHQ0FBkZ2fL2sd7772Ha9eu4cEHHzS7TWJiIoKCggw/ERGOiZbv69wIj/Vsgs8f6eKQ/Sth7cv5no7ufadcw8y03XpOyPvwqtBzYusY+PP977DpeQ0C/fBYzyaKntMtqi58bUxi7tS4NpY/E6voOe0aBqFJsPbDndUxQXn3KwOw7oU+Dn8dV5wMIKryG8iXh7TCnS1N97KFBmrTA2spQfc9Bb3dzmBTQmzludCSJMmaH71kyRLMnDkTy5YtQ0iI+a7RadOmIT8/3/CTmZlpSzOt8vL0wIzhbVX9YDauq7wk/r7XEjBvTGfV2uCKKmeKzxvTGVMHtkDXJtoO+Tjajn/Ho3WDQEXPaRBYdQbTwse6ynruT8/2VLwsuqmPthZ5F+68bML+mQkmHw+u5WsUPDtKpA3XLS2ImqTev8JQ3/8Nbomt/9cPE/o2M9sj8kCM8Q13xZsiP2/Hne/n+t2BGcNNl2cIMXFd0ZKi/4Xg4GB4enpW6SXJycmp0ptS2bJlyzB+/Hh8//33GDDAcleur68vAgMDjX6cxd7qoB0jait+TlBNb1WK30Q3NP5/siVQUlOrBrbP6rmrXRiej28uVFEgi+xo5x2hyrpTTf2f9DNzh2aNo5KmZdHo1Grxlupg4brQuXFtBPo5PyHSGUmY1hf2U9ZVGSxIzldlFd9SOp0OjerYfu0VbUqvVhQFJz4+PoiJiUFSUpLR40lJSYiLizP7vCVLluDRRx/Fd999h6FDh9rWUifp0qQuXrcjh+Thbo3Nrs/yx4t9bd6vnAuqZ6WN2ltIWHWGXyb2RPJL/TRtgxoe7BKBuv4+eLCLY3oLOjeug09Gd8bvk3o5ZP+W8EKoTE0bqwa3bhCAfya0MPk3rfJJvDzUP/c/Tvh7yPDt+9vjXoVVry0l8VvKiejvgCTlbg4cRuzRtJ5D9vtcP/VzErWiuP9o6tSp+OKLL/C///0Phw8fxgsvvICMjAxMmDABwK0hmUceecSw/ZIlS/DII4/gvffeQ48ePZCdnY3s7Gzk5+erdxQqGxfXBHVsvKvw8tThHz0am/xbs/pVP1yD2qo3pGTqHmRY+zDV9q+Ur5cnIlyku9iS2jV9sGv6ALx9v+PGZIe2D0N0wyDD7/ZOd1WT0pwYJVytzsU7Zt4DcmbsWRuSGN9L3ErNcnVp8vcXeoQdvQem/Daxl9k3TB8XGfLbPq0/vn2iO3o5qL0T+jYzmyz/ytDWAIC7O4QDgM3fcc6iODgZNWoU5s6di1mzZqFjx47YsmULVq1ahcjIW9n8WVlZRjVPPvvsM5SWluK5555DWFiY4Wfy5MnqHYVA/H2U3QXVVLi9Uu+P6ohVz/fGsbeGmN3GmV8QH4/uhJGdG6JDoyDrGzvBW/dGy9rO08JdpitVgVTquye7495Ojukx+veQ1qrub3yvKFnJ7fb0F9Sp6Y2mJpKDh9++4Fti7WMW66C7aTlcYfTUXNK8KwkLqoGedzg2kHq4W2NDIFJucNsGeKL3rVmH9Wr54q83h2DPq2JPX7Yp8+bZZ5/F6dOnUVRUhNTUVPTp83c2+aJFi7Bp0ybD75s2bYIkSVV+Fi1aZG/bhTJjeBs83bcp2oRbzo9RUqdEDd6eHmgTHghvJyTVyTGsfTjmPNhRiMUG/9GjMcZ0t3+K7HP9bJup4wpM9fapZYDKM0ReHdYGA9uEOjxPKaJuTaM1n9ZP7at5rR57ifB5dGXhKq5k3jrM/grctSoNFXp5Gn8mfLw8hM/n4ztSoch6t7oqKxdte6xnFKbJuBOc0LcZjr452PB75bfHAxpXnzQ3Dq1l4uRPz8ahd/NgtFS5VH9QDbG7Naurn5/r6dD9q5FLUDEYEa0+hCsL8HOtei6PxjXBfZ0b4Z9mek99bQj6XhnWxuIwoZxZax4VruNRwf6Ydpe6vZTOwOBEgeiGgVg9uTfWTumDDx/qZPN+LNWoCFMxAlfihwmxaN8oCD9MUFYDwxHeuMc4IblT4zr4enx3tLBx9o+5IKSWr3OCk3fub48m9WrCy0OHz8bKK/Zny1DbT8+aT0q3V1iQ7e9LJdOOFz7W1eyMNzWGH18c2AJv3ausgJ0aVYE9LAwLVudy+JXPqSMSW00ZaiYXr+Jszb4t6hv9bUGlz+6TvaPw2rA2eO/BDmavMRWv9W3C5M06DfTztljYc8ZwZRM2Nv7zTrtnoWqBwYkCv0/qjZo+XmjZIMBi7QG7ustkXIHNzQayR9cmdfHrxF7o1Fib7umKawSNjW0i6zntGwXJuguedU/VD3PfFvUxLk69qqd1LSSXPdAlApv+1Q9H3xyCQSpWmH1xoPHsj06N66BPpQuqWuQmbv/3PudVrlX6pe6hAybFN1fUY1Y/wBcrnrEv6IuoW8PsgqIT+91hSFC0lU4HTHPycLEpcoO4ihWXK3NWBdiKBcc6VVh8r+KyG0/1+budc0d1REKFz26rBgGYPrSNxaCz3G8Te2F4h3DZNybWqPlf5IjvErUwOHFBalfyk9OVOvN2tD5RoPyKXyf2QhcrY/0tQmvhrnZhVdbk+OrxbqomIz/cvTGirFRRtZRUa4toE0nFix/vpuprlNPpdLLyKkZ1bYzTs+0vFyCnRk/FL5KK6x+Zs/Spqr2ClqbTjuzcENtf7m/3jLPkl/ojNNCvyhDuoLah+Oeglk4d+5e7DpjcGigPdY3Aew90wNN9mhpNIzZnwdgYi5nBWiyAV3G6c8W3g1p5eu0aBeGjhzs5deai3HeUyMn8DE60pnFO0sA2oRjd3fTU54riW4fi4OuDjMZWnZ1EZ0sl3zWT+zglGdjXyxNzHpQXNAqeh2bWx6M74aGuEVg9uerq2GpbO8V0yfa5txdLW/iocUVca/2Ncc3qmexlq2shqHm+f3OHVWd9uFsEPv2HOnfSSsidor73tQT879EuCLBSg2X2fe1xX0wjTLurNSLrVQ3OG9cz/kJOsNJz6Oft2Bk5tWt648tx5md0aTWzveI1wdo5crXp97ZicOJGbHnTfv5IF9kXhMrFohJHtnNqFdrh7cPw3ZPdMXtkO7PF3Sr3TpR3u7rjB9rPxvV1lKjYsxAWVAOz72uP1jLHzu1hbtroiE4NcXjWYPRTMTdh6kDTxdEqKh+zb29DBWhTdDqdJrMldDqdYbji1WGmy5iX698qFPtmmC6rL9eoLs5ZRVjuxzvt1YGIrzSUIScXxNGnSqfT4fdJvbD8mVjUrilOjSMtuVZqNMlmKqP7+fjm+H5XJu6LaYhPNp6w+zWa1a+FLS/1Q5OXV9q9Lzl0Oh3imgUjzkwRxPce6IDuTeui1383OqU9WuseVRdD24fhDgdM933mzmYoLdM7ZeFFpawNj5n6q73B6aZ/3YmSMr1hKLBpfe0XQiyng7I7/mlDWmFC32ao6++DyUv3WtxWTk6FJc5YF0gJUwFhSIAfNv/rTtTy9cKxnKsatOqWikUYnUWNZG9HYXCiscpF29S6wZ8yoGoC3tSBLfDCgOZYvP2MSq8ilvuq2awHDw8dPhntmAUjLZURd0WShU+WnK9fb08Po+HBAD9v7H5lgE1TRZ3tnfvb418/7jf8rtPpLA5nOVqrCnU8RndvjI82HNesLeXKh6TkBiei98SW9yy2sLJuV11/Hyx/Jtbhw2m2EP+TpRFHZzHPHtkOMZF1TAYR1vhWWLXS3IJh5rLendGVvPSpHg5/jXKumr9hq4e6Ku8mH2xlnD++VQjq1PQWosZEtybmZ19V7o5Xi63voeBavgjQYME+pR5QaWhl+l2tkaZCVdF7OjTE63e3xe+TepmdxaQWOddxS4FrZW/dG406Nb3xnsz8Mq0E1/LFgdcHYdXz1vPDYiLrom24GBW7K9L+aiSomXe3xQ+pZwHc6j5/3cR0VHs81K0xHupmPRHVFD9vTyx8rCv0egl9W9RHTGQdtA4LwL5MeesVlU83rVxFUC2OWtRKBEouZI4wpJ3ytZLiW4dgzcFss3//YlwXlOkl9H57IwpvltrTPJNCA+V3HZtbBO/VYW2sVl/WWmzTeth+Mk/Wto/JnDXjLJYWgPzgoY5Iy7iC8b2iZA3zNK3vj1O519DezBIVHh46jNP4+G0NSMd0j8Tobo2Fr64KOO767iyu3XoHqniRfObOZmjVQKwLY7+WfycELr9dh2H4R1sNj1m6kEcF+2Pr//VDnWqUeDWmeySW7sq0up0o1xxHh0A+nh4oLtMjJrIOdDpdlfLW1shNhA6u5YNn77zD7hwnRyVeq3m+v3q8Gz784xg+3mh6mKLiOW2ucrVjR7qnY0Pc01H+6sJJL/RFSZneIUMF9nwu1JrKr3Vg4ojVpEXEYR2B/TrR9jLelqrQAkCjOjU1W6pdC+0aBVn8gnuoawRaNQiwOgvE0h1mRd63v+xFTThbM6U3XhjQwmIlSksqFqiy5N0HOjjlfSZVSAIwNaTpjBwBHy8PvJjQAosf74Zn71S+dH0TK3VybNHKxqrK9vD00AmZwyDCsKUaIuu5/krvcjA4cYB2KmVdhwT4qbIfR/t9Ui883lPZcu+O7LUpn+paORmsjoUkwNn3tceaKX2sBnVyrZ7cB/d2aoglT6qbf6PWOi5N69fC5AHKqqVW5OftKWvJdTUXRJPLntVrK65qXFdmTZCKdDod+rSob9MspztCalWp32KPRY91xQoHLmmgtdkjnVeNWCRa99w4i3uEkoKp6++DndPjUdPHC9Ez1ip6rj13eH7e2sSa0Q2D4Omhw/+2nZL9nNfvaYuCmyWyK1Yq8fX4bli2K9Ppiyi2DguEp4cODQL9cEdILbw/qqOq+/99Ui+XWSNjyZM9cKHgJlo4afjC1gu2JBk/d2CbUKx6vjdKyvRmk81ltcfG56lZv6VdwyBVqyDbakDrUKw/fEHx8xY91hWPLtxl9u8PdWuMl1ekG35vExaIQ1kFVbYL8PVCYVEpYmQszdFasOF7pdSuQq0l7d+5ArsjpBaO51y1aTl0LXo9Zt/XHk8u3q1JifnWYYH46OFOsu+Uw4JqmCwnrobgWr54ToP/Az9vTxx8fZDDxoS1qINgq9hm8pKi5VYZduYlV/TEW62N6d4Y3/6ZgYdlJvQ/H3+HTcGJv68X+rWsj41HL1rd1tJN3W+TemH5nrN4TEbvblBNb6S+MkDIYSlravl6CRGMqsV9jsQB1k7pg+JSvV3dxM7UrH4tbHjxTs1ef7idC5g52oiO4diXeQXNHbjEvaMuanKGUFyRrcNKpJ2Zd7fFyM4N0b5RbYfs/1+DWuLkxWvoElkHzUMDZAUnljQJ9seLCfLXkBGx8KAcz9iQ5yQyBicWeHroNA1MKnbRuU9nnbo6K1hF+ZHYJmgRGoB2ZqY4kjYi69XEmbzrDn2NwdHqrQYtGmeXO/f29EBMpPXVwG1Vsdfz+fjmKLhRgqHtlU+hr256NHXcOdECgxOB1Q/wxaguEfDy1LlEsSct9G8Vgk//0VnWVG9PDx163lG1rL8rUJJT8caIaLz68wEHtsb1PBLbROsmOMyn/4hB6pnLsrZ1tVzKWr5emH1fe5ufb+54vTz+Hk6sXcO1Syr8+e94ZFy67tCAUQucrSO4/97fHm/d6/5Z6U/0ujU1dZjCOySdTofB0WEOmYbpqsb2iETnxrUd/jr2/J/LWQn7t4m9ZO+vtpXhIUuJguXvOWcsaOgI5kqUC15hXXVKjtfTQ4efno3Dsqd6IMjFh0xDA/3Q1UJlZVfFnhNyuPdHdcALy/bhlaGtzW7TJjwQB18fhJoukt8DAN892V312TPRDQNx4FwBhrcPw1dOXgOpcd2ayMq/KXv7jx7uhNmrj6BJPX+zhcdM+faJ7kYX08h6/iaHdZQMv3WLsv3iHFnPH2mvDnSbOhg9mtaFJAH1FKyf42o9KmropGBIWAsrn++FeRtPYGV6ltZN0YR7fBrdyCOxkVi47RTu7ih2cqkS93ZqhIQ2DawW43K1onBxzdQfIvpmfHckH8vFwDahTg9O3h/VEW+tOiy7Zk2jOjXx8ejOuFZUikUpp2UtPQ+gytDaO/e3x39WHbZr6MXeL1dLNXBcTXltHVHqYTijAJ4gh6qqtuFBmDygOYMTEkNIoB/2zUgQbqlxe7la4OFIlpa4r13TR5VZTw93a4w9GVfQtYn8u8Pw2jVsWuXY39cLe14daKiKq1RooB8+eKiTTc+lqkQJStwJ/0+dz72+Ad2E1oHJXe1uzWyYfldrdIyojUWPqVe1kpzj/phGWPl8L3w9vrtTXs/Hy8OhF/DWCmuP/D5Jfr6KsyhZANFROliY/vv27cTTN0dEO6k1RObxdpaq+Pjhzsi9uwghAX54UuYaKuRYY2QkkFak0+mEXAZdqZ3T41Fwo0Rxbk90wyCMi420OjTmqHiqjn/VJMt+LUMwOb452qpc5M1bxs3Mhhf74ljOVYuz1R7sGoFhHcJUL+Sl5qjOkOgwvPbLQXSvlGPkjKEjLYQFucYSJo7A4ISq8PDQucy6Pu6scd2ayLh0HbFN62FyfHOtm6OJkAA/me9Fsbrdh7cPx/YTeUaJvzqdDi8MbKHaa/z3vnbw0OlkDZk2rV8LTetbLz4oeoXR+gG+ODxrMHxlVhZ2dQF+3kh+qR9e/eUANtlZjM7ViP1OJKrGfn6uJ3aeuoT41iGaD/WpxZHVeUXi5emBt+/v4NDXGNVVWW+au3CVit1qiahbE6HV8GbRPa54RG6orr8PBkc3kNVt7yqYVygmdzgt9QO0z+kh9bjPVY+IyI29+4Bje2Jc3eyR7dC7eTAT+N0Eh3WIiFzA/TGNUKemN8Z/tRt3V5puLlW7erBVhdeu4bTZaeR4DE6IyGU1re+PkxevVfmydlfxrUOxc3o86rvoyrmupkNEbezLvIK+zetr3ZRqh8EJEbmsVc/3xrkrN9BMxkwUd8GZdM7z0zNxKC7Tw8+7eiXhioDBCRG5LD9vz2oVmJBzeXjo4OfBwEQLTIglIod7bVgb1PD2xDsOnl5L4rE2E+jOlreGTL4e383m13D3jJtn+zWDt6cOj8Y10bopTsOeEwcLquGN/BslNq87QmQvEdYFebxXFMbFNYGnh/ZtIeeKbhiErk3qINxMld8vx3XFxcIiNKjG1VCtiaznj8OzBrtNvSM5qs+RamTJkz3Qp0V9rHimp9ZNIdKUSIHJqC4RAIAXBqhXsZVM8/TQ4YcJcWYXd/T00DEwkaE6BSYAe04crk14IBY/bnt3JRGpb/Z97TAp/g40qlNT66YQkQnVKxQjIsKtoS4GJkTiYnBC5GRfPd4N/j6e+OChjlo3hYhISBzWIXKy3s3rI33mIHgIlINB5E5ahwXicFYB7uvcSOumkI0YnBBpgIEJqUly97m0Cv30bBxO511DqwaBWjeFbMRhHSIicit+3p4MTFwcgxMiInJpEruO3A6DEyKiao4zl0g0zDkhIrfEL1zrFj/eDdtP5uG+GCaOklgYnBCRW3okLhLn82+gX8sQrZsirD4t6qNPi/paN4OoCg7rECkw4c5mqOHtWa0W4HJVvl6emDG8Lb98iVwQe06IFGhYuwbSZyZUu3UuiIiciVdYIoUYmBARORavskRERCQUBidEREQkFAYnRETk0vy8PbVuglP5ern/V7f7HyERkQtZ9lQPNA32x7dPdNe6KcJ7eUgrxDWrh/urWZ2Wb5/ojoa1a+DzR7po3RSH4WwdIiKBdG9aDxv+eafWzXAJE/o2w4S+zbRuhtN1aVIX217ur3UzHIo9J0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEREQkFAYnREREJBQGJ0RERCQUBidEbq5OTW+tm0BEpAiLsBG5uX4tQ/BoXBO0DQ/UuinkIK0aBGjdBCJV6SRJkrRuhDUFBQUICgpCfn4+AgN5gSUiqkiSJPyYehZtwgPRNjxI6+YQGdj6/c2eEyIiF6fT6fBAlwitm0GkGuacEBERkVAYnBAREZFQGJwQERGRUBicEBERkVBsCk7mzZuHqKgo+Pn5ISYmBsnJyWa3zcrKwujRo9GyZUt4eHhgypQptraViIiIqgHFwcmyZcswZcoUTJ8+HWlpaejduzeGDBmCjIwMk9sXFRWhfv36mD59Ojp06GB3g4mIiMi9Ka5z0r17d3Tu3Bnz5883PNa6dWuMGDECiYmJFp975513omPHjpg7d66iRrLOCRERkeux9ftbUc9JcXExUlNTkZCQYPR4QkICUlJSlOzKoqKiIhQUFBj9EBERUfWgKDjJzc1FWVkZQkNDjR4PDQ1Fdna2ao1KTExEUFCQ4ScigsWFiIiIqgubEmJ1Op3R75IkVXnMHtOmTUN+fr7hJzMzU7V9ExERkdgUla8PDg6Gp6dnlV6SnJycKr0p9vD19YWvr69q+yMiIiLXoajnxMfHBzExMUhKSjJ6PCkpCXFxcao2jIiIiKonxQv/TZ06FWPHjkWXLl0QGxuLBQsWICMjAxMmTABwa0jm3LlzWLx4seE5e/fuBQBcvXoVFy9exN69e+Hj44M2bdqocxRERETkNhQHJ6NGjUJeXh5mzZqFrKwsREdHY9WqVYiMjARwq+ha5ZonnTp1Mvw7NTUV3333HSIjI3H69GlZr1k+25mzdoiIiFxH+fe2wqolyuucaOHs2bOcsUNEROSiMjMz0ahRI9nbu0Rwotfrcf78eQQEBKg6K6igoAARERHIzMx02+Ju7n6MPD7X5+7H6O7HB7j/MfL4bCdJEgoLCxEeHg4PD/lproqHdbTg4eGhKOJSKjAw0C3fcBW5+zHy+Fyfux+jux8f4P7HyOOzTVBQkOLncFViIiIiEgqDEyIiIhJKtQ5OfH19MWPGDLcu+Obux8jjc33ufozufnyA+x8jj8/5XCIhloiIiKqPat1zQkREROJhcEJERERCYXBCREREQmFwQkREREKp1sHJvHnzEBUVBT8/P8TExCA5OVnrJmHmzJnQ6XRGPw0aNDD8XZIkzJw5E+Hh4ahRowbuvPNOHDx40GgfRUVFmDRpEoKDg+Hv74+7774bZ8+eNdrm8uXLGDt2LIKCghAUFISxY8fiypUrRttkZGRg+PDh8Pf3R3BwMJ5//nkUFxcrOp4tW7Zg+PDhCA8Ph06nw88//2z0d9GOJz09HX379kWNGjXQsGFDzJo1y+KaENaO79FHH61yPnv06OEyx5eYmIiuXbsiICAAISEhGDFiBI4ePWq0jaufQznH6Mrncf78+Wjfvr2hwFZsbCxWr15t+Lurnz9rx+fK586UxMRE6HQ6TJkyxfCYq59Dk6RqaunSpZK3t7f0+eefS4cOHZImT54s+fv7S2fOnNG0XTNmzJDatm0rZWVlGX5ycnIMf589e7YUEBAgLV++XEpPT5dGjRolhYWFSQUFBYZtJkyYIDVs2FBKSkqS9uzZI/Xr10/q0KGDVFpaathm8ODBUnR0tJSSkiKlpKRI0dHR0rBhwwx/Ly0tlaKjo6V+/fpJe/bskZKSkqTw8HBp4sSJio5n1apV0vTp06Xly5dLAKSffvrJ6O8iHU9+fr4UGhoqPfTQQ1J6erq0fPlyKSAgQHr33XdtPr5x48ZJgwcPNjqfeXl5RtuIfHyDBg2SFi5cKB04cEDau3evNHToUKlx48bS1atX3eYcyjlGVz6Pv/76q7Ry5Urp6NGj0tGjR6V///vfkre3t3TgwAG3OH/Wjs+Vz11lO3fulJo0aSK1b99emjx5suFxVz+HplTb4KRbt27ShAkTjB5r1aqV9PLLL2vUoltmzJghdejQweTf9Hq91KBBA2n27NmGx27evCkFBQVJn376qSRJknTlyhXJ29tbWrp0qWGbc+fOSR4eHtKaNWskSZKkQ4cOSQCkHTt2GLbZvn27BEA6cuSIJEm3vnQ9PDykc+fOGbZZsmSJ5OvrK+Xn59t0bJW/vEU7nnnz5klBQUHSzZs3DdskJiZK4eHhkl6vV3x8knTrwnjPPfeYfY4rHZ8kSVJOTo4EQNq8ebMkSe53Dk0doyS533msU6eO9MUXX7jl+at4fJLkPueusLBQat68uZSUlCT17dvXEJy46zmslsM6xcXFSE1NRUJCgtHjCQkJSElJ0ahVfzt27BjCw8MRFRWFhx56CCdPngQAnDp1CtnZ2Ubt9vX1Rd++fQ3tTk1NRUlJidE24eHhiI6ONmyzfft2BAUFoXv37oZtevTogaCgIKNtoqOjER4ebthm0KBBKCoqQmpqqirHKdrxbN++HX379jUqRDRo0CCcP38ep0+ftvk4N23ahJCQELRo0QJPPvkkcnJyDH9ztePLz88HANStWxeAe57DysdYzh3OY1lZGZYuXYpr164hNjbW7c5f5eMr5w7n7rnnnsPQoUMxYMAAo8fd7RyWq5bBSW5uLsrKyhAaGmr0eGhoKLKzszVq1S3du3fH4sWLsXbtWnz++efIzs5GXFwc8vLyDG2z1O7s7Gz4+PigTp06FrcJCQmp8tohISFG21R+nTp16sDHx0e1/yPRjsfUNuW/23rMQ4YMwbfffosNGzbgvffew65du9C/f38UFRW53PFJkoSpU6eiV69eiI6ONnqeu5xDU8cIuP55TE9PR61ateDr64sJEybgp59+Qps2bdzm/Jk7PsD1zx0ALF26FHv27EFiYmKVv7nLOazMJVYldhSdTmf0uyRJVR5ztiFDhhj+3a5dO8TGxqJZs2b46quvDElctrS78jamtrdlGzWIdDym2mLuuXKMGjXK8O/o6Gh06dIFkZGRWLlyJUaOHGn2eSIe38SJE7F//35s3bq1yt/c5RyaO0ZXP48tW7bE3r17ceXKFSxfvhzjxo3D5s2bLe7Plc6fueNr06aNy5+7zMxMTJ48GevWrYOfn5/Z9rr6OaysWvacBAcHw9PTs0oUl5OTUyXi05q/vz/atWuHY8eOGWbtWGp3gwYNUFxcjMuXL1vc5sKFC1Ve6+LFi0bbVH6dy5cvo6SkRLX/I9GOx9Q25d2/ah1zWFgYIiMjcezYMcNrusLxTZo0Cb/++is2btyIRo0aGR53p3No7hhNcbXz6OPjgzvuuANdunRBYmIiOnTogA8++MBtzp+54zPF1c5damoqcnJyEBMTAy8vL3h5eWHz5s348MMP4eXlZbZXwtXOYWXVMjjx8fFBTEwMkpKSjB5PSkpCXFycRq0yraioCIcPH0ZYWBiioqLQoEEDo3YXFxdj8+bNhnbHxMTA29vbaJusrCwcOHDAsE1sbCzy8/Oxc+dOwzZ//vkn8vPzjbY5cOAAsrKyDNusW7cOvr6+iImJUeXYRDue2NhYbNmyxWha3Lp16xAeHo4mTZqocsx5eXnIzMxEWFiYSxyfJEmYOHEiVqxYgQ0bNiAqKsro7+5wDq0doymudh5NHXNRUZFbnD9Lx2eKq527+Ph4pKenY+/evYafLl26YMyYMdi7dy+aNm3qluew2s7WKZ9K/OWXX0qHDh2SpkyZIvn7+0unT5/WtF0vvviitGnTJunkyZPSjh07pGHDhkkBAQGGds2ePVsKCgqSVqxYIaWnp0sPP/ywySljjRo1ktavXy/t2bNH6t+/v8kpY+3bt5e2b98ubd++XWrXrp3JKWPx8fHSnj17pPXr10uNGjVSPJW4sLBQSktLk9LS0iQA0pw5c6S0tDTDlG2RjufKlStSaGio9PDDD0vp6enSihUrpMDAQItT4CwdX2FhofTiiy9KKSkp0qlTp6SNGzdKsbGxUsOGDV3m+J555hkpKChI2rRpk9FUzOvXrxu2cfVzaO0YXf08Tps2TdqyZYt06tQpaf/+/dK///1vycPDQ1q3bp1bnD9Lx+fq586cirN1JMn1z6Ep1TY4kSRJ+uSTT6TIyEjJx8dH6ty5s9HUQa2Uz0/39vaWwsPDpZEjR0oHDx40/F2v10szZsyQGjRoIPn6+kp9+vSR0tPTjfZx48YNaeLEiVLdunWlGjVqSMOGDZMyMjKMtsnLy5PGjBkjBQQESAEBAdKYMWOky5cvG21z5swZaejQoVKNGjWkunXrShMnTjSaHibHxo0bJQBVfsaNGyfk8ezfv1/q3bu35OvrKzVo0ECaOXOmxelvlo7v+vXrUkJCglS/fn3J29tbaty4sTRu3LgqbRf5+EwdGwBp4cKFhm1c/RxaO0ZXP4+PP/644TpXv359KT4+3hCYSJLrnz9Lx+fq586cysGJq59DU3SSpLRsGxEREZHjVMucEyIiIhIXgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEgqDEyIiIhIKgxMiIiISCoMTIiIiEsr/A8XHnvieecYOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "stepi = []\n",
    "lossi = []\n",
    "for i in range(400000): # overfit 32 ;p of 5 words \n",
    "    \n",
    "    ix = torch.randint(0, Xtr.shape[0], (32,))\n",
    "    \n",
    "    emb = C[Xtr[ix]] # 32, 3, 2\n",
    "    h = torch.relu((emb.view(-1, 15) @ W1) + b1) # 32, 100\n",
    "    logits = (h @ W2) + b2 # 32, 27 \n",
    "    loss = F.cross_entropy(logits, Ytr[ix])\n",
    "    for p in parameters:\n",
    "        p.grad = None \n",
    "    \n",
    "    loss.backward()\n",
    "    lr = 0.01\n",
    "    for p in parameters:\n",
    "        p.data += -lr * p.grad\n",
    "        \n",
    "    lossi.append(loss.log10().item()) # log10 squashes \n",
    "    stepi.append(i)\n",
    "print(\"TRAIN LOSS: \" + str(loss.item()))\n",
    "plt.plot(stepi, lossi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "89e6f7c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.1852, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb = C[Xtr] # 32, 3, 2\n",
    "h = torch.relu((emb.view(-1, 15) @ W1) + b1) # 32, 100\n",
    "logits = (h @ W2) + b2 # 32, 27 \n",
    "loss = F.cross_entropy(logits, Ytr)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "75dab567",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.2040, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb = C[Xdev] # 32, 3, 2\n",
    "h = torch.relu((emb.view(-1, 15) @ W1) + b1) # 32, 100\n",
    "logits = (h @ W2) + b2 # 32, 27 \n",
    "loss = F.cross_entropy(logits, Ydev)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "1e0fbe76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'plt.figure(figsize=(8,8))\\nplt.scatter(C[:, 0].data, C[:,1].data,s=200)\\nfor i in range(C.shape[0]):\\n    plt.text(C[i,0].item(), C[i,1].item(), itos[i], ha=\"center\", va=\"center\", color=\\'white\\')\\n    \\nplt.grid(\\'minor\\')'"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"plt.figure(figsize=(8,8))\n",
    "plt.scatter(C[:, 0].data, C[:,1].data,s=200)\n",
    "for i in range(C.shape[0]):\n",
    "    plt.text(C[i,0].item(), C[i,1].item(), itos[i], ha=\"center\", va=\"center\", color='white')\n",
    "    \n",
    "plt.grid('minor')\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "bf7ac539",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "carma.\n",
      "famorilvi.\n",
      "kimlex.\n",
      "taty.\n",
      "ska.\n",
      "sane.\n",
      "mahnee.\n",
      "den.\n",
      "arci.\n",
      "aqui.\n",
      "ner.\n",
      "kiah.\n",
      "maiivon.\n",
      "leigph.\n",
      "bro.\n",
      "kin.\n",
      "quinn.\n",
      "shor.\n",
      "marian.\n",
      "quisorro.\n"
     ]
    }
   ],
   "source": [
    "g = torch.Generator().manual_seed(2147483647 + 10)\n",
    "\n",
    "\n",
    "for _ in range(20):\n",
    "    \n",
    "    out = []\n",
    "    context = [0] * block_size\n",
    "    \n",
    "    while True:\n",
    "                \n",
    "        emb = C[torch.tensor([context])] # 32, 3, 2\n",
    "        h = torch.relu((emb.view(-1, 15) @ W1) + b1) # 32, 100\n",
    "        logits = (h @ W2) + b2 # 32, 27 \n",
    "        probs = F.softmax(logits, dim=1)\n",
    "        ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
    "        context = context[1:] + [ix]\n",
    "        out.append(ix)\n",
    "        if ix == 0:\n",
    "            break\n",
    "    print(''.join(itos[i] for i in out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e748803",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}