makemore/part5_wavenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import math"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1- Starter Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']\n",
      "32033\n"
     ]
    }
   ],
   "source": [
    "words = open(\"../data/names.txt\", \"r\").read().splitlines()\n",
    "print(words[:8])\n",
    "print(len(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "chars = sorted(list(set(\"\".join(words))))\n",
    "stoi = {s: i + 1 for i, s in enumerate(chars)}\n",
    "stoi[\".\"] = 0\n",
    "itos = {i: s for s, i in stoi.items()}\n",
    "vocab_size = len(stoi)\n",
    "\n",
    "print(itos)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182625, 3]) torch.Size([182625])\n",
      "torch.Size([22655, 3]) torch.Size([22655])\n",
      "torch.Size([22866, 3]) torch.Size([22866])\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "block_size = 3\n",
    "\n",
    "\n",
    "def build_dataset(words):\n",
    "\n",
    "    X, Y = [], []\n",
    "\n",
    "    for w in words:\n",
    "\n",
    "        context = [0] * block_size\n",
    "        for ch in w + '.':\n",
    "\n",
    "            ix = stoi[ch]\n",
    "\n",
    "            X.append(context)\n",
    "            Y.append(ix)\n",
    "\n",
    "            context = context[1:] + [ix]\n",
    "\n",
    "    X = torch.tensor(X)\n",
    "    Y = torch.tensor(Y)\n",
    "    print(X.shape, Y.shape)\n",
    "    return X, Y\n",
    "\n",
    "\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "\n",
    "Xtr, Ytr = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xte, Yte = build_dataset(words[n2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... ----> y\n",
      "..y ----> u\n",
      ".yu ----> h\n",
      "yuh ----> e\n",
      "uhe ----> n\n",
      "hen ----> g\n",
      "eng ----> .\n",
      "... ----> d\n",
      "..d ----> i\n",
      ".di ----> o\n",
      "dio ----> n\n",
      "ion ----> d\n",
      "ond ----> r\n",
      "ndr ----> e\n",
      "dre ----> .\n"
     ]
    }
   ],
   "source": [
    "for x, y in zip(Xtr[:15], Ytr[:15]):\n",
    "    print(\"\".join(itos[ix.item()] for ix in x), \"---->\", itos[y.item()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Layers made in part 3\n",
    "class Linear:\n",
    "    def __init__(self, fan_in, fan_out, bias=True):\n",
    "        self.weight = torch.randn((fan_in, fan_out))\n",
    "        self.weight /= fan_in ** 0.5\n",
    "        self.bias = torch.zeros((fan_out)) if bias else None\n",
    "\n",
    "    def __call__(self, x):\n",
    "        self.out = x @ self.weight\n",
    "        if self.bias is not None:\n",
    "            self.out += self.bias\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.weight] + ([] if self.bias is None else [self.bias])\n",
    "\n",
    "\n",
    "class BatchNorm1d:\n",
    "    def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "        self.eps = eps\n",
    "        self.momentum = momentum\n",
    "        self.training = True\n",
    "\n",
    "        # parameters (trained with backprop)\n",
    "        self.gamma = torch.ones(dim)\n",
    "        self.beta = torch.zeros(dim)\n",
    "\n",
    "        # buffers (trained while running `momentum update`)\n",
    "        self.running_mean = torch.zeros(dim)\n",
    "        self.running_var = torch.ones(dim)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        if self.training:\n",
    "            # batch mean\n",
    "            xmean = x.mean(0, keepdim=True)\n",
    "            # batch variance\n",
    "            xvar = x.var(0, keepdim=True)\n",
    "        else:\n",
    "            xmean = self.running_mean\n",
    "            xvar = self.running_var\n",
    "\n",
    "        xhat = (x - xmean) / torch.sqrt(xvar + self.eps)\n",
    "        self.out = self.gamma * xhat + self.beta\n",
    "\n",
    "        # update the buffers in training\n",
    "        if self.training:\n",
    "            with torch.no_grad():\n",
    "                self.running_mean = (1 - self.momentum) * \\\n",
    "                    self.running_mean + self.momentum * xmean\n",
    "                self.running_var = (1 - self.momentum) * \\\n",
    "                    self.running_var + self.momentum * xvar\n",
    "\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.gamma, self.beta]\n",
    "\n",
    "\n",
    "class Tanh:\n",
    "    def __call__(self, x):\n",
    "        self.out = torch.tanh(x)\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f8680f191b0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num parameters: 12097\n"
     ]
    }
   ],
   "source": [
    "n_embd = 10\n",
    "n_hidden = 200\n",
    "\n",
    "C = torch.randn((vocab_size, n_embd))\n",
    "layers = [\n",
    "    Linear(n_embd * block_size, n_hidden, bias=False),\n",
    "    BatchNorm1d(n_hidden),\n",
    "    Tanh(),\n",
    "    Linear(n_hidden, vocab_size)\n",
    "]\n",
    "\n",
    "with torch.no_grad():\n",
    "    layers[-1].weight *= 0.1\n",
    "\n",
    "parameters = [C] + [p for l in layers for p in l.parameters()]\n",
    "print(f\"num parameters: {sum(p.numel() for p in parameters)}\")\n",
    "\n",
    "for p in parameters:\n",
    "    p.requires_grad_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0 loss 3.2966105937957764\n"
     ]
    }
   ],
   "source": [
    "# same optimization as last time\n",
    "max_steps = 200000\n",
    "batch_size = 32\n",
    "lossi = []\n",
    "\n",
    "for i in range(max_steps):\n",
    "    ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n",
    "    Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "\n",
    "    # forward pass\n",
    "    emb = C[Xb]  # embed characters into vector space\n",
    "    x = emb.view((emb.shape[0], -1))  # flatten\n",
    "    for layer in layers:\n",
    "        x = layer(x)\n",
    "    # compute loss\n",
    "    loss = F.cross_entropy(x, Yb)\n",
    "\n",
    "    # backward pass\n",
    "    for layer in layers:\n",
    "        layer.out.retain_grad()\n",
    "    for p in parameters:\n",
    "        p.grad = None\n",
    "    loss.backward()\n",
    "\n",
    "    # update\n",
    "    lr = 0.1 if i < 10000 else 0.01\n",
    "    for p in parameters:\n",
    "        p.data -= lr * p.grad\n",
    "\n",
    "    # track stats\n",
    "    if i % 10000 == 0:\n",
    "        print(f\"step {i} loss {loss.item()}\")\n",
    "\n",
    "    lossi.append(loss.item())\n",
    "    break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Fixing the Learning Rate Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(torch.tensor(lossi).view(-1, 1000).mean(1, keepdim=True).data)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2- Pytorchifying our code\n",
    "add `Embedding`, `Flatten` and `Sequential` Classes"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Classes Definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Layers made in part 3\n",
    "class Linear:\n",
    "    def __init__(self, fan_in, fan_out, bias=True):\n",
    "        self.weight = torch.randn((fan_in, fan_out))\n",
    "        self.weight /= fan_in ** 0.5\n",
    "        self.bias = torch.zeros((fan_out)) if bias else None\n",
    "\n",
    "    def __call__(self, x):\n",
    "        self.out = x @ self.weight\n",
    "        if self.bias is not None:\n",
    "            self.out += self.bias\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.weight] + ([] if self.bias is None else [self.bias])\n",
    "\n",
    "\n",
    "class BatchNorm1d:\n",
    "    def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "        self.eps = eps\n",
    "        self.momentum = momentum\n",
    "        self.training = True\n",
    "\n",
    "        # parameters (trained with backprop)\n",
    "        self.gamma = torch.ones(dim)\n",
    "        self.beta = torch.zeros(dim)\n",
    "\n",
    "        # buffers (trained while running `momentum update`)\n",
    "        self.running_mean = torch.zeros(dim)\n",
    "        self.running_var = torch.ones(dim)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        if self.training:\n",
    "            # batch mean\n",
    "            xmean = x.mean(0, keepdim=True)\n",
    "            # batch variance\n",
    "            xvar = x.var(0, keepdim=True)\n",
    "        else:\n",
    "            xmean = self.running_mean\n",
    "            xvar = self.running_var\n",
    "\n",
    "        xhat = (x - xmean) / torch.sqrt(xvar + self.eps)\n",
    "        self.out = self.gamma * xhat + self.beta\n",
    "\n",
    "        # update the buffers in training\n",
    "        if self.training:\n",
    "            with torch.no_grad():\n",
    "                self.running_mean = (1 - self.momentum) * \\\n",
    "                    self.running_mean + self.momentum * xmean\n",
    "                self.running_var = (1 - self.momentum) * \\\n",
    "                    self.running_var + self.momentum * xvar\n",
    "\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.gamma, self.beta]\n",
    "\n",
    "\n",
    "class Tanh:\n",
    "    def __call__(self, x):\n",
    "        self.out = torch.tanh(x)\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []\n",
    "\n",
    "# ---------------- new ----------------\n",
    "\n",
    "\n",
    "class Embedding:\n",
    "    def __init__(self, num_embeddings, embedding_dim):\n",
    "        self.weight = torch.randn((num_embeddings, embedding_dim))\n",
    "\n",
    "    def __call__(self, x):\n",
    "        self.out = self.weight[x]\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.weight]\n",
    "\n",
    "\n",
    "class Flatten:\n",
    "    def __call__(self, x):\n",
    "        self.out = x.view((x.shape[0], -1))\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []\n",
    "\n",
    "\n",
    "class Sequential:\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "\n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "\n",
    "        self.out = x\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [p for l in self.layers for p in l.parameters()]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Initialize the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num parameters: 12097\n"
     ]
    }
   ],
   "source": [
    "n_embd = 10\n",
    "n_hidden = 200\n",
    "\n",
    "model = Sequential([\n",
    "    Embedding(vocab_size, n_embd),\n",
    "    Flatten(),\n",
    "\n",
    "    Linear(n_embd * block_size, n_hidden, bias=False),\n",
    "    BatchNorm1d(n_hidden),\n",
    "    Tanh(),\n",
    "\n",
    "    Linear(n_hidden, vocab_size)\n",
    "])\n",
    "\n",
    "with torch.no_grad():\n",
    "    layers[-1].weight *= 0.1\n",
    "\n",
    "parameters = model.parameters()\n",
    "print(f\"num parameters: {sum(p.numel() for p in parameters)}\")\n",
    "for p in parameters:\n",
    "    p.requires_grad_()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0 loss 3.631577253341675\n"
     ]
    }
   ],
   "source": [
    "# same optimization as last time\n",
    "max_steps = 200000\n",
    "batch_size = 32\n",
    "lossi = []\n",
    "update_to_data_ratio = []\n",
    "\n",
    "for i in range(max_steps):\n",
    "    ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n",
    "    Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "\n",
    "    # forward pass is now simpler\n",
    "    logits = model(Xb)\n",
    "    loss = F.cross_entropy(logits, Yb)\n",
    "\n",
    "    for p in parameters:\n",
    "        p.grad = None\n",
    "    loss.backward()\n",
    "\n",
    "    # update\n",
    "    lr = 0.1 if i < 10000 else 0.01\n",
    "    for p in parameters:\n",
    "        p.data -= lr * p.grad\n",
    "\n",
    "    # track stats\n",
    "    if i % 10000 == 0:\n",
    "        print(f\"step {i} loss {loss.item()}\")\n",
    "\n",
    "    lossi.append(loss.item())\n",
    "\n",
    "    break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.4 Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layer in model.layers:\n",
    "    layer.training = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 3.451620578765869\n",
      "valid 3.4509992599487305\n"
     ]
    }
   ],
   "source": [
    "@torch.no_grad()\n",
    "def split_loss(split):\n",
    "    x, y = {\n",
    "        \"train\": (Xtr, Ytr),\n",
    "        \"valid\": (Xdev, Ydev),\n",
    "        \"test\": (Xte, Yte)\n",
    "    }[split]\n",
    "    logits = model(x)\n",
    "    loss = F.cross_entropy(logits, y)\n",
    "    return loss.item()\n",
    "\n",
    "\n",
    "print(\"train\", split_loss(\"train\"))\n",
    "print(\"valid\", split_loss(\"valid\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.5- Sample from the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oezwcqorwedcve\n",
      "xmin\n",
      "ejtzsgxiycvzkoycihmo\n",
      "zuzjwfhcspltyxcnrzifjkdaiymwjgbzcg\n",
      "cudt\n",
      "ssvlpzyuxedxxjpvdzjtrvwgwdcdjduoexe\n",
      "owp\n",
      "rnwnj\n",
      "fl\n",
      "qfouarko\n",
      "e\n",
      "mvomiknkdqfxlzya\n",
      "erlllyoohutgyisdomggurjbzbanruvtccvdwij\n",
      "slssfsqazlhwnkvggwkouuqxvwdgnosyjcgatowdsvqzwdfvwapsusxpkzkszqvlzcwdtitdfmiobkblwfrpsbvigvqawilrngbyqbddjexivtsifoucphhp\n",
      "zbrzqf\n",
      "pajwmybc\n",
      "cvxajrjbqgkwsjhvpcrdojhqcgl\n",
      "cfenecniyafpejhgzkqhltpchnsnyxqjewigdqiwdyouzxn\n",
      "fjehbkvopkjtneueuuaid\n",
      "uyxxovejhylbczebrpjviigagewuuhvjepahdajzlduxfyexjrlrpfbpiazcnersmabjaecoegmisoxobkpxopoddtbn\n"
     ]
    }
   ],
   "source": [
    "# sampling from the model\n",
    "g = torch.Generator().manual_seed(2147483647 + 10)\n",
    "\n",
    "for _ in range(20):\n",
    "    out = []\n",
    "    context = [0] * block_size\n",
    "    while True:\n",
    "        # Forward pass\n",
    "        logits = model(torch.tensor([context]))\n",
    "        probs = F.softmax(logits, dim=1)\n",
    "\n",
    "        ix = torch.multinomial(probs, num_samples=1).item()\n",
    "\n",
    "        # Shift the Context Window\n",
    "        context = context[1:] + [ix]\n",
    "\n",
    "        if ix == 0:\n",
    "            break\n",
    "\n",
    "        out.append(ix)\n",
    "\n",
    "    print(\"\".join(itos[i] for i in out))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3- Building the WaveNet Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![dilated_casual_conv](../images/dilated_casual_conv.png)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Changing Dataset blocksize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182580, 8]) torch.Size([182580])\n",
      "torch.Size([22767, 8]) torch.Size([22767])\n",
      "torch.Size([22799, 8]) torch.Size([22799])\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "block_size = 8\n",
    "\n",
    "\n",
    "def build_dataset(words):\n",
    "\n",
    "    X, Y = [], []\n",
    "\n",
    "    for w in words:\n",
    "\n",
    "        context = [0] * block_size\n",
    "        for ch in w + '.':\n",
    "\n",
    "            ix = stoi[ch]\n",
    "\n",
    "            X.append(context)\n",
    "            Y.append(ix)\n",
    "\n",
    "            context = context[1:] + [ix]\n",
    "\n",
    "    X = torch.tensor(X)\n",
    "    Y = torch.tensor(Y)\n",
    "    print(X.shape, Y.shape)\n",
    "    return X, Y\n",
    "\n",
    "\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "\n",
    "Xtr, Ytr = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xte, Yte = build_dataset(words[n2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........ ----> e\n",
      ".......e ----> b\n",
      "......eb ----> r\n",
      ".....ebr ----> i\n",
      "....ebri ----> m\n",
      "...ebrim ----> a\n",
      "..ebrima ----> .\n",
      "........ ----> h\n",
      ".......h ----> i\n",
      "......hi ----> l\n",
      ".....hil ----> t\n",
      "....hilt ----> o\n",
      "...hilto ----> n\n",
      "..hilton ----> .\n",
      "........ ----> j\n"
     ]
    }
   ],
   "source": [
    "for x, y in zip(Xtr[:15], Ytr[:15]):\n",
    "    print(\"\".join(itos[ix.item()] for ix in x), \"---->\", itos[y.item()])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Initializing a normal network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num parameters: 22097\n"
     ]
    }
   ],
   "source": [
    "n_embd = 10\n",
    "n_hidden = 200\n",
    "\n",
    "model = Sequential([\n",
    "    Embedding(vocab_size, n_embd),\n",
    "    Flatten(),\n",
    "\n",
    "    Linear(n_embd * block_size, n_hidden, bias=False),\n",
    "    BatchNorm1d(n_hidden),\n",
    "    Tanh(),\n",
    "\n",
    "    Linear(n_hidden, vocab_size)\n",
    "])\n",
    "\n",
    "with torch.no_grad():\n",
    "    layers[-1].weight *= 0.1\n",
    "\n",
    "parameters = model.parameters()\n",
    "print(f\"num parameters: {sum(p.numel() for p in parameters)}\")\n",
    "for p in parameters:\n",
    "    p.requires_grad_()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Implementing WaveNet"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.1 Shape Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 8])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0,  0,  0,  0,  0,  0, 20],\n",
       "        [ 0,  0,  0,  0,  0,  0,  0,  8],\n",
       "        [ 0,  0,  0,  0,  0,  0,  0,  1],\n",
       "        [ 0,  0,  0,  0, 18,  1, 17, 21],\n",
       "        [ 0,  0,  0,  0,  0,  0,  0,  0]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# look at batch of 5 examples (it's 4 in the original video but I changed it to 5 to prevent confusion)\n",
    "ix = torch.randint(0, Xtr.shape[0], (5,))\n",
    "Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "\n",
    "logits = model(Xb)\n",
    "print(Xb.shape)\n",
    "Xb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding has output size of: torch.Size([5, 8, 10])\n",
      "Flatten has output size of: torch.Size([5, 80])\n",
      "Linear has output size of: torch.Size([5, 200])\n",
      "BatchNorm1d has output size of: torch.Size([5, 200])\n",
      "Tanh has output size of: torch.Size([5, 200])\n",
      "Linear has output size of: torch.Size([5, 27])\n"
     ]
    }
   ],
   "source": [
    "for layer in model.layers:\n",
    "    print(f\"{layer.__class__.__name__} has output size of: {layer.out.shape}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we don't want to process the 8 characters at the same time\n",
    "```\n",
    "1 2 3 4 5 6 7 8\n",
    "```\n",
    "but we want to process them in 4 groups of 2 characters in parallel\n",
    "```\n",
    "(1 2) (3 4) (5 6) (7 8)\n",
    "```\n",
    "\n",
    "so instead of multiplying `(5, 80) @ (80, 200) = (5, 200)` we want to multiply `(5, 4, 20) @ (20, 200) = (5, 4, 200)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# output of layer 0\n",
    "e = torch.randn(5, 8, 10)\n",
    "# contacenate even and odd (on character dimension) elements of the last dimension\n",
    "explicit = torch.cat([e[:, ::2, :], e[:, 1::2, :]], dim=2)\n",
    "# you can do the same using view\n",
    "implicit = e.view(5, 4, 20)\n",
    "\n",
    "(implicit == explicit).all()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.2 - FlattenConsectutive Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reimplement Flatten\n",
    "class FlattenConsecutive:\n",
    "    def __init__(self, n):\n",
    "        # n is the number of consecutive elements we want (2 in our example)\n",
    "        self.n = n\n",
    "\n",
    "    def __call__(self, x):\n",
    "        # in our example: B = 5, T = 8, C = 10\n",
    "        B, T, C = x.shape\n",
    "        # we want to convert X to (5, 4, 20)\n",
    "        x = x.view(B, T // self.n, C * self.n)\n",
    "\n",
    "        if x.shape[1] == 1:\n",
    "            x = x.squeeze(1)\n",
    "\n",
    "        self.out = x\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.3 - previous behavior using FlattenConsecutive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num parameters: 22097\n"
     ]
    }
   ],
   "source": [
    "n_embd = 10\n",
    "n_hidden = 200\n",
    "\n",
    "model = Sequential([\n",
    "    Embedding(vocab_size, n_embd),\n",
    "    # calling FlattenConsecutive(block_size) will return in the same previous behavior\n",
    "    FlattenConsecutive(block_size),\n",
    "\n",
    "    Linear(n_embd * block_size, n_hidden, bias=False),\n",
    "    BatchNorm1d(n_hidden),\n",
    "    Tanh(),\n",
    "\n",
    "    Linear(n_hidden, vocab_size)\n",
    "])\n",
    "\n",
    "with torch.no_grad():\n",
    "    layers[-1].weight *= 0.1\n",
    "\n",
    "parameters = model.parameters()\n",
    "print(f\"num parameters: {sum(p.numel() for p in parameters)}\")\n",
    "for p in parameters:\n",
    "    p.requires_grad_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 8])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0,  0, 13, 15,  8,  1, 13],\n",
       "        [ 0,  0,  0,  0, 11,  1, 19,  9],\n",
       "        [ 0,  0,  0,  0,  0, 10,  1, 25],\n",
       "        [ 0,  0,  0,  0,  0, 18,  9,  8],\n",
       "        [ 0,  0,  0,  0,  0,  0,  0,  4]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ix = torch.randint(0, Xtr.shape[0], (5,))\n",
    "Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "\n",
    "logits = model(Xb)\n",
    "print(Xb.shape)\n",
    "Xb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding has output size of: torch.Size([5, 8, 10])\n",
      "FlattenConsecutive has output size of: torch.Size([5, 80])\n",
      "Linear has output size of: torch.Size([5, 200])\n",
      "BatchNorm1d has output size of: torch.Size([5, 200])\n",
      "Tanh has output size of: torch.Size([5, 200])\n",
      "Linear has output size of: torch.Size([5, 27])\n"
     ]
    }
   ],
   "source": [
    "for layer in model.layers:\n",
    "    print(f\"{layer.__class__.__name__} has output size of: {(layer.out.shape)}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.4 - Processing Hierarchically: FlattenConsecutive(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num parameters: 22397\n"
     ]
    }
   ],
   "source": [
    "n_embd = 10\n",
    "# changing the number of hidden units to 68 keeps the same number of parameters as the previous model (22k)\n",
    "n_hidden = 68\n",
    "\n",
    "model = Sequential([\n",
    "    Embedding(vocab_size, n_embd),\n",
    "    FlattenConsecutive(2), Linear(n_embd * 2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    FlattenConsecutive(2), Linear(n_hidden * 2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    FlattenConsecutive(2), Linear(n_hidden * 2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    Linear(n_hidden, vocab_size),\n",
    "])\n",
    "\n",
    "with torch.no_grad():\n",
    "    layers[-1].weight *= 0.1\n",
    "\n",
    "parameters = model.parameters()\n",
    "print(f\"num parameters: {sum(p.numel() for p in parameters)}\")\n",
    "for p in parameters:\n",
    "    p.requires_grad_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 8])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0,  0,  0,  0,  0,  0, 16],\n",
       "        [ 0,  0,  0,  0,  0, 20,  5, 25],\n",
       "        [ 0,  0,  0,  0,  0,  0,  2, 18],\n",
       "        [ 0,  0,  0,  6,  1, 18,  8,  1],\n",
       "        [ 0,  0,  0,  0,  0,  0, 19,  8]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ix = torch.randint(0, Xtr.shape[0], (5,))\n",
    "Xb, Yb = Xtr[ix], Ytr[ix]\n",
    "\n",
    "logits = model(Xb)\n",
    "print(Xb.shape)\n",
    "Xb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding has output size of: torch.Size([5, 8, 10])\n",
      "FlattenConsecutive has output size of: torch.Size([5, 4, 20])\n",
      "Linear has output size of: torch.Size([5, 4, 68])\n",
      "BatchNorm1d has output size of: torch.Size([5, 4, 68])\n",
      "Tanh has output size of: torch.Size([5, 4, 68])\n",
      "FlattenConsecutive has output size of: torch.Size([5, 2, 136])\n",
      "Linear has output size of: torch.Size([5, 2, 68])\n",
      "BatchNorm1d has output size of: torch.Size([5, 2, 68])\n",
      "Tanh has output size of: torch.Size([5, 2, 68])\n",
      "FlattenConsecutive has output size of: torch.Size([5, 136])\n",
      "Linear has output size of: torch.Size([5, 68])\n",
      "BatchNorm1d has output size of: torch.Size([5, 68])\n",
      "Tanh has output size of: torch.Size([5, 68])\n",
      "Linear has output size of: torch.Size([5, 27])\n"
     ]
    }
   ],
   "source": [
    "for layer in model.layers:\n",
    "    print(f\"{layer.__class__.__name__} has output size of: {layer.out.shape}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "however, this network gives the same loss = 2.0"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.5 - Fixing BatchNorm Bug\n",
    "we implemented batchnorm for X 2D only\n",
    "\n",
    "we calculated mean and variance for the first dimension only\n",
    "\n",
    "we don't want to average over the batch dimension only, but also over the 2nd dimension (the 4 groups of 2 characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 4, 68])\n",
      "shape of running mean is torch.Size([1, 4, 68])\n"
     ]
    }
   ],
   "source": [
    "e = torch.rand(32, 4, 68)\n",
    "emean = e.mean(dim=(0, 1), keepdim=True)  # (1, 1, 68)\n",
    "evar = e.var((0, 1), keepdim=True)  # (1, 1, 68)\n",
    "ehat = (e - emean) / torch.sqrt(evar + 1e-5)\n",
    "\n",
    "print(ehat.shape)\n",
    "print(f\"shape of running mean is {model.layers[3].running_mean.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchNorm1d:\n",
    "    def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "        self.eps = eps\n",
    "        self.momentum = momentum\n",
    "        self.training = True\n",
    "\n",
    "        # parameters (trained with backprop)\n",
    "        self.gamma = torch.ones(dim)\n",
    "        self.beta = torch.zeros(dim)\n",
    "\n",
    "        # buffers (trained while running `momentum update`)\n",
    "        self.running_mean = torch.zeros(dim)\n",
    "        self.running_var = torch.ones(dim)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        if self.training:\n",
    "            # determine the dimension to reduce over\n",
    "            if x.ndim == 2:\n",
    "                dim = 0\n",
    "            elif x.ndim == 3:\n",
    "                dim = (0, 1)\n",
    "\n",
    "            xmean = x.mean(dim, keepdim=True)\n",
    "            # batch variance\n",
    "            xvar = x.var(dim, keepdim=True)\n",
    "        else:\n",
    "            xmean = self.running_mean\n",
    "            xvar = self.running_var\n",
    "\n",
    "        xhat = (x - xmean) / torch.sqrt(xvar + self.eps)\n",
    "        self.out = self.gamma * xhat + self.beta\n",
    "\n",
    "        # update the buffers in training\n",
    "        if self.training:\n",
    "            with torch.no_grad():\n",
    "                self.running_mean = (1 - self.momentum) * \\\n",
    "                    self.running_mean + self.momentum * xmean\n",
    "                self.running_var = (1 - self.momentum) * \\\n",
    "                    self.running_var + self.momentum * xvar\n",
    "\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.gamma, self.beta]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num parameters: 22397\n"
     ]
    }
   ],
   "source": [
    "n_embd = 10\n",
    "# changing the number of hidden units to 68 keeps the same number of parameters as the previous model (22k)\n",
    "n_hidden = 68\n",
    "\n",
    "model = Sequential([\n",
    "    Embedding(vocab_size, n_embd),\n",
    "    FlattenConsecutive(2), Linear(n_embd * 2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    FlattenConsecutive(2), Linear(n_hidden * 2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    FlattenConsecutive(2), Linear(n_hidden * 2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    Linear(n_hidden, vocab_size),\n",
    "])\n",
    "\n",
    "with torch.no_grad():\n",
    "    layers[-1].weight *= 0.1\n",
    "\n",
    "parameters = model.parameters()\n",
    "print(f\"num parameters: {sum(p.numel() for p in parameters)}\")\n",
    "for p in parameters:\n",
    "    p.requires_grad_()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model improves a little bit (2.029 -> 2.022)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4- Final Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Layers made in part 3\n",
    "class Linear:\n",
    "    def __init__(self, fan_in, fan_out, bias=True):\n",
    "        self.weight = torch.randn((fan_in, fan_out))\n",
    "        self.weight /= fan_in ** 0.5\n",
    "        self.bias = torch.zeros((fan_out)) if bias else None\n",
    "\n",
    "    def __call__(self, x):\n",
    "        self.out = x @ self.weight\n",
    "        if self.bias is not None:\n",
    "            self.out += self.bias\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.weight] + ([] if self.bias is None else [self.bias])\n",
    "\n",
    "\n",
    "class BatchNorm1d:\n",
    "    def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "        self.eps = eps\n",
    "        self.momentum = momentum\n",
    "        self.training = True\n",
    "\n",
    "        # parameters (trained with backprop)\n",
    "        self.gamma = torch.ones(dim)\n",
    "        self.beta = torch.zeros(dim)\n",
    "\n",
    "        # buffers (trained while running `momentum update`)\n",
    "        self.running_mean = torch.zeros(dim)\n",
    "        self.running_var = torch.ones(dim)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        if self.training:\n",
    "            # determine the dimension to reduce over\n",
    "            if x.ndim == 2:\n",
    "                dim = 0\n",
    "            elif x.ndim == 3:\n",
    "                dim = (0, 1)\n",
    "\n",
    "            xmean = x.mean(dim, keepdim=True)\n",
    "            # batch variance\n",
    "            xvar = x.var(dim, keepdim=True)\n",
    "        else:\n",
    "            xmean = self.running_mean\n",
    "            xvar = self.running_var\n",
    "\n",
    "        xhat = (x - xmean) / torch.sqrt(xvar + self.eps)\n",
    "        self.out = self.gamma * xhat + self.beta\n",
    "\n",
    "        # update the buffers in training\n",
    "        if self.training:\n",
    "            with torch.no_grad():\n",
    "                self.running_mean = (1 - self.momentum) * \\\n",
    "                    self.running_mean + self.momentum * xmean\n",
    "                self.running_var = (1 - self.momentum) * \\\n",
    "                    self.running_var + self.momentum * xvar\n",
    "\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.gamma, self.beta]\n",
    "\n",
    "\n",
    "class Tanh:\n",
    "    def __call__(self, x):\n",
    "        self.out = torch.tanh(x)\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []\n",
    "\n",
    "\n",
    "class Embedding:\n",
    "    def __init__(self, num_embeddings, embedding_dim):\n",
    "        self.weight = torch.randn((num_embeddings, embedding_dim))\n",
    "\n",
    "    def __call__(self, x):\n",
    "        self.out = self.weight[x]\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.weight]\n",
    "\n",
    "\n",
    "class Flatten:\n",
    "    def __call__(self, x):\n",
    "        self.out = x.view((x.shape[0], -1))\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []\n",
    "\n",
    "\n",
    "class Sequential:\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "\n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x)\n",
    "\n",
    "        self.out = x\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [p for l in self.layers for p in l.parameters()]\n",
    "\n",
    "\n",
    "class FlattenConsecutive:\n",
    "    def __init__(self, n):\n",
    "        # n is the number of consecutive elements we want (2 in our example)\n",
    "        self.n = n\n",
    "\n",
    "    def __call__(self, x):\n",
    "        # in our example: B = 5, T = 8, C = 10\n",
    "        B, T, C = x.shape\n",
    "        # we want to convert X to (5, 4, 20)\n",
    "        x = x.view(B, T // self.n, C * self.n)\n",
    "\n",
    "        if x.shape[1] == 1:\n",
    "            x = x.squeeze(1)\n",
    "\n",
    "        self.out = x\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "76579\n"
     ]
    }
   ],
   "source": [
    "n_embd = 24\n",
    "n_hidden = 128\n",
    "\n",
    "model = Sequential([\n",
    "    Embedding(vocab_size, n_embd),\n",
    "    FlattenConsecutive(2), Linear(n_embd * 2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    FlattenConsecutive(2), Linear(n_hidden*2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    FlattenConsecutive(2), Linear(n_hidden*2, n_hidden,\n",
    "                                  bias=False), BatchNorm1d(n_hidden), Tanh(),\n",
    "    Linear(n_hidden, vocab_size),\n",
    "])\n",
    "\n",
    "with torch.no_grad():\n",
    "    model.layers[-1].weight *= 0.1  # last layer make less confident\n",
    "\n",
    "parameters = model.parameters()\n",
    "print(sum(p.nelement() for p in parameters))  # number of parameters in total\n",
    "for p in parameters:\n",
    "    p.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0/ 200000: 2.1721\n",
      "  10000/ 200000: 1.8982\n",
      "  20000/ 200000: 2.0865\n",
      "  30000/ 200000: 1.7060\n",
      "  40000/ 200000: 1.5556\n",
      "  50000/ 200000: 1.8686\n",
      "  60000/ 200000: 1.6639\n",
      "  70000/ 200000: 1.8981\n",
      "  80000/ 200000: 1.8715\n",
      "  90000/ 200000: 1.8957\n",
      " 100000/ 200000: 2.0003\n",
      " 110000/ 200000: 1.7974\n",
      " 120000/ 200000: 1.6178\n",
      " 130000/ 200000: 1.9566\n",
      " 140000/ 200000: 1.6385\n",
      " 150000/ 200000: 1.7304\n",
      " 160000/ 200000: 1.9413\n",
      " 170000/ 200000: 1.8178\n",
      " 180000/ 200000: 1.7655\n",
      " 190000/ 200000: 1.3521\n"
     ]
    }
   ],
   "source": [
    "# same optimization as last time\n",
    "max_steps = 200000\n",
    "batch_size = 32\n",
    "lossi = []\n",
    "\n",
    "for i in range(max_steps):\n",
    "\n",
    "    # minibatch construct\n",
    "    ix = torch.randint(0, Xtr.shape[0], (batch_size,))\n",
    "    Xb, Yb = Xtr[ix], Ytr[ix]  # batch X,Y\n",
    "\n",
    "    # forward pass\n",
    "    logits = model(Xb)\n",
    "    loss = F.cross_entropy(logits, Yb)  # loss function\n",
    "\n",
    "    # backward pass\n",
    "    for p in parameters:\n",
    "        p.grad = None\n",
    "    loss.backward()\n",
    "\n",
    "    # update: simple SGD\n",
    "    lr = 0.1 if i < 150000 else 0.01  # step learning rate decay\n",
    "    for p in parameters:\n",
    "        p.data += -lr * p.grad\n",
    "\n",
    "    # track stats\n",
    "    if i % 10000 == 0:  # print every once in a while\n",
    "        print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
    "    lossi.append(loss.log10().item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABhOklEQVR4nO3dd3TV9f3H8ee9N8nN3iQhi7A3BAhEVBA1ghbrqFakViy2WgdaG+sPqRVsbQuOWlul2GpdqFVb0bqKSiSKElYCInsnIWSQhOyd+/39cZMLVwIkIclNwutxTs4h9zvy/nIl9+VnmgzDMBARERHpxsyuLkBERETkTBRYREREpNtTYBEREZFuT4FFREREuj0FFhEREen2FFhERESk21NgERERkW5PgUVERES6PTdXF9ARbDYbR44cwc/PD5PJ5OpyREREpBUMw6C8vJzIyEjM5tO3ofSKwHLkyBFiYmJcXYaIiIi0Q3Z2NtHR0ac9p1cEFj8/P8D+wP7+/i6uRkRERFqjrKyMmJgYx+f46fSKwNLcDeTv76/AIiIi0sO0ZjiHBt2KiIhIt6fAIiIiIt2eAouIiIh0ewosIiIi0u0psIiIiEi3p8AiIiIi3Z4Ci4iIiHR7CiwiIiLS7SmwiIiISLenwCIiIiLdngKLiIiIdHsKLCIiItLtKbCcRnlNPX/6dDfz/7MVwzBcXY6IiMg5S4HlNNwtZp75fB9vbcqmtLre1eWIiIicsxRYTsPT3UKorwcAh49Vu7gaERGRc5cCyxlEBXoBkFOiwCIiIuIqCixnEBVkDyxHFFhERERcRoHlDBwtLOoSEhERcRkFljNQl5CIiIjrKbCcQVSQN6DAIiIi4koKLGcQGegJqEtIRETElRRYziA60N7CUlRZR3Vdo4urEREROTcpsJyBv5cbvlY3QN1CIiIirqLAcgYmk0kDb0VERFxMgaUVmtdi0TgWERER11BgaYXjLSxVLq5ERETk3KTA0gpqYREREXEtBZZW0BgWERER11JgaQW1sIiIiLiWAksrRDe1sOSV1VDfaHNxNSIiIuceBZZWCPW14mExYzMgr7TG1eWIiIiccxRYWsFsNh1fol/jWERERLpcuwLL0qVLiYuLw9PTk8TERDZs2HDKc1esWEFCQgKBgYH4+PgQHx/P8uXLT3n+HXfcgclk4umnn25PaZ0mJti+RP+hwkoXVyIiInLuaXNgeeutt0hOTmbRokVkZGQwduxYZsyYQUFBQYvnBwcH89BDD5GWlsbWrVuZO3cuc+fO5ZNPPjnp3HfffZd169YRGRnZ9ifpZEPC/QDYk1/h4kpERETOPW0OLE899RS33XYbc+fOZcSIETz33HN4e3vz4osvtnj+tGnTuPbaaxk+fDgDBw7kF7/4BWPGjOGrr75yOi8nJ4d77rmH119/HXd39/Y9TScaEu4LwJ78chdXIiIicu5pU2Cpq6sjPT2dpKSk4zcwm0lKSiItLe2M1xuGQUpKCrt372bq1KmO1202GzfffDMPPPAAI0eObEtJXeZ4C4sCi4iISFdza8vJhYWFNDY2Eh4e7vR6eHg4u3btOuV1paWlREVFUVtbi8Vi4W9/+xuXXXaZ4/hjjz2Gm5sb9957b6vqqK2tpba21vF9WVlZWx6jXQY3BZaC8lqOVdYR5OPR6T9TRERE7NoUWNrLz8+PLVu2UFFRQUpKCsnJyQwYMIBp06aRnp7OX/7yFzIyMjCZTK263+LFi/ntb3/byVU787W6ERXoRU5JNXvyy0kcENKlP19ERORc1qYuodDQUCwWC/n5+U6v5+fnExERceofYjYzaNAg4uPjuf/++7n++utZvHgxAGvWrKGgoIDY2Fjc3Nxwc3MjMzOT+++/n7i4uBbvt2DBAkpLSx1f2dnZbXmMdhsaoW4hERERV2hTYPHw8GDChAmkpKQ4XrPZbKSkpDB58uRW38dmszm6dG6++Wa2bt3Kli1bHF+RkZE88MADLc4kArBarfj7+zt9dQXNFBIREXGNNncJJScnc8stt5CQkMCkSZN4+umnqaysZO7cuQDMmTOHqKgoRwvK4sWLSUhIYODAgdTW1vLxxx+zfPlyli1bBkBISAghIc7dK+7u7kRERDB06NCzfb4ONTTCPlNot1pYREREulSbA8usWbM4evQoCxcuJC8vj/j4eFauXOkYiJuVlYXZfLzhprKykrvuuovDhw/j5eXFsGHDeO2115g1a1bHPUUXGRx2vEvIMIxWj7kRERGRs2MyDMNwdRFnq6ysjICAAEpLSzu1e6imvpERC1diM2DDry8lzN+z036WiIhIb9eWz2/tJdQGnu4W4kJ8AI1jERER6UoKLG00OFzjWERERLqaAksbDY2wN1ntzO38xepERETEToGljcZGBwCwJbvEtYWIiIicQxRY2ig+JhCAfQUVlFbXu7YYERGRc4QCSxuF+FqJDfYGYOvhEtcWIyIico5QYGmHcbGBAGzOKnFpHSIiIucKBZZ2aO4W2px1zLWFiIiInCMUWNphXGwQYB942wvW3RMREen2FFjaYURffzzczByrqiezqMrV5YiIiPR6Cizt4OFmZmSkfT2WzdnqFhIREelsCiztNC7G3i2kgbciIiKdT4GlnZpnCq3akU9OSbVrixEREenlFFjaaeqQPkT4e3KktIZrl37NtpxSV5ckIiLSaymwtFOAlzvv3HU+Q8P9KCiv5aYX1lNeo5VvRUREOoMCy1mICvTi33dOJsLfk9LqerYeViuLiIhIZ1BgOUv+nu5M6GcfgKtuIRERkc6hwNIBRkXZd3DedqTMxZWIiIj0TgosHWBUlH1NFrWwiIiIdA4Flg4wMtLewnKwsFIDb0VERDqBAksHCPbxICrQC4Ad6hYSERHpcAosHaR5qX6NYxEREel4CiwdpHng7XaNYxEREelwCiwdZHRTYPlWgUVERKTDKbB0kJFNM4X2H62gqq7BxdWIiIj0LgosHSTMz5MwPys2A3bmlru6HBERkV5FgaUDNXcLbc465uJKREREehcFlg40sX8wAOsOFLu4EhERkd5FgaUDTR4QAsD6g0U02gwXVyMiItJ7KLB0oJGR/vhZ3SivaWBnrtZjERER6SgKLB3IzWJ2dAul7S9ycTUiIiK9hwJLB2vuFko7UERtQyOPfriDtzdlu7gqERGRns3N1QX0NpMH2gPLhoPF/PGjnbySlomv1Y0fTojGZDK5uDoREZGeSS0sHWx4X3/8Pd2oqG3glbRMACpqGyiurHNxZSIiIj2XAksHs5hNTOofctLrWcVVLqhGRESkd1Bg6QTThvYBYFSUP/ExgYACi4iIyNnQGJZOcOPEGAK93ZkyqA+/+3AHW7JLOHys2tVliYiI9FgKLJ3AzWLmyjGRAMQGewOQVaQWFhERkfZSl1Aniw3xAtQlJCIicjYUWDpZTJC9hSX7mAKLiIhIeymwdLLmLqEjJdXUN9pcXI2IiEjPpMDSyfr4WbG6mbEZ9tAiIiIibafA0slMJhMxTa0s2cUKLCIiIu2hwNIFHDOFNPBWRESkXdoVWJYuXUpcXByenp4kJiayYcOGU567YsUKEhISCAwMxMfHh/j4eJYvX+44Xl9fz/z58xk9ejQ+Pj5ERkYyZ84cjhw50p7SuiUFFhERkbPT5sDy1ltvkZyczKJFi8jIyGDs2LHMmDGDgoKCFs8PDg7moYceIi0tja1btzJ37lzmzp3LJ598AkBVVRUZGRk8/PDDZGRksGLFCnbv3s1VV111dk/WjUQH2ac2a6aQiIhI+5gMwzDackFiYiITJ07k2WefBcBmsxETE8M999zDgw8+2Kp7jB8/npkzZ/Loo4+2eHzjxo1MmjSJzMxMYmNjz3i/srIyAgICKC0txd/fv/UP00U+3Z7H7cvTGRMdwOWjInjxq4P8/eYJTOgX7OrSREREXKYtn99tamGpq6sjPT2dpKSk4zcwm0lKSiItLe2M1xuGQUpKCrt372bq1KmnPK+0tBSTyURgYGCLx2traykrK3P66s5iQ+xdQtuPlPH4yt0UVtTxn/TDLq5KRESk52hTYCksLKSxsZHw8HCn18PDw8nLyzvldaWlpfj6+uLh4cHMmTN55plnuOyyy1o8t6amhvnz5zN79uxTpq3FixcTEBDg+IqJiWnLY3S55sXjGm3HG7PS9he5qhwREZEep0tmCfn5+bFlyxY2btzIH/7wB5KTk0lNTT3pvPr6em644QYMw2DZsmWnvN+CBQsoLS11fGVnZ3di9WfPx+pGuL8VgB+Mj8JsgkNFVeSWapqziIhIa7Rp88PQ0FAsFgv5+flOr+fn5xMREXHK68xmM4MGDQIgPj6enTt3snjxYqZNm+Y4pzmsZGZm8vnnn5+2L8tqtWK1WttSuss9dUM8e/LLmTM5jn0FFWw9XEra/iJ+MD7a1aWJiIh0e21qYfHw8GDChAmkpKQ4XrPZbKSkpDB58uRW38dms1FbW+v4vjms7N27l1WrVhESEtKWsnqECwaFMveC/ljMJiYPsD/fugPqFhIREWmNNrWwACQnJ3PLLbeQkJDApEmTePrpp6msrGTu3LkAzJkzh6ioKBYvXgzYx5skJCQwcOBAamtr+fjjj1m+fLmjy6e+vp7rr7+ejIwMPvzwQxobGx3jYYKDg/Hw8OioZ+02zhsYwt+/PECaAouIiEirtDmwzJo1i6NHj7Jw4ULy8vKIj49n5cqVjoG4WVlZmM3HG24qKyu56667OHz4MF5eXgwbNozXXnuNWbNmAZCTk8P7778P2LuLTrR69WqnbqPeYmJcMBazieziag4fqyK6aVCuiIiItKzN67B0R919HZaWXPu3r9mcVcIPxkdRVl3PmOhA7r10sKvLEhER6TKdtg6LdJzmcSwrMnJYtbOApz7bQ1lNvYurEhER6Z4UWFzkB+OjCPX1YERff/w97T1zu3LLXVyViIhI96TA4iKDwvzY9JvL+PgXU5jU397asv1IqYurEhER6Z4UWLqBEZH2frsdR7r3FgMiIiKuosDSDYzo2xRYchVYREREWqLA0g2MbGph2ZtfQV2DzcXViIiIdD8KLN1AdJAXfp5u1DXa2FdQ4epyREREuh0Flm7AZDKpW0hEROQ0FFi6CQ28FREROTUFlm7ieAuLpjaLiIh8lwJLNzEyMgCwt7B8d7cEm83g1+9+y8PvbTvpmIiIyLlAgaWbGBTmi7vFRFlNA1MeX82Uxz9nzd6jAHy8LZc31mexfF0me/I1KFdERM49CizdhIebmQn9ggA4fKya7OJqfvnWNxSU1/CnT/c4zvtiT4GrShQREXEZ7dbcjZRW1/Pt4VI83c0sWPEtewsqiAr0Iqek2nHOBYNCeP1n57mwShERkY6h3Zp7qAAvdy4cHEpCXDBP3RCPxWxyhJU5k/sBsPHgMSprG1xZpoiISJdTYOmmRkcHMO/iQQBEBXrx0MzhxAR7UddoY92BIhdXJyIi0rXcXF2AnNo9lwyib4AnE/oFYXWzcNGQPry2Losv9hzl0uHhri5PRESky6iFpRtzs5i5cVIsg8P9ALhoSBgAX+w56sqyREREupwCSw9y/sAQ3C0mMouqOFhY6epyREREuowCSw/iY3VjXIx96nNG5jEXVyMiItJ1FFh6mCERvgDsP6oF5ERE5NyhwNLDDOpjDyz7ChRYRETk3KHA0sMMDFMLi4iInHsUWHqYQU2BJbOoivpGm4urERER6RoKLD1MhL8nPh4WGmwGmUWaKSQiIucGBZYexmQyObqF9hU4B5YPvjnCa+syXVGWiIhIp9JKtz3QwD6+bD1c6jSOJbu4il+8uRmbAZMHhjCwaXCuiIhIb6AWlh6oeRzL/hNmCr2y9hC2pn230w9pjRYREeldFFh6oIF9fADY19TCUlHbwFubsh3HM7IUWEREpHdRYOmBTmxhMQyDd9IPU17TgJvZBCiwiIhI76PA0gPFBvtgMZuorGvkSGkNL689BMBdFw8CYG9BBWU19S6sUEREpGMpsPRAHm5m+oV4A/CzVzZxsLASf083fj51ALHB3hgGbMkqcW2RIiIiHUiBpYdqngW0M7cMd4uJP1w7Gh+rG+NjAwF1C4mISO+iwNJDDY/wA8Df041Xb03k+2MjARjfr2k35zO0sNTUN7IluwTDMDq1ThERkY6gdVh6qFvOj8PNYub7YyPpH+rjeH18rD2wbM46hs1mYG4aiPtdT36ymxe+OsizPxrHlWMiu6RmERGR9lILSw8V4mvl3ksHO4UVgGERfnh7WCivaXBMe27JF3uOArDxYHGn1ikiItIRFFh6GTeLmfiYQAB+8942yluYLVRRezzM7D+q/YhERKT7U2DphR6YMRQ/qxsbDhYz+/l1FJTXOB3/9nApzUNXDpymFUZERKS7UGDphcbFBvGv288jxMeDbTllXPxEKk+v2kNlbQMAWw+XOM49UlpDVV2DiyoVERFpHQWWXmpUVABv3zGZMdEBVNY18vSqvdz8z/UYhsE3JwQWgAPqFhIRkW5OgaUXG9jHl/fuuoBnfzQOT3czGVklpGce45vsUgCsbva3f7+6hUREpJtTYOnlzGYTV46J5KqmdVqe+XwfOSXVmEyQNDwcUAuLiIh0fwos54jZk2KB49OZB/bxZWxMAKAWFhER6f4UWM4R8TGBDGtaHRdgTHQAA0Lty/urhUVERLq7dgWWpUuXEhcXh6enJ4mJiWzYsOGU565YsYKEhAQCAwPx8fEhPj6e5cuXO51jGAYLFy6kb9++eHl5kZSUxN69e9tTmpyCyWTiR4mxju/jYwIZGNYUWAoraGi0seR/u3ilaednERGR7qTNgeWtt94iOTmZRYsWkZGRwdixY5kxYwYFBQUtnh8cHMxDDz1EWloaW7duZe7cucydO5dPPvnEcc7jjz/OX//6V5577jnWr1+Pj48PM2bMoKampsV7SvtcHR+Fp7v9LR8XE0RMkBfuFhM19TaWrt7Pc1/s55EPtlNadfJicyIiIq5kMtq4+11iYiITJ07k2WefBcBmsxETE8M999zDgw8+2Kp7jB8/npkzZ/Loo49iGAaRkZHcf//9/OpXvwKgtLSU8PBwXn75ZW688cYz3q+srIyAgABKS0vx9/dvy+Occ1J3F5BdXMXNk+MASHrqC/YVVGAxm2i02f9TeGFOAkkjwl1YpYiInAva8vndphaWuro60tPTSUpKOn4Ds5mkpCTS0tLOeL1hGKSkpLB7926mTp0KwMGDB8nLy3O6Z0BAAImJiae8Z21tLWVlZU5f0jrThoY5wgrAgKa9iJrDCsD6g0VdXZaIiMhptSmwFBYW0tjYSHi48/99h4eHk5eXd8rrSktL8fX1xcPDg5kzZ/LMM89w2WWXATiua8s9Fy9eTEBAgOMrJiamLY8hJ2gexwLHpzmvO6ANEUVEpHvpkllCfn5+bNmyhY0bN/KHP/yB5ORkUlNT232/BQsWUFpa6vjKzs7uuGLPMWOi7FOb42MC+d3VIwHYfqSUshY2TRQREXEVt7acHBoaisViIT8/3+n1/Px8IiIiTnmd2Wxm0KBBAMTHx7Nz504WL17MtGnTHNfl5+fTt29fp3vGx8e3eD+r1YrVam1L6XIKM0ZG8NyPJ5DYP5ggHw/6hXiTWVTFpkPFXDJM41hERKR7aFMLi4eHBxMmTCAlJcXxms1mIyUlhcmTJ7f6PjabjdraWgD69+9PRESE0z3LyspYv359m+4p7WM2m7h8VARBPh4AJPYPBmC9uoVERKQbaVMLC0BycjK33HILCQkJTJo0iaeffprKykrmzp0LwJw5c4iKimLx4sWAfbxJQkICAwcOpLa2lo8//pjly5ezbNkywL4+yH333cfvf/97Bg8eTP/+/Xn44YeJjIzkmmuu6bgnlVY5b0AIb286zLoDReSX1bArr5wLB4ViMZtcXZqIiJzD2hxYZs2axdGjR1m4cCF5eXnEx8ezcuVKx6DZrKwszObjDTeVlZXcddddHD58GC8vL4YNG8Zrr73GrFmzHOf83//9H5WVldx+++2UlJRw4YUXsnLlSjw9PTvgEaUtEgeEAPDN4VLOW5yCYcDdFw/kgRnDXFyZiIicy9q8Dkt3pHVYOtYlf0p1Wq7fy93CmvkXE+qrcUMiItJxOm0dFjk3PPfjCTx23Wi+mn8xY6IDqK5v5B9fHmjzfarqGtiWU0ovyMQiIuJiCixykiHhfsyaGEt0kDe/TBoCwKtphzhaXtvqe+zOK+eKv6zhyme+cuwQLSIi0l5tHsMi55ZpQ/sQHxPIluwSnvtiPw9fOaLF8+oabKzcnkdBWQ3lNQ08v+YAVXWNAHy9r5BpQ8O6smwREellFFjktEwmE/clDeYnL23ktXWZ/HzqAML8jw+Grmuw8d6WHP6yai85JdVO14b5WSkor+Wb7NKuLltERHoZBRY5o4uG9GF8bCAZWSX8LXU/j1w1ktzSal7++hD/ST9MUWUdAH38rJw/MAQ3s5nhff24cHAolz+9hm9zSmlotOFmUQ+kiIi0jwKLnJHJZOKXlw3h5n9u4I0NWUweGMKvV3zrCCphflZ+emF/5kyOw8vD4rjOZjPwtbpRUdvA3oIKhvfVDC4REWkfBRZplQsHhZLQL4hNmcf4+fJ0AIb39Sf5siFcPLRPi60nZrOJMdEBrN1fxDfZJQosIiLSbmqjl1YxmUwkXzbE8f3FQ/vwnzsmc9mI8NN29YyNCQTgm8MlAOzNL6egvKYzSxURkV5ILSzSapMHhrDwyhHUNdr42YX9WzUmZWx0IABbskvZcLCY2c+vw+pm5o/XjuaqsZGs2VfInrxybp7cD093y+lvJiIi5yytdCudKq+0hvMWp2A2wYA+vuwrqHAcC/W1UlhhX9vlwSuGccdFA11VpoiIuIBWupVuIyLAk3B/KzYD9hVUEOTtzp3TBmIyQWFFLc17Kq7eVeDaQkVEpFtTl5B0uviYQD7Zng/A/MuHceOkWC4bEU52cRVDI/y4/Ok1pGceo7ymHj9PdzYeKgYgoV8QJpN2iRYREQUW6QIT44L5ZHs+42IDuSEhBoDxsUGMjw0CIC7Em0NFVaTtLyI6yJsb/p6GYdgDy53TBpI4IARfq/5TFRE5l+lTQDrdj8/rh9XdwhWjIjCbT24xmTqkD4fSMvliz1Gq6xppHlW1KfMYP31lE2YTjIj058kfjmVYhMYoiYicizSGRTqdp7uFm8/rR6ivtcXjFw3pA8An2/N5/5sjALwwJ4FbL+hPVKAXNgO25ZTxwpqDXVaziIh0Lwos4nLnDQjB3WKisKKWBpvBeQOCSRoRzsLvj+DrBy/hpbkTAUjZmU9Do83F1YqIiCsosIjL+VjdmBgX7Pj+9qkDnI5PGRRKoLc7x6rqSc881tXliYhIN6DAIt1Cc7fQoDBfpg0JczrmZjFzyTD7a5/uyO/y2kRExPUUWKRbuHlyP+64aCB/vXFciwNzp4+IAODTHXn0grUORUSkjTRLSLoFbw83Hrxi2CmPTx0SitXNTHZxNbvzyzVbSETkHKMWFukRvD3cmDK4aTbRNnULiYicaxRYpMeYMTIcgBfWHODbw6UurkZERLqSAov0GFfFR5LYP5jy2gbmvLiePfnlri5JRES6iAKL9BhWNwsv3JLA2OgAjlXVc8uLG6iobXB1WSIi0gUUWKRH8fN055VbJxET7EVuaQ3/+PKAq0sSEZEuoMAiPU6gtwcLrhgOwPNfHiC/rMbFFYmISGdTYJEe6YpREYyPDaS6vpE/f7bH1eWIiEgnU2CRHslkMvHQTHsry9ubstmrAbgiIr2aAov0WBP6BTNjZDg2A5al7nd1OSIi0okUWKRHu/viQQD895sjHD5W5eJqRESksyiwSI82JjqQCweF0mgzeF4zhkREei0FFunx7pw2EIA3N2ZTWFHr4mpERKQzKLBIj3f+wBDGRgdQ22Dj5a8PubocERHpBAos0uOZTCZHK8uraYcor6k/4zUVtQ3UN9oc3+/NL+eJT3a16loREel6bq4uQKQjTB8RwYA+Phw4Wskb67O4feoAnl9zgJSdBeSW1mBg8LcfTWB0dADbckq58R/rGBbhx1s/n4wJuPuNDPbkV1DfaPDr79mnSzc02rCYTZhMJtc+nIiIqIVFegez2cQdF9lbWV746iAL/7udP368i/UHi8kqriK7uJrbl2/iUGEld7+RQUVtA5syj/FOxmE++jaXPfkVALy5IYuqugYKymq49KkvuOZvaymurHPlo4mICGAyDMNwdRFnq6ysjICAAEpLS/H393d1OeIidQ02LnpiNbml9qX6TSb41fShjIsN5DfvbuNAYSWe7mZq6m24W0zUNxr08bPi5+nGgaOVjvv8/ppRbDhYzPvfHAFgTHQAr/8sET9Pd5c8l4hIb9WWz2+1sEiv4eFm5mdTBji+X/KD0dx98SDOHxjKP+ZMwNfqRk29DTezidd+mki/EG+Oltdy4GglAV7u/DJpCABPfbaH9785gtkEAV7ubD1cys9e2eQY82KzGXzwzRHSM4vpBXlfRKRHUGCRXuWmxFh+emF//nbTeGZNjHW8PijMj2dmj2NAqA9/uHYUiQNCWHDFMMfx26b0Z+6Fcfh4WBxdQDef14/XfpqIr9WN9QeL+e8We4vLGxuyuOdfm7luWRqX/flLR0vMmdTUN5JVpMXtRETaQ4FFehVPdwsPXzmC743ue9Kxi4eF8fmvpjmCzIyREVw/IZpJccH85IL++Hu688OEGABCfa0kTx/K6OgAx2q6L6w5QH2jjee+sG8DYDbBvoIKfvHmZrbllJ6xtgUrvuWiJ1fz4dbWBRwRETlOgUXOWSaTiSd/OJa375iMr9U+Ye6eSwZxQ0I0y348ngAv+5iVHyXG4uNhYVdeOfPf2crhY9WE+Hiw7teXMn1EOIYBv/9ox2m7h2rqG/nftlwMAxb+d7sG8oqItJECi8gJQnytPH79WCbGBTteC/Byd7TKrMjIAeDWC/sT5ufJoqtGYnUzs+5AMZ/uyD/lfdcdKKKm3j4Gpriyjt9/uKMTn0JEpPdRYBFphbkXxGEx29dj8bW68ePz+gEQFejFbU0DfRd/vJO6BnsoKaupJ/mtLby7+TAAq3cVADA+NhCTCVZsziFl56kDjoiIOFNgEWmFmGBvrhxjHxczZ3I/R3cRwB3TBtLHz8qhoipeTTsEwNLP97Ficw4LVnxLflkNq3cfBeDOaYP4yflxgH2xurX7Crv0OUREeqp2BZalS5cSFxeHp6cniYmJbNiw4ZTnPv/880yZMoWgoCCCgoJISko66fyKigrmzZtHdHQ0Xl5ejBgxgueee649pYl0mj9eO5qlPxrPLy8b4vS6r9WNX023v/aXlL1sP1LKS2sPAVBTb+NX//6GrOIqPCxmzh8YwoNXDOPioX2oqbdx6ysbWbu/baHFMAxtISAi55w2B5a33nqL5ORkFi1aREZGBmPHjmXGjBkUFBS0eH5qaiqzZ89m9erVpKWlERMTw/Tp08nJyXGck5yczMqVK3nttdfYuXMn9913H/PmzeP9999v/5OJdDAfqxszx/TF3XLyP5vrJ8QwvK8/5TUN3Pj3ddQ12IgJ9gJgzV57IEkcEIyP1Q2rm4Xnbp7gCC3z39napvVcXll7iNGPfOroZhIRORe0ObA89dRT3HbbbcydO9fREuLt7c2LL77Y4vmvv/46d911F/Hx8QwbNowXXngBm81GSkqK45y1a9dyyy23MG3aNOLi4rj99tsZO3bsaVtuRLoTi9nEwzPtexCV1zYA8NcbxzFlcKjjnGlDwxx/trpZePZH43G3mMguria7uLrVP6u5e+njb3M7onQRkR6hTYGlrq6O9PR0kpKSjt/AbCYpKYm0tLRW3aOqqor6+nqCg4/Pwjj//PN5//33ycnJwTAMVq9ezZ49e5g+fXqL96itraWsrMzpS8TVzh8UStLwcAAuHxnBuNggHpgx1HH84qF9nM73sboxNjoQsM8iaq3MIvs2AlsPn3ntFxGR3qJNgaWwsJDGxkbCw8OdXg8PDycvL69V95g/fz6RkZFOoeeZZ55hxIgRREdH4+HhweWXX87SpUuZOnVqi/dYvHgxAQEBjq+YmJi2PIZIp3nyh2P4zczhLLluNABjogN54vox/PHa0Qzo43vS+YkD7MF93cHWBZb6RhuHj9lbY/YWlFNV19BBlYuIdG9dOktoyZIlvPnmm7z77rt4eno6Xn/mmWdYt24d77//Punp6fzpT3/i7rvvZtWqVS3eZ8GCBZSWljq+srOzu+oRRE4r0NuDn00ZQKC3h+O1HybE8KPE2BbPP29ACADrD7RuX6IjJdU02Ozn2QzYfkStiyJybnBry8mhoaFYLBby853Xj8jPzyciIuK01z755JMsWbKEVatWMWbMGMfr1dXV/PrXv+bdd99l5syZAIwZM4YtW7bw5JNPOrXENLNarVit1raULtItTegXhJvZRE5JNYePVRMT7H3a8w99Zy+ib7JLnBa5ExHprdrUwuLh4cGECROcBsw2D6CdPHnyKa97/PHHefTRR1m5ciUJCQlOx+rr66mvr8dsdi7FYrFgs9naUp5Ij+Pt4caY6ADg1ONYth4ucWya2Dx+pdm3rdjDSESkN2hzl1BycjLPP/88r7zyCjt37uTOO++ksrKSuXPnAjBnzhwWLFjgOP+xxx7j4Ycf5sUXXyQuLo68vDzy8vKoqKgAwN/fn4suuogHHniA1NRUDh48yMsvv8yrr77Ktdde20GPKdJ9NXcLrTtQfNKx/6Qf5qpnv2b28+swDINDhfbgMjTcD2jbwNtGm9Gm6dMiIt1Jm7qEAGbNmsXRo0dZuHAheXl5xMfHs3LlSsdA3KysLKfWkmXLllFXV8f111/vdJ9FixbxyCOPAPDmm2+yYMECbrrpJoqLi+nXrx9/+MMfuOOOO87i0UR6hvMGhPC31P0ntbB8uj2P+e9sBSCnpJp9BRWOFpbvj+3L7k/LOVhYSWl1vdPKuy2pqW/k2r+tpdFm48N7puDhpkWuRaRnaXNgAZg3bx7z5s1r8VhqaqrT94cOHTrj/SIiInjppZfaU4pIj3fiOJa9+eUMDvdjW04p8/61mUabgZvZRIPNYN3BYg41BZb4mCBigr3ILq7m3YzDvLkxG5thMGdyHNeNj8bLw+L0M/751UF25toH6G46VMz5g0JPqkNEpDvT/2aJuJiP1Y2Lh9kXlVu+LhOAP3+2h7oGG5cMC+POaQMBWLe/yLHAXL8Qb8Y0reHyyAc72JVXzp78Cn7z3jYufOxzNh063r1UVFHLstT9ju9T9xztiscSEelQCiwi3UDzhoj/ST9MemYxKbsKMJngNzOHc/5Ae2tIyq586hptuFtMRAZ6MSYqwHF9Yv9gfjNzODHBXhRV1nHTC+sdu0H/JWUvFbUNeLnbW11Sd7d9Sf+iilouWPI5c17coHEwIuIS7eoSEpGOdf7AEAaH+bK3oILbXk0H7KvlDujjS2SgF+4WEzX19llzMcHeWMwmvje6L/9OP8y0IX2Yf8Uw3C1mbkrsx91vZPD5rgJ+9uomvN0tVNY1AvDUDWO5+40M9uRXcKSkmshAr1bXt3xdJjkl1eSUVJORdYwJ/TSVWkS6llpYRLoBk8nELU2tLMWVdQDccZG9K8jT3eJYwh8gLsQHsAeXVckX8ZsrRzg2ZPTysPD3mydw/YRoDANHWJk5pi9XjO5LfIz9Pqm7j1JUUcsLaw6QW3ryPkaGYTjqqKlv5LWmriqA19dlddyDi4i0klpYRLqJa8dF8djKXZTXNHD+wBDGNoULsC/hvynzGGAfv3I67hYzT/5wLL+8bAgNjTa83C308bMvtDhtaBgZWSV89O0RXk07xK68ct5Yn8W7d11AgLd9ppHNZnDPvzbz8bZcfjV9KH18rRRW1OFndaO8toEPv83l4StHEOTjcboyREQ6lFpYRLoJH6sbv7h0MEHe7vzqhE0TASb1D3H8ubmF5UyiAr3oF+JDmL8nJpMJgIuG2Ddg/HpfEbvyygE4UFjJPW9upqHR3uX09y8P8NG3uRgGPPHJbh79cAcA8y4ZxKgof+oabPw7XdthiEjXUmAR6UZ+NmUAmxdOZ3xskNPrE/oFYbZnjjO2sJzO6KgAQppaRvw93fjzrLF4uVv4cs9RfvLSRp76dDdPfLILgEuaZi6V1zbg7WHhxkmx/DixH2Af07L+QBE19Y3trkVEpC0UWER6AF+rGzckxDAozJfx/YLOfMEpmM0mfjZlAHEh3rw0dxLXjovmTzeMxWSCr/YV8tfP92Ez4Lrx0fzzlgR+M3M4ZhPcekF/ArzcuSo+Ej9PN7KLq5n1j3VMePQzpynUIiKdxWT0gjmKZWVlBAQEUFpair+/v6vLEelxtuWUkrq7gI2HjuHn6cYT1491LD5X2dTC0tyttHZfIa9vyGLtvkKOVdVz+9QB/Pp7w1u87+pdBby5MYt7LhnMqBOmYYuIQNs+vzXoVkQYFRVwykDhY3X+NXH+oFDOHxTKa+sy+c1729ibX37SNYZh8LfU/Tz56W4MA7Zkl/DRvVMI9dUu6yLSPuoSEpF2GdK0AePegoqTjv3uwx088Yk9rPh7upFfVss9bxwf2Csi0lYKLCLSLoPDfAE4fKyaytoGx+uFFbUsT7Ov2/L7a0ax4q4L8PGwkHagiGc+3+eSWkWk51NgEZF2CfLxcHTx7D96vJXl3YwcGmwGY6MD+PF5/RgU5stvrx4F2LceOJWGRhuPrdzFp9vzOrdwEemRFFhEpN2aW1n25NsDi2EYvLXJvkbLDRNjHOdNHxmOyQQ5JdUUVtS2eK//bctjWep+fvXvb6hX15GIfIcCi4i02+Bwe2DZW2AfeJuRVcK+ggo83c18f2yk4zx/T3cGhNoXvNt6uKTFe63cZm9ZKatpYMNBTZUWEWcKLCLSboObBt7ua2pheXujvXXle6P74u/p7nRu81YD32SXnnSfmvpGVp+wi/Qn3+kWqm+08eHWIxSU1XRY7SLSsyiwiEi7ObqECsopr6nnw61HAJiVEHPSuc0bOH7TQgvLl3uOUlXXiKVpOd9Pt+dz4hJRj364g3lvbOZ7f/2KLdknXy8ivZ8Ci4i024kzhZ5fc5DKukYGhfkyqX/wSec2t7BsPVzKd9erbO4OunFiDN4eFvLKath62N4Sk555jOVNu0UXVtQy6+9pPPv5XtbsPUppVX1nPZqIdDMKLCLSbiG+VkJ8PDAMWJZqn7J850UDHavinmh4Xz/cLSaKK+s4fKwasA/SrWuw8dnOfACuGRfFtKH2DRo/3ZFHXYONB9/ZimHAVWMjuXRYGLUNNp78dA83/3MDFz72+UndRIZh8MKaA/x70/ENGnccKWPeGxlOs5l6kpe/PsjTq/a4ugwRl9JKtyJyVgaF+VJ0sJj6RoOoQC+uio9s8Tyrm4Xhff3ZeriUbw6X8NwX+/n3psP08bNSXtNAqK+V8bFBHCmp5uNv83hrYzapu4+yt6CCEB8PfnvVSPy93Hl9fSZp+4tYd6CIY1X1fLA1l59e2N/xc9L2F/H7j3ZiMsHo6ACGhvvxf+98w7acMg4fq2bFnedjNp8cqNrrhTUH2FdQwR+uHe3o0upI5TX1/PbDHRgGXD8hmuig9m9+KdKTqYVFRM5K80whgNunDsDdcupfK2Oi7cv/P/XZHl5fn0Vdo42cEntry5Vj+mIxm5g2NAwPi5nCijq2HynDYjbx6DWjCPLxwGI2MWdyHMt+PIH7koYA8ME3R5x+xgtfHQTAMOCpT/fw6Y58tuWUAfYtAlZszqGytoGH3v2WP36886TuKYBjlXWt2om6sraBJf/bxZsbs085+wngtx9s56InVrerC2tnbjnNJWYWVbX5epHeQi0sInJWmpfoD/X1YNbEkwfbnmhsdCCvkcWBo5UA3HvpYBL7B1NUWcelw8IACPByZ+lN49mcdYyhEX6MiwkiNuTkVoUrRkfw2w+2syW7hOziKmKCvdlXUMHnuwowmcAEfLojn29z7GNh4kK8OVRUxZL/7eLltQcdIebHif0c988pqeaJlbt4b8sRpo8I5x9zEk77PJsyj9Fgs6eJQ0WVjItteSftFRk5lFbXszn7GNOGhp32nt+1/cjxWVWHiiq5YFBom64X6S0UWETkrFw9Nop1B4q4fkI0nu6W057bPPAWYNrQPtx36eAWu2cuGxHOZSPCT3uvMD9PzhsQwtr9RXz0bS53XDSQF7+2t64kDQ/H39OddzIOk1tag5/VjbfvmMwNz6VxqKjKafG6dQeKiA3xZv2BIua8uIHaBvuidSm7CqiobcDXeupfk+sOFDn+fKiw5daPkqo6SqvtLSt5pW2flt0crACy1MIi5zB1CYnIWQnwdudvN03gkmGnDxgAA/v4MjoqgEFhvjx1Q/xZjyW5cox9vMwH3xxh/9EKVmTYl/7/6YX9uS9pMG5N9//plP6E+XnyyFUjcTObGNjHhx+MiwIgrSl0PL/mALUNNsbFBhLmZ6XRZpCReey0Pz9t//HAkllkbzXKLq5i5l/X8N7mnKbXj4eMvHasI/PdFhaRc5UCi4h0GYvZxPvzLuCT+6YS7ONx1ve7fFQEFrOJ7UfKuOypL6iptzEqyp/E/sHEBHvzyFUjuTo+kp9NGQDAtKFhfP3gJXxy31SumxAN2FtJKmsb+HJvIQB/vHY0Fw62d7usP1jU8g8GKmobHN1NAAebgsl/t+Sw/UgZr6QdAiCz+ITA0sYWltqGRvadsBu2xrDIuUyBRUS6lMlk6rDZNME+HkxpChc2w97N9Ozs8Y5p1T8+rx9/uXGcU7dOuL8nbhYz42OD8LCYyS2t4dW0TOoabMQGezMswo/z+ocAsP7AqbcI2HiomEabgUfTIOPmFpadufZtCvbklWMYBpmFx1tF2trCsievggabQfNfV1ZxVYuDhEXOBQosItKj/f6aUSRfNoSP753Cy3MnEde0Z9GZeHlYiG8aU7N0tX0NmRkjwzGZTCQOsC98983hkpNmC+WV1tDQaHOMX5k+0t4VVlJVT0lVHTty7WNOKusaySmpPqsWlm1N3UEJccGYTVBV18jRU2weKdLbKbCISI8WHeTNvZcOZkSkf5uvPa8pmFTUNgAwY2QEALHB3oT7W6lvNMjIOj6O5cOtRzhvcQrf++sax+q8lwwLI9zfCtgXqDtxnMne/ApHywu0vYWlefzKuJhAIgO9AHULyblLgUVEzlnnDQhx/DnU1+qYlmwymUhs6hY6cefofzat8bInv8IRHM4bEEK/EHurzsrteZzYY7M7v9wpYJRU1bdqfZdm24/YW2tGRgXQr2nqtQKLnKsUWETknDW+X5BjDMplI8KcxtY0dws1j2PZV1DB5qwSLGYT10+IxmSCkZH+RAZ6EdcUJv63zXmX6W+ySygot3fhNM9Yym9lK0ujzWBX03iYkZH+jlCUqZlCco5SYBGRc5anu4WpQ/pgMsE18VFOxxKbNnDMyDpGaVU97zRNmZ42pA9P/nAsXz5wMf+6/TwAR5g42hROmjeF/HLPUcC+GF5MsD3U5LZyHMvBwgqq6xvx9rDQP8THEYq+28KyM7es1SFIpCdTYBGRc9pTs8ay8hdTSTyhewjsa8YMCPWhtsHGvH9lONZ4+WGCfTp0TLA3/p7uAPT/zkDfa5rWeKmss3f/xIV4O8a5tDZc7G9aDXhwmC9ms4nY4JNbWLYfKeXKZ77ixy+s1+wh6fUUWETknObv6c7QCL+TXjeZTDz7o/F4e1hYs7eQ/LJagrzdW1wgr993tg6YMTICq9vxX6+xIT70DbAPmm1tC0tu0x5LzYNt40KbWlhOmHX0+vosGm0Gewsq2J1f3qr7ivRUCiwiIqcwItKfp24Y6/j+6vgoPNxO/rXZ3CUE4O1hYUCoj9OmkPYWFk/g+NTmnbllHDhaccqWkeZg0xx0Ypu6lEqq6imtqqeytoH3txzf+HHVjvx2PaNIT6G9hERETuPyUX155Psj+E/GYW69oH+L5/ha3Qj1tVJYUcvwvv6YzSaGhPk59gGKDfamsmnqdH5ZDWn7i5j9/DrAPr4lLtQHX6uF2GAfFn1/BJ7uFo44Aos96Hh7uBHmZ6WgvJbM4kp25ZY7pmMDrNpZwLxLBnfa34OIqymwiIicwU8u6M9PThFWmvUP9aawopYRfe3rwQw5oZspLtSHooo6wN5y8v43x1tGSqvr+Sa7BICvKeKiIX24fFSEo0uob6DnCT/Dh4LyWh5buYuSKvuGinMviOOlrw+xJbuEgvIawvyOny/Sm6hLSESkA4xvWsPl/IH2wbtDTugS6hfsTUTA8S6hz3fZu29emJPAB/Mu5Pk5CUxuGvR7oNC+d9B3u4QA5l0yCC93C1/vK2L7kTLczCbumjaIMdEBAKzeVdCZj3hajTYN+pXOpcAiItIBHpgxlJT7L+LyUfbVckdGBmAxmwj28aCPn5WI5jEsZTXkl9Xi7WHhwsGhjI4O4LIR4Y5F7A4VVtJoMxyr4kae0MIyZXAf3rv7AsespBkjI+jjZyVpuH0g8KqdxwPLu5sPc94fU3jqsz2O19YdKOLBd7byw+fWMv3PXzhads7W7rxyJv1hFbe9uqlD7ifSEnUJiYh0ADeLmYF9jreqhPt78uqtkwjwcsdkMtHHz4rFbHK0REwd3AdPd4vj/OZZQIeKqjhaXkujzcBiNp3UxTM0wo//zruA1bsKuGRYGACXDg/jqc/28OWeo/zjy/0UVdTx9y8PAPDXlL34e7oR4OXO/He2cmJDyDOf7+OFWxLO6rnLauq547V0iirr+GxHPumZx5jQL+is7inSErWwiIh0kgsGhTIqyt5dYzGb6ONrdRy7dHiY07lxTTONDhVWcqTUPn4lvCnkfJe/pztXx0fh17QOzIi+/gzoY18z5o8f73KElQsG2Vttfv/RTh74jz2sfG90BI98fwQAq3cXUPCddWFeTTvEj55fx9p9hWd8PsMw+NXb33DwhB2p//Hl/jNeJ9IeCiwiIl0kvGkci8mEo3WkWfMu0wXltewvsI9j6RvoRWuYTCb+c8f5PHr1SM4fGEKYn5WnZ8Xz2k8TuWVyP8d5P7uwP8/OHs9PLujP+NhAGm0GKzbnOI5nF1fxuw92sHZ/ET96YT3/959vKKmqO+XPfXtTNp/uyMfDYuZPP7RP//50R75TgBHpKAosIiJdpG/TOJbxsUGEnNDaAvbpzcE+HgCkHSiynx/Q+hk/wT4e3Dw5jjduO48NDyVxzbgoTCYTC78/kgVXDONPPxzLQzOHY25qsZk1MQaAtzdmO9aCeXrVXhpsBmF+9tre3nSYpKe+4MOtR05aL8YwDF786hAAv7xsCNdNiObSYWEYBryw5oDjvILyGn751hY2HipG5GwosIiIdJH42EAArh0X1eLx5v2C0vbbA0tkK1tYTsdiNvHziwZy3YRoTKbj3Uszx0Ti5W7hQGEl6ZnH2FdQzrub7dsP/GNOAv+5YzKDwnwprKhj3hub+dOne5zuuynzGLvzy/F0N/OjxFgAbps6AID/pB+msMK+r9IzKft4d3MOf//iACJnQ4FFRKSL/PTC/qy8bwo3NX3Af1fzOJbc7ywa1xl8rW7MHNMXgLvfyOD25enYDLhsRDjxMYEkxAXz0b0XcsdFAwF4c2O20/XL0zIBuHpsFAFe9rE0if2DGRsdQG2DjVfTMqmqa+C9pi6n5gAj0l7tCixLly4lLi4OT09PEhMT2bBhwynPff7555kyZQpBQUEEBQWRlJTU4vk7d+7kqquuIiAgAB8fHyZOnEhWVlZ7yhMR6ZbcLWaGRfg7tXScKO47myieuAZLZ/jJ+XF4uJnJL6vlwNFKTCa4f/oQx3Grm4V7Lx2EyWQPHM27URdW1PK/bbkA3HzCGBmTycTtU+0BZ3naId7emE1502q8xZWnHgsj0hptDixvvfUWycnJLFq0iIyMDMaOHcuMGTMoKGh5waLU1FRmz57N6tWrSUtLIyYmhunTp5OTc3yg1/79+7nwwgsZNmwYqampbN26lYcffhhPT63YKCLnju8GlhPXYOkMo6IC2PhQEq/cOonky4bw7OzxDIvwdzrH28PN0fKzO8++weJbG7OpbzQYGxPomAXV7PJREcQEe3Gsqp4/frzL8boCi5wtk9HGPckTExOZOHEizz77LAA2m42YmBjuueceHnzwwTNe39jYSFBQEM8++yxz5swB4MYbb8Td3Z3ly5e34xGgrKyMgIAASktL8ff3P/MFIiLd0LeHS/n+s185vt/4UBJ9/KynuaJr3PlaOv/blsdvZg7nZ1MGMPOva9h+pIzHrxvDDU2Dd0/0ytpDLHp/O4DT2jO7f385VjfLSefLuastn99tamGpq6sjPT2dpKSk4zcwm0lKSiItLa1V96iqqqK+vp7g4GDAHng++ugjhgwZwowZMwgLCyMxMZH33nvvlPeora2lrKzM6UtEpKfr17R4HICHxUxI06whV2tuddmZW05ZTT07cu2/cy8a2qfF83+YEE2gt31cy/dG98WtaWaSWlnkbLQpsBQWFtLY2Eh4eLjT6+Hh4eTl5bXqHvPnzycyMtIRegoKCqioqGDJkiVcfvnlfPrpp1x77bX84Ac/4IsvvmjxHosXLyYgIMDxFRNzcsIXEelp/D3dHSElIsDTMQXZ1Yb1tW/kuCuvjPRDxzAM+4ymcP+Wu6y8Pdz49feGM7CPD/MuHkRQ0zM1bwAp0h5dujT/kiVLePPNN0lNTXWMT7HZbABcffXV/PKXvwQgPj6etWvX8txzz3HRRReddJ8FCxaQnJzs+L6srEyhRUR6hbhQH4oq6zp1hlBbDW9qYdmbX8Ha/fYVcBP7h5z2mhsSYrghwf57OcTHg6PltWphkbPSphaW0NBQLBYL+fn5Tq/n5+cTERFx2muffPJJlixZwqeffsqYMWOc7unm5saIESOczh8+fPgpZwlZrVb8/f2dvkREeoPmAa4dsQZLR4kO8sLHw0Jdo40VGfYJE5P6B7f6+uYF8RRY5Gy0KbB4eHgwYcIEUlJSHK/ZbDZSUlKYPHnyKa97/PHHefTRR1m5ciUJCc4bbXl4eDBx4kR2797t9PqePXvo168fIiLnkub9f9oSCDqb2WxiaIS9W6ioKXS0pb7mVX21FoucjTZ3CSUnJ3PLLbeQkJDApEmTePrpp6msrGTu3LkAzJkzh6ioKBYvXgzAY489xsKFC3njjTeIi4tzjHXx9fXF19e+s+kDDzzArFmzmDp1KhdffDErV67kgw8+IDU1tYMeU0SkZ/jB+GguHRZOQNOg1e5iWF9/MrJKAIgM8CQ6qPUtQCFqYZEO0ObAMmvWLI4ePcrChQvJy8sjPj6elStXOgbiZmVlYTYfb7hZtmwZdXV1XH/99U73WbRoEY888ggA1157Lc899xyLFy/m3nvvZejQobzzzjtceOGFZ/FoIiI9U3cLKwDDm1pYwN66cqrF71qiLiHpCO0adDtv3jzmzZvX4rHvtoocOnSoVfe89dZbufXWW9tTjoiIdLJhfY+PFZx0hgG339UcWIoUWOQsaC8hERE5o6ERfjQ3qiQOaNv4GnUJSUfo0mnNIiLSM/l7uvO7q0dRWdvAwD6+bbpWXULSERRYRESkVW4+r30zN0N8mxeO0ywhaT91CYmISKcK8bFPay6raaCuwebiaqSnUmAREZFOFeDljqVpm4FjVeoWkvZRYBERkU5lNpsIapqqrf2EpL0UWEREpNNp4K2cLQUWERHpdMfXYtHAW2kfBRYREel0zQNv1cIi7aXAIiIinU5dQnK2FFhERKTTOdZiUWCRdlJgERGRTudYnl+zhKSdFFhERKTTBWsMi5wlBRYREel0zWNYCjVLSNpJgUVERDrd8f2E1MIi7aPAIiIinS4y0AuA0up6ymvqXVyN9EQKLCIi0ul8rW6OgbdZxVUurkZ6IgUWERHpEjHB3gBkK7BIOyiwiIhIl+gXYg8smUUKLNJ2CiwiItIlYptaWNQlJO2hwCIiIl1CgUXOhgKLiIh0CQUWORsKLCIi0iVim8aw5ByrpqHR5uJqpKdRYBERkS4R7ueJh5uZBptBbmmNq8uRHkaBRUREuoTZbCImyL6AnLqFpK0UWEREpMv0C/EBNLVZ2k6BRUREuowG3kp7KbCIiEiXOR5YKl1cifQ0CiwiItJl1MIi7aXAIiIiXebE5fkNw3BxNdKTKLCIiEiXad4AsbymgdLqehdXIz2JAouIiHQZT3cLEf6eAGzLKXNxNdKTKLCIiEiXunhYGADvbs5xcSXSkyiwiIhIl7p+QhQA/9uWS2Vtg4urkZ5CgUVERLrU+Ngg+of6UFXXyMff5rq6HOkhFFhERKRLmUwmrp8QDcB/0g+7uBrpKRRYRESky107LgqTCdYfLCZba7JIKyiwiIhIl4sM9OLCQaEAvL0p28XVSE+gwCIiIi4xa2IMAG9tzKa+0ebiaqS7U2ARERGXmD4iglBfDwrKa0nZme/qcqSbU2ARERGX8HAzc0OCvZXl9fVZLq5GujsFFhERcZnZk2IxmWDN3kIOFrZuB+eGRpv2IToHKbCIiIjLxAR7M21IHwD+teHMrSyNNoOrl35N0lNfaNzLOaZdgWXp0qXExcXh6elJYmIiGzZsOOW5zz//PFOmTCEoKIigoCCSkpJOe/4dd9yByWTi6aefbk9pIiLSwzQPvl3VinEse/LL2X6kjP1HK8nSdOhzSpsDy1tvvUVycjKLFi0iIyODsWPHMmPGDAoKClo8PzU1ldmzZ7N69WrS0tKIiYlh+vTp5OScvIfEu+++y7p164iMjGz7k4iISI80eUAoJhMcOFpJQXkNAIUVtby3OYcXvzrIP77cT2mVfWfnTYeKHdcpsJxb2hxYnnrqKW677Tbmzp3LiBEjeO655/D29ubFF19s8fzXX3+du+66i/j4eIYNG8YLL7yAzWYjJSXF6bycnBzuueceXn/9ddzd3dv3NCIi0uMEeLszLMIfgA0H7YHk9lc3cd9bW/jdhzv448e7+POqPQBsyjzmuE4Lzp1b2hRY6urqSE9PJykp6fgNzGaSkpJIS0tr1T2qqqqor68nODjY8ZrNZuPmm2/mgQceYOTIkW0pSUREeoHE/vbPhA0HizlUWElGVgkWs8mxuNwn2/MwDINNh44HlqwiBZZzSZsCS2FhIY2NjYSHhzu9Hh4eTl5eXqvuMX/+fCIjI51Cz2OPPYabmxv33ntvq+5RW1tLWVmZ05eIiPRczYFl/YFiPmraEPH8gSG8cEsCPh4Wcktr+GR7Pjkl1Y5r1CV0bunSWUJLlizhzTff5N1338XT0xOA9PR0/vKXv/Dyyy9jMpladZ/FixcTEBDg+IqJienMskVEpJNNagosu/PLHUv1XzmmL57uFqYNDQPg8ZW7ADA3fVS0JrC8tzmHy5/+kv1HKzqhaulKbQosoaGhWCwW8vOdR3Ln5+cTERFx2muffPJJlixZwqeffsqYMWMcr69Zs4aCggJiY2Nxc3PDzc2NzMxM7r//fuLi4lq814IFCygtLXV8ZWdrHwoRkZ4sxNfKoDBfADKLqnAzm5g+wv65Mn2kvVX/QNM6LRc0dRNlF1edcT2W19dnsiuvnFfXHjrp2KHCStYfKOqoR5BO1qbA4uHhwYQJE5wGzDYPoJ08efIpr3v88cd59NFHWblyJQkJCU7Hbr75ZrZu3cqWLVscX5GRkTzwwAN88sknLd7ParXi7+/v9CUiIj1bc7cQ2ENJkI8HABcPC8PdcrwF/pr4KAAq6xo51jR76FQOFtpbYT7bke8UbgzDYM6LG7jx+XXszivvsGeQztPmLqHk5GSef/55XnnlFXbu3Mmdd95JZWUlc+fOBWDOnDksWLDAcf5jjz3Gww8/zIsvvkhcXBx5eXnk5eVRUWFvngsJCWHUqFFOX+7u7kRERDB06NAOekwREenuJp0QWGaO6ev4s7+nO5MHhjq+v2BQKBH+9mEF3+0W2nq4hLxS+9To8pp6CitqAThSWsP2I8fHO+4rqCCruArDgDV7j3b8w0iHa3NgmTVrFk8++SQLFy4kPj6eLVu2sHLlSsdA3KysLHJzcx3nL1u2jLq6Oq6//nr69u3r+HryySc77ilERKTHO29ACG5mE1Y3M9NHOE/umNHULRQV6EVEgCexwd6Ac2BZs/coVz37Nbe9ugmAQ4XOYebT7ccnh3y1r9Dx5+ap1NK9ubXnonnz5jFv3rwWj6Wmpjp9f+jQoTbfvz3XiIhIzxbu78krt07C6mYm0NvD6dh146PZmVvGRUPsA3Bjgr3ZcKjYsRZLo83gDx/tBGDbkVIqahs4WGQf82I2gc2AT3fkkzzd3nL/9b7jY1c2HCrGZjMwm1s38UNcQ3sJiYhIt3HBoFAS4oJPet3T3cLvrxnNZU0tL44Wlqa1WN7JOMyuprEohgG7css41DRI99Lh4VjMJnbllZNVVEVDo411Jwy2LamqZ2+BZhF1dwosIiLS48SGeAH2LqGqugb+9OluADzc7B9r24+UOXZ/HhcbyKSmEPTJ9jy+OWxvgQn0duf8gSEArD/Y9tlC/92Swzvph8/6WaR1FFhERKTHiQk6Poblmc/3kV9WS3SQFz85Pw6A7UdKHYGlf4gPV4y2T5F+5vO9vLHeviv0+QNDmDygObC0bRxLYUUt9721hV/95xuOltd2xCPJGSiwiIhIj9PcJXSktJq/f7EfgIevHMG4mEAAduSWcahpDEtcqA+zJsYwoV8QZTUNvJNhbxW5YFAoic2B5UDxGdd0OdHGg8UYRlP3U55WW+8KCiwiItLj9PGzYnUzYxj2AbXXT4hmxsgIRkYGALAzt5ySpjVa4kJ8sLpZWHbTeML9rY57XDgolDHRAXi4mSmsqHUsTNcaJ7bIaB2XrqHAIiIiPY7JZHK0skQHebHo+yMAiAn2ws/qRqPN3lrSN8ATLw8LAGH+nvz95gS83C2MivInNtgbT3eLo1UmbX/rx7GcOBV6Z+6ZA8u6A0U8/N42ympOv9CdnJoCi4iI9EhXjIog0Nudp2fF4+fpDtiDzPDI46uf9w/1cbomPiaQrx+8hH///HzH/nUXDe0DwEdbc/mujKxj/DVlLyu35TkWpCurqWfnCd1AzV1CDY02Pt+VT019o9M91h0o4pYXN7B8XSZvb9RWMu3VrnVYREREXC15+lDuSxpy0vopIyP9HS0gcd8JLADBPs5rvFw1NpLHV+5m3cEijpRUExlon4FUWFHL3Jc2UlptbxUxm+Cx68YQ6mvFMMDP6kZ5bQN78ytoaLTxjzUHeHzlbm6b0p+HZtpbfLbllPKzVzZR22AD7C0zP5syoGP/Is4RamEREZEeq6XF3prHsYB9htCZRAd5M6l/MIYB739zxPH6Hz/aSWl1PVGBXgwN98NmwO8+3MH/ttlbYmaMisDHw0Jdo42DhZWOFpqV2/MwDINGm8HPl6dTUdvgaOnZ2LRInbSdAouIiPQqI/qeukvoVJo3VHxvcw4Aa/cVsmJzDiYTLL1pPB//YgojI/0pr2ng7U32WUaJ/YMZEuEHQMquAsdeRdnF1RwqqmLDwWJySqoJ9HZnxZ3n4+Vu4VhVPfuPnn6RugNHK1iw4ls2Hur8LQOq6xq5Y3k6z36+t9N/1tlSYBERkV5lcLgv1qYF5AaG+bbqmpmj++JhMbMrr5wX1hzggf9sBeDm8/oRHxOIxWzikatGOl0zqX8wwyLs4eiFNQedjn2xu8DREnPZ8HCCfDwYFxsIHJ9htK+gnP9uyWFZ6n7+/sV+0jOL+WhrLlc9+zX/2pDFPW9sprbBeTzMf9IPk/TUF6w/0PaF7lqyYvNhVm7P468p+6hr6rbqrjSGRUREehV3i5k/3TCWvNKaVrewBHi7c/GwPnyyPZ/fN+1JFBngya9mDHWcMzEumKvGRvL+N0cI97cSG+zN8L72FpbmXaGjAr3IKalm9e6j7Mi1t7h8b7R95+lJ/YNZu7+IjYeKGR8bxFXPfkXDabqH8spqWJGRw+xJsQC8vj6Th97dBsAfP97Je3df4Bg43JIjJdX85KUNXDo8nPmXDzvpuGEYLE/LBKCu0cae/HJGRQWcdF53oRYWERHpda4cE9nmwa03JfYDwN/TjXkXD+KDey7Ev2n2UbPfzBzORUP68MukIZhMJkcLy4nHAb7Yc5Sj5bX4ebpx/iD74nTN2wNsOFjMHz7eQYPNYECoDz8YF8WMkeEE+3jgZjZx25T+LLjCHjCWpe6nodHGC2sOOMIKwDeHS884DftPn+5hT37FKbcPSM885th/CewDhAGKK+v4T/ph6hu7V4uLWlhERESAqUP68OUDFxPi64GPteWPx7CmHaWbDW0awwIwoI8Pl4+KINzfSn6ZvcXlsuHhWN3s68CMiw3CzWwit7SG3NIaPCxmXrl1EjFN68nYbAZ1jTY83S1U1TXw9y8PkFVcxfef/ZqdTa01t08dQE19I6+mZbLsi/2cPyi0xTp355WzYrM9qBytqKW+0Ya7xbmNYvk6e+uKm9lEg83g25xSbgQWvb+dD745Qk19Iz8+r19b/xo7jVpYREREmsSGeJ8yrLQkwMudqKZp0JcND8dkMnHRkD6O41c0dQcBeHlYGB19vMtl7gVxjrAC9hlPnu72cOPt4catF8QBsDO3DHeLiYVXjmDBFcO4bcoALGYTa/YWOlpFvuuJT3bRvNOAYUB+WY3T8cKKWj7+1j7G5qdT+gP2Fpb6RhupuwoA2Hq4pNV/D11BgUVEROQsXDmmL75WN66fEA3ARUPCAPDxsDBlsHMLSHO3ULCPB3ddPOi09715chwD+vgwKMyXd+48n1sv7I/JZCIm2Jsrx9iD0PNrDpx03aZDxazaWYDZBH6e9vDVvOhds+dS91PfaBAfE8jsifYxMjvzyll/oJjy2gag+205oC4hERGRs7Dge8N58IphjgGwl40I5+bz+jG+X6CjxaTZTYn92Jxdwu1TBhDg5d7S7RwCvNz57JcXYWlhrZmfXtif/245wspteVTUNuDb1Cpksxk82jRo+IaEGA4UVrLhYDG5JwSWrYdLePFr+6ymX1w6mH4h3vh5ulFe0+AUgPbkV2CzGS2udeMKamERERE5SyfO1vFwM/PoNaO4dlz0SefFhnjz9s8nkzQivFX3bSmsAIyOCmBAqA+1DTZW7ch3vP7elhy+yS7Bx8NC8vQh9A3wBCC3tBqA+kYb//efrdgMuDo+kouHhWEymRjVtNjeF3uOOu5VXd/I4WPVraqzKyiwiIiI9DAmk4krx0YC8EHT6ryVtQ08tnIXAHdfMogwP08iHIHF3sLy0tcH2ZVXTpC3OwuvHOG435gTxtaYTDjG5ew6Yc8kV1NgERER6YG+3zSO5cu9RympquOvKXvJL6slJtiLWy+wD6SNDLAHj9wSe2D5+Ns8wL4PU4iv1XGvE9dfGRMVwKT+9rE2e/Lt41gau8F2AgosIiIiPdDgcD+GRfhR32hw1+sZ/P1L+/iTh743wjF2xtHCUlaDYRjsbQog5zUFkmajTwgsFw0NY0i4fbr27vwKGm0GM57+kgf+/Y1jgTxXUGARERHpob7f1C20tmkRufuSBnP5qAjH8eYxLHml1eSUVFNZ14i7xXTSLtb9Qrwdu1hfPLQPw5rWl9mTV87qXQXsK6jgs535jsG9rqDAIiIi0kN9f0yk4893ThvILy4d7HS8b1OXUEF5LTuaNmccEOp70iJyJpOJZTeN56kbxjIuNsixqeP+oxX88yv7jKJZCTEnzXrqSprWLCIi0kPFhnjzxPVjqG2wcVNi7El7C4X4eOBuMVHfaPDVvkIARxj5rsQBIY4/RwZ44mt1o6K2gbQDRZhMuHzVWwUWERGRHuyHCTGnPGY2m4gI8CS7uJrU3fYpy0PDz7yDtclkYki4LxlZJQBcOizMaVVeV1CXkIiISC/W19/eLZRVXAXYB+u2xtATNnacMzmuw+tqKwUWERGRXqx5plCzoa0MLMP72s8bEOrDhafYZLErqUtIRESkF+sbeDywWN3Mre7auW58NPsKKrh2XFS3WJ5fgUVERKQX6+t/PLAMDvc95XL/3+VjdeN3V4/qrLLaTF1CIiIivVjfpmX2AceCcD2RAouIiEgv1veEMSwKLCIiItItnTjotrUDbrsjBRYREZFeLNTHiq/VDZMJhvf1P/MF3ZQG3YqIiPRiZrOJf9w8gbKa+pOmOPckCiwiIiK93PndYB2Vs6UuIREREen2FFhERESk21NgERERkW5PgUVERES6PQUWERER6fYUWERERKTbU2ARERGRbk+BRURERLo9BRYRERHp9toVWJYuXUpcXByenp4kJiayYcOGU577/PPPM2XKFIKCgggKCiIpKcnp/Pr6eubPn8/o0aPx8fEhMjKSOXPmcOTIkfaUJiIiIr1QmwPLW2+9RXJyMosWLSIjI4OxY8cyY8YMCgoKWjw/NTWV2bNns3r1atLS0oiJiWH69Onk5OQAUFVVRUZGBg8//DAZGRmsWLGC3bt3c9VVV53dk4mIiEivYTIMw2jLBYmJiUycOJFnn30WAJvNRkxMDPfccw8PPvjgGa9vbGwkKCiIZ599ljlz5rR4zsaNG5k0aRKZmZnExsae8Z5lZWUEBARQWlqKv3/P3YlSRETkXNKWz+82tbDU1dWRnp5OUlLS8RuYzSQlJZGWltaqe1RVVVFfX09wcPApzyktLcVkMhEYGNji8draWsrKypy+REREpPdq027NhYWFNDY2Eh4e7vR6eHg4u3btatU95s+fT2RkpFPoOVFNTQ3z589n9uzZp0xbixcv5re//e1Jryu4iIiI9BzNn9ut6expU2A5W0uWLOHNN98kNTUVT0/Pk47X19dzww03YBgGy5YtO+V9FixYQHJysuP7nJwcRowYQUxMTKfULSIiIp2nvLycgICA057TpsASGhqKxWIhPz/f6fX8/HwiIiJOe+2TTz7JkiVLWLVqFWPGjDnpeHNYyczM5PPPPz9tX5bVasVqtTq+9/X1JTs7Gz8/P0wmU1se6YzKysqIiYkhOzu7146P6e3P2NufD/SMvUFvfz7QM/YGHf18hmFQXl5OZGTkGc9tU2Dx8PBgwoQJpKSkcM011wD2QbcpKSnMmzfvlNc9/vjj/OEPf+CTTz4hISHhpOPNYWXv3r2sXr2akJCQtpSF2WwmOjq6Tde0lb+/f6/8j+9Evf0Ze/vzgZ6xN+jtzwd6xt6gI5/vTC0rzdrcJZScnMwtt9xCQkICkyZN4umnn6ayspK5c+cCMGfOHKKioli8eDEAjz32GAsXLuSNN94gLi6OvLw8wN4q4uvrS319Pddffz0ZGRl8+OGHNDY2Os4JDg7Gw8OjrSWKiIhIL9PmwDJr1iyOHj3KwoULycvLIz4+npUrVzoG4mZlZWE2H598tGzZMurq6rj++uud7rNo0SIeeeQRcnJyeP/99wGIj493Omf16tVMmzatrSWKiIhIL9OuQbfz5s07ZRdQamqq0/eHDh067b3i4uJaNTrYVaxWK4sWLXIaM9Pb9PZn7O3PB3rG3qC3Px/oGXsDVz5fmxeOExEREelq2vxQREREuj0FFhEREen2FFhERESk21NgERERkW5PgeUMli5dSlxcHJ6eniQmJrJhwwZXl9QuixcvZuLEifj5+REWFsY111zD7t27nc6ZNm0aJpPJ6euOO+5wUcVt98gjj5xU/7BhwxzHa2pquPvuuwkJCcHX15frrrvupFWbu7O4uLiTns9kMnH33XcDPfP9+/LLL/n+979PZGQkJpOJ9957z+m4YRgsXLiQvn374uXlRVJSEnv37nU6p7i4mJtuugl/f38CAwP56U9/SkVFRRc+xemd7hnr6+uZP38+o0ePxsfHh8jISObMmcORI0ec7tHSe79kyZIufpKWnek9/MlPfnJS7ZdffrnTOT35PQRa/HdpMpl44oknHOd05/ewNZ8Prfn9mZWVxcyZM/H29iYsLIwHHniAhoaGDqtTgeU03nrrLZKTk1m0aBEZGRmMHTuWGTNmUFBQ4OrS2uyLL77g7rvvZt26dXz22WfU19czffp0Kisrnc677bbbyM3NdXw9/vjjLqq4fUaOHOlU/1dffeU49stf/pIPPviAf//733zxxRccOXKEH/zgBy6stm02btzo9GyfffYZAD/84Q8d5/S096+yspKxY8eydOnSFo8//vjj/PWvf+W5555j/fr1+Pj4MGPGDGpqahzn3HTTTWzfvp3PPvuMDz/8kC+//JLbb7+9qx7hjE73jFVVVWRkZPDwww+TkZHBihUr2L17N1ddddVJ5/7ud79zem/vueeerij/jM70HgJcfvnlTrX/61//cjrek99DwOnZcnNzefHFFzGZTFx33XVO53XX97A1nw9n+v3Z2NjIzJkzqaurY+3atbzyyiu8/PLLLFy4sOMKNeSUJk2aZNx9992O7xsbG43IyEhj8eLFLqyqYxQUFBiA8cUXXzheu+iii4xf/OIXrivqLC1atMgYO3Zsi8dKSkoMd3d349///rfjtZ07dxqAkZaW1kUVdqxf/OIXxsCBAw2bzWYYRs9//wDj3XffdXxvs9mMiIgI44knnnC8VlJSYlitVuNf//qXYRiGsWPHDgMwNm7c6Djnf//7n2EymYycnJwuq721vvuMLdmwYYMBGJmZmY7X+vXrZ/z5z3/u3OI6QEvPd8sttxhXX331Ka/pje/h1VdfbVxyySVOr/WU99AwTv58aM3vz48//tgwm81GXl6e45xly5YZ/v7+Rm1tbYfUpRaWU6irqyM9PZ2kpCTHa2azmaSkJNLS0lxYWccoLS0F7NsfnOj1118nNDSUUaNGsWDBAqqqqlxRXrvt3buXyMhIBgwYwE033URWVhYA6enp1NfXO72fw4YNIzY2tke+n3V1dbz22mvceuutTht+9vT370QHDx4kLy/P6T0LCAggMTHR8Z6lpaURGBjotEdZUlISZrOZ9evXd3nNHaG0tBSTyURgYKDT60uWLCEkJIRx48bxxBNPdGhTe2dLTU0lLCyMoUOHcuedd1JUVOQ41tvew/z8fD766CN++tOfnnSsp7yH3/18aM3vz7S0NEaPHu1Y9R5gxowZlJWVsX379g6pq10r3Z4LCgsLaWxsdPrLBwgPD2fXrl0uqqpj2Gw27rvvPi644AJGjRrleP1HP/oR/fr1IzIykq1btzJ//nx2797NihUrXFht6yUmJvLyyy8zdOhQcnNz+e1vf8uUKVPYtm0beXl5eHh4nPQhEB4e7ti7qid57733KCkp4Sc/+YnjtZ7+/n1X8/vS0r/B5mN5eXmEhYU5HXdzcyM4OLhHvq81NTXMnz+f2bNnO20sd++99zJ+/HiCg4NZu3YtCxYsIDc3l6eeesqF1bbO5Zdfzg9+8AP69+/P/v37+fWvf80VV1xBWloaFoul172Hr7zyCn5+fid1N/eU97Clz4fW/P7My8tr8d9q87GOoMByDrr77rvZtm2b0/gOwKnPePTo0fTt25dLL72U/fv3M3DgwK4us82uuOIKx5/HjBlDYmIi/fr14+2338bLy8uFlXW8f/7zn1xxxRVOW7L39PfvXNe8a71hGCxbtszpWHJysuPPY8aMwcPDg5///OcsXry42y8Bf+ONNzr+PHr0aMaMGcPAgQNJTU3l0ksvdWFlnePFF1/kpptuwtPT0+n1nvIenurzoTtQl9AphIaGYrFYThoFnZ+fT0REhIuqOnvz5s3jww8/ZPXq1URHR5/23MTERAD27dvXFaV1uMDAQIYMGcK+ffuIiIigrq6OkpISp3N64vuZmZnJqlWr+NnPfnba83r6+9f8vpzu32BERMRJg+AbGhooLi7uUe9rc1jJzMzks88+c2pdaUliYiINDQ1n3KutOxowYAChoaGO/y57y3sIsGbNGnbv3n3Gf5vQPd/DU30+tOb3Z0RERIv/VpuPdQQFllPw8PBgwoQJpKSkOF6z2WykpKQwefJkF1bWPoZhMG/ePN59910+//xz+vfvf8ZrtmzZAkDfvn07ubrOUVFRwf79++nbty8TJkzA3d3d6f3cvXs3WVlZPe79fOmllwgLC2PmzJmnPa+nv3/9+/cnIiLC6T0rKytj/fr1jvds8uTJlJSUkJ6e7jjn888/x2azOQJbd9ccVvbu3cuqVasICQk54zVbtmzBbDaf1JXSExw+fJiioiLHf5e94T1s9s9//pMJEyYwduzYM57bnd7DM30+tOb35+TJk/n222+dwmdz+B4xYkSHFSqn8OabbxpWq9V4+eWXjR07dhi33367ERgY6DQKuqe48847jYCAACM1NdXIzc11fFVVVRmGYRj79u0zfve73xmbNm0yDh48aPz3v/81BgwYYEydOtXFlbfe/fffb6SmphoHDx40vv76ayMpKckIDQ01CgoKDMMwjDvuuMOIjY01Pv/8c2PTpk3G5MmTjcmTJ7u46rZpbGw0YmNjjfnz5zu93lPfv/LycmPz5s3G5s2bDcB46qmnjM2bNztmyCxZssQIDAw0/vvf/xpbt241rr76aqN///5GdXW14x6XX365MW7cOGP9+vXGV199ZQwePNiYPXu2qx7pJKd7xrq6OuOqq64yoqOjjS1btjj922yeWbF27Vrjz3/+s7FlyxZj//79xmuvvWb06dPHmDNnjoufzO50z1deXm786le/MtLS0oyDBw8aq1atMsaPH28MHjzYqKmpcdyjJ7+HzUpLSw1vb29j2bJlJ13f3d/DM30+GMaZf382NDQYo0aNMqZPn25s2bLFWLlypdGnTx9jwYIFHVanAssZPPPMM0ZsbKzh4eFhTJo0yVi3bp2rS2oXoMWvl156yTAMw8jKyjKmTp1qBAcHG1ar1Rg0aJDxwAMPGKWlpa4tvA1mzZpl9O3b1/Dw8DCioqKMWbNmGfv27XMcr66uNu666y4jKCjI8Pb2Nq699lojNzfXhRW33SeffGIAxu7du51e76nv3+rVq1v87/KWW24xDMM+tfnhhx82wsPDDavValx66aUnPXtRUZExe/Zsw9fX1/D39zfmzp1rlJeXu+BpWna6Zzx48OAp/22uXr3aMAzDSE9PNxITE42AgADD09PTGD58uPHHP/7R6QPflU73fFVVVcb06dONPn36GO7u7ka/fv2M22677aT/6evJ72Gzv//974aXl5dRUlJy0vXd/T080+eDYbTu9+ehQ4eMK664wvDy8jJCQ0ON+++/36ivr++wOk1NxYqIiIh0WxrDIiIiIt2eAouIiIh0ewosIiIi0u0psIiIiEi3p8AiIiIi3Z4Ci4iIiHR7CiwiIiLS7SmwiIiISLenwCIiIiLdngKLiIiIdHsKLCIiItLtKbCIiIhIt/f/F7JgIsRk7aEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(torch.tensor(lossi).view(-1, 1000).mean(1, keepdim=True).data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layer in model.layers:\n",
    "    layer.training = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train 1.716454267501831\n",
      "valid 2.0466954708099365\n"
     ]
    }
   ],
   "source": [
    "print(\"train\", split_loss(\"train\"))\n",
    "print(\"valid\", split_loss(\"valid\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meliha\n",
      "staphoor\n",
      "alepa\n",
      "damarius\n",
      "chloe\n",
      "kaven\n",
      "darahet\n",
      "iosir\n",
      "noah\n",
      "pabriani\n",
      "kaylan\n",
      "akhilynn\n",
      "adae\n",
      "ori\n",
      "rowen\n",
      "bodey\n",
      "bekar\n",
      "fawliamal\n",
      "laymarie\n",
      "holsy\n"
     ]
    }
   ],
   "source": [
    "# sampling from the model\n",
    "g = torch.Generator().manual_seed(2147483647 + 10)\n",
    "\n",
    "for _ in range(20):\n",
    "    out = []\n",
    "    context = [0] * block_size\n",
    "    while True:\n",
    "        # Forward pass\n",
    "        logits = model(torch.tensor([context]).reshape(1, -1))\n",
    "        probs = F.softmax(logits, dim=1)\n",
    "\n",
    "        ix = torch.multinomial(probs, num_samples=1).item()\n",
    "\n",
    "        # Shift the Context Window\n",
    "        context = context[1:] + [ix]\n",
    "\n",
    "        if ix == 0:\n",
    "            break\n",
    "\n",
    "        out.append(ix)\n",
    "\n",
    "    print(\"\".join(itos[i] for i in out))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "validation loss becomes 1.993"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytor",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}