makemore_3_gradients.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPcO5qJQUaIPXRBK0fTVAyQ"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "https://www.youtube.com/watch?v=P6sfmUTpUmc&ab_channel=AndrejKarpathy"
      ],
      "metadata": {
        "id": "PxQaa54s1p_x"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LonGYYTj1lW6"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import matplotlib.pyplot as plt # for making figures\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# download the names.txt file from github\n",
        "!wget https://raw.githubusercontent.com/karpathy/makemore/master/names.txt"
      ],
      "metadata": {
        "id": "NpRF2WrW1toh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# read in all the words\n",
        "words = open('names.txt', 'r').read().splitlines()\n",
        "print(words[:8])\n",
        "print(len(words))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4bsAoAQy1u6B",
        "outputId": "0f9c7662-1f1c-4cae-cb50-20c4eb347e63"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['emma', 'olivia', 'ava', 'isabella', 'sophia', 'charlotte', 'mia', 'amelia']\n",
            "32033\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# build the vocabulary of characters and mappings to/from integers\n",
        "chars = sorted(list(set(''.join(words))))\n",
        "stoi = {s:i+1 for i,s in enumerate(chars)}\n",
        "stoi['.'] = 0\n",
        "itos = {i:s for s,i in stoi.items()}\n",
        "vocab_size = len(itos)\n",
        "print(itos)\n",
        "print(vocab_size)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y6EmhY8Y12YB",
        "outputId": "80d029ff-9fff-4d40-eaff-f4552591c35c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n",
            "27\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# build the dataset\n",
        "block_size = 3 # context length: how many characters do we take to predict the next one?\n",
        "\n",
        "def build_dataset(words):\n",
        "  X, Y = [], []\n",
        "\n",
        "  for w in words:\n",
        "    context = [0] * block_size\n",
        "    for ch in w + '.':\n",
        "      ix = stoi[ch]\n",
        "      X.append(context)\n",
        "      Y.append(ix)\n",
        "      context = context[1:] + [ix] # crop and append\n",
        "\n",
        "  X = torch.tensor(X)\n",
        "  Y = torch.tensor(Y)\n",
        "  print(X.shape, Y.shape)\n",
        "  return X, Y\n",
        "\n",
        "import random\n",
        "random.seed(42)\n",
        "random.shuffle(words)\n",
        "n1 = int(0.8*len(words))\n",
        "n2 = int(0.9*len(words))\n",
        "\n",
        "Xtr,  Ytr  = build_dataset(words[:n1])     # 80%\n",
        "Xdev, Ydev = build_dataset(words[n1:n2])   # 10%\n",
        "Xte,  Yte  = build_dataset(words[n2:])     # 10%"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uBwKhHp_16CC",
        "outputId": "ae4fa60c-e372-4fb1-c5aa-9b1eb8650f30"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([182484, 3]) torch.Size([182484])\n",
            "torch.Size([22869, 3]) torch.Size([22869])\n",
            "torch.Size([22793, 3]) torch.Size([22793])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Softmax confidently wrong\n",
        "\n",
        "# Logits start to high. You can change the bias2 to init with 0 instead of random numbers\n",
        "# Weights should also be small but never 0 because of syncronosation opties\n",
        "# Bij training is the start loss too high (Around 25 - 30) Fake Confidence\n",
        "# For this project you know the biggest propability loss. Because everyhing (27 chars) have the same change to be the first character.\n",
        "# So loss is around 3.3 at init\n",
        "-torch.tensor(1/27.0).log()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pvo6tB1o_pe4",
        "outputId": "b9c9d460-e43e-42cd-87ad-91b6495442f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(3.2958)"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Tanh Layer too saturated\n",
        "\n",
        "# Hidden Layer (h) werkt slecht bij een flat tail (-1 or 1)\n",
        "# Killing the gradient/als de output er close bij 1 / -1 is\n",
        "# Je wilt Dead Neurons voorkomen, weights and bias never learn\n",
        "# Daarom moet hpreact aka hidden layer pre-activation dichter naar 0 toe, net zoals logits\n",
        "\n",
        "##  De eerste training uitdraaien en daar kijken wat er op de hidden layers gebeurd"
      ],
      "metadata": {
        "id": "-vxBb2WJGqSu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# https://pytorch.org/docs/stable/nn.init.html\n",
        "# torch.nn.init.kaiming_normal_(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu')\n",
        "# Fills the input Tensor with values according to the method described in Delving deep into rectifiers: \\\n",
        "#         Surpassing human-level performance on ImageNet classification\n",
        "\n",
        "#   So Gain / Square of fan in\n",
        "#   or\n",
        "#   (5/3)/((n_embd * block_size)**0.5)"
      ],
      "metadata": {
        "id": "1VK4Dyz8PAym"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# # Batch Normalization is a technique that normalizes the activations of each layer in a neural network, \\\n",
        "#   stabilizing the training process, allowing higher learning rates, and acting as a regularizer to improve generalization\n",
        "\n",
        "# Batch Normalization normalizes the activations of each layer in a neural network by subtracting the batch \\\n",
        "#      mean and dividing by the batch standard deviation.\n",
        "\n",
        "# Scale and Shift"
      ],
      "metadata": {
        "id": "MfCBFfCdSTlG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# MLP revisited\n",
        "n_embd = 10 # the dimensionality of the character embedding vectors\n",
        "n_hidden = 200 # the number of neurons in the hidden layer of the MLP\n",
        "\n",
        "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
        "C  = torch.randn((vocab_size, n_embd),            generator=g)\n",
        "W1 = torch.randn((n_embd * block_size, n_hidden), generator=g)    * (5/3)/((n_embd * block_size)**0.5)                              #* 0.2\n",
        "# b1 = torch.randn(n_hidden,                        generator=g)    * 0.01\n",
        "W2 = torch.randn((n_hidden, vocab_size),          generator=g)    * 0.01 # logits very small at init. Dont put Weights on 0\n",
        "b2 = torch.randn(vocab_size,                      generator=g)    * 0 # Not a bias of random numbers at init\n",
        "\n",
        "# BatchNorm parameters\n",
        "bngain = torch.ones((1, n_hidden))\n",
        "bnbias = torch.zeros((1, n_hidden))\n",
        "bnmean_running = torch.zeros((1, n_hidden))\n",
        "bnstd_running = torch.ones((1, n_hidden))\n",
        "\n",
        "parameters = [C, W1, W2, b2, bngain, bnbias]\n",
        "print(sum(p.nelement() for p in parameters)) # number of parameters in total\n",
        "for p in parameters:\n",
        "  p.requires_grad = True"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KnPGg7nB8SgR",
        "outputId": "bc60c0c6-f63a-4a46-e9c4-d24290249b52"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "12097\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# same optimization as last time\n",
        "max_steps = 200000\n",
        "batch_size = 32\n",
        "lossi = []\n",
        "\n",
        "for i in range(max_steps):\n",
        "\n",
        "  # minibatch construct\n",
        "  ix = torch.randint(0, Xtr.shape[0], (batch_size,), generator=g)\n",
        "  Xb, Yb = Xtr[ix], Ytr[ix] # batch X,Y\n",
        "\n",
        "  # forward pass\n",
        "  emb = C[Xb] # embed the characters into vectors\n",
        "  embcat = emb.view(emb.shape[0], -1) # concatenate the vectors\n",
        "  # Linear layer\n",
        "  hpreact = embcat @ W1\n",
        "  # -------------------------------------------------------------\n",
        "  # hpreact = bngain * (hpreact - hpreact.mean(0, keepdim=True)) / hpreact.std(0, keepdim=True) + bnbias\n",
        "  # BatchNorm layer\n",
        "  bnmeani = hpreact.mean(0, keepdim=True)\n",
        "  bnstdi = hpreact.std(0, keepdim=True)\n",
        "  hpreact = bngain * (hpreact - bnmeani) / bnstdi + bnbias\n",
        "\n",
        "  with torch.no_grad():\n",
        "    bnmean_running = 0.999 * bnmean_running + 0.001 * bnmeani\n",
        "    bnstd_running = 0.999 * bnstd_running + 0.001 * bnstdi\n",
        "  # -------------------------------------------------------------\n",
        "  # Non-linearity\n",
        "  h = torch.tanh(hpreact) # hidden layer\n",
        "  logits = h @ W2 + b2 # output layer\n",
        "  loss = F.cross_entropy(logits, Yb) # loss function\n",
        "\n",
        "  # backward pass\n",
        "  for p in parameters:\n",
        "    p.grad = None\n",
        "  loss.backward()\n",
        "\n",
        "  # update\n",
        "  lr = 0.1 if i < 100000 else 0.01 # step learning rate decay\n",
        "  for p in parameters:\n",
        "    p.data += -lr * p.grad\n",
        "\n",
        "  # track stats\n",
        "  if i % 10000 == 0: # print every once in a while\n",
        "    print(f'{i:7d}/{max_steps:7d}: {loss.item():.4f}')\n",
        "  lossi.append(loss.log10().item())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k0Z7muiS8V7n",
        "outputId": "509ea41f-1386-458b-a95a-f66b4e9cae78"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "      0/ 200000: 3.2788\n",
            "  10000/ 200000: 2.1573\n",
            "  20000/ 200000: 2.3661\n",
            "  30000/ 200000: 2.1708\n",
            "  40000/ 200000: 2.2585\n",
            "  50000/ 200000: 2.7153\n",
            "  60000/ 200000: 2.0828\n",
            "  70000/ 200000: 2.3331\n",
            "  80000/ 200000: 2.3124\n",
            "  90000/ 200000: 1.9479\n",
            " 100000/ 200000: 1.8613\n",
            " 110000/ 200000: 1.7870\n",
            " 120000/ 200000: 2.3108\n",
            " 130000/ 200000: 1.9558\n",
            " 140000/ 200000: 2.4277\n",
            " 150000/ 200000: 2.0879\n",
            " 160000/ 200000: 2.0539\n",
            " 170000/ 200000: 2.3074\n",
            " 180000/ 200000: 1.8473\n",
            " 190000/ 200000: 2.1584\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(lossi)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "Y39-XZWP91mP",
        "outputId": "c212d8b6-c1eb-4c5e-8ed5-a36e29254881"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7b55d35c8bb0>]"
            ]
          },
          "metadata": {},
          "execution_count": 72
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRDklEQVR4nO3deVhU9f4H8Pewg7KoCIiiuO/iCmKZpri1WVp6zdKobLXlR4uXW2nZLbzmVe8t065mWnZTK1tumaa4K4qiuGvuuAGisqps8/39QYwzMNuZOTPnzPB+PQ/PAzNnzvkcBuZ8znf5fDVCCAEiIiIilfBQOgAiIiIifUxOiIiISFWYnBAREZGqMDkhIiIiVWFyQkRERKrC5ISIiIhUhckJERERqQqTEyIiIlIVL6UDsIZWq8WlS5cQGBgIjUajdDhERERkBSEEioqKEBkZCQ8P69tDXCI5uXTpEqKiopQOg4iIiGxw/vx5NGvWzOrtXSI5CQwMBFB1ckFBQQpHQ0RERNYoLCxEVFSU7jpuLZdITqq7coKCgpicEBERuRipQzI4IJaIiIhUhckJERERqQqTEyIiIlIVJidERESkKkxOiIiISFWYnBAREZGqMDkhIiIiVWFyQkRERKrC5ISIiIhUhckJERERqQqTEyIiIlIVJidERESkKkxOVOpYdiEWbT2N8kqt0qEQERE5lUusSlzXHLlUiHv+vVX389P9WykYDRERkXOx5USF9BOTI5cKFYyEiIjI+ep0y8m8jSeRdfUGHo9vgS5Ng5UOh4iIiFDHk5PUoznYm5WPxoG+TE6IiIhUgt06AD7ZeBI7TuUpcuxj2YXYcVKZYxMREakRk5M/bTyWW+uxwlvlWLjlNC7l33TYcYfP3YpHF+3CuaslDjsGERGRK6nTyYlGozH7/Ds/HsIHq49i1Kc7HB7L2as3HH4MIiIiV1CnkxNLtvxxBQCQXXhL4UiIiIjqjjqdnFwvKVM6BCIiIqqhTicnp/NcYJyH+Z4nIiIit1OnkxMiIiJSHyYnREREpCo2JSfz5s1DdHQ0/Pz8EBcXh/T0dJPbLlmyBBqNxuDLz8/P5oAdRQh59jN3/R+Y/r8j8uwMgIb9OkZdLrjJRRGJiNyU5ORkxYoVSEpKwrRp07B3717ExMRg2LBhyM2tXSekWlBQEC5fvqz7OnfunF1BO4vUfEWrFZi7/gQWbz+DLE4Ndpi9WdcRn7IBo+c7foo3ERE5n+TkZPbs2Zg0aRISExPRqVMnLFiwAAEBAVi8eLHJ12g0GkREROi+wsPD7QpaDYpLK1CpNUxf9H8qq6x0bkB1yLd7LgAADlwoUDgSIiJyBEnJSVlZGTIyMpCQkHB7Bx4eSEhIQFpamsnXFRcXo0WLFoiKisLIkSNx+PBhs8cpLS1FYWGhwZeaXC64iS7T1uLhBbxzJyIikpuk5CQvLw+VlZW1Wj7Cw8ORnZ1t9DXt27fH4sWL8dNPP2HZsmXQarXo168fLly4YPI4KSkpCA4O1n1FRUVJCdPhfj1wGQCwLytf2UBsVF6pxb9TT+D3w9n4etc5lJRWKB0SERGRjsNXJY6Pj0d8fLzu5379+qFjx4747LPP8P777xt9TXJyMpKSknQ/FxYWqi5BcRYLFfZt8mXaOcxe94fu58ysfHz0SIz8ByIiIrKBpOQkNDQUnp6eyMnJMXg8JycHERERVu3D29sbPXr0wMmTJ01u4+vrC19fXymhyWrn6atYdyQHt8rdc9zIiZwig59T/1z0sLi0AgHenvDw4AwhIiJSjqRuHR8fH/Tq1Qupqam6x7RaLVJTUw1aR8yprKzEwYMH0aRJE2mROtFf/rMTn287g1vldWeq6qX8qnE0j3xmeuwQERGRM0ierZOUlISFCxdi6dKlOHr0KJ5//nmUlJQgMTERADBhwgQkJyfrtp8+fTp+//13nD59Gnv37sVjjz2Gc+fO4emnn5bvLGRw4EIBXvg6w+QU4L1Z13Xfl1fWmKUjV5EUBf1y4BIAIOPcdQtbSiOEwMX8m27xOyIiIueQnJyMHTsWs2bNwtSpU9G9e3dkZmZizZo1ukGyWVlZuHz5sm7769evY9KkSejYsSPuueceFBYWYseOHejUqZN8ZyGD9LPXsPpgNp5dlmH0+cQvduu+/2jtMd33GeeuIfbDVPxv/yWHx+gqbpZVYvfZa6jUCsxZ9wfumLEBn246pXRY5GTFpRVI/CIdK/ecVzoUInIxNg2InTx5MiZPnmz0uU2bNhn8PGfOHMyZM8eWwyji3FXjiwGWVdzu4tEvb/L00j24fqMcL32zD/d0VW9XlbPkFt7CiH9txdWSMrwxrD3+vaFqbNFHa4/jxbvbKBwdOdPCLaex8fgVbDx+BWN6180B7URkG66tY6cKrWO7K9Q8NPVWeSWeX5aB7zJuTwuP/TAVV0vKAAD/3ZWlVGiqcDK3CAcu5CsdhmIKbpYrHQIRuSgmJyphKgmpnkqcce46nvsqA+evqacs/ldp5/DboWy8/u1+xWKo1Aq8/eNBrNprum6OUhJmb8EDn2xHXnGp0qEQEbkUJic13CgzPn34ZnklCm5Yfyd4ueAWvt1zHqUV8kxHHj1/B9YczsbLy/fJsj973CqvxM/7L+GsiS4wR9Ov/bL64GUs25mFpJXKJUiWXM6/pXQIRGQntgQ6F5MTCZJ/OGD1to9/no43vjuAeRtM13OxhRpaTj5cfRQvf7MPX0vstvlqp/wLPl77swvJ3VRqBWc4EanEqr0XEPPe7/jX+hNKh1JnMDmRYNuJPMzbKC3Z2PzHFcxccwxJKzMhhEClVuiKu208fnsl50oh8H8rMrF42xlZY3aEn22cmfTOj4dkOb7WweN8lFZRqcWQ2Zvx0Kc7mKAQWbA36zoWbjnt0M+F5FUHAQBz1v9hYUuSi8PL17ubj9Yel/ya6mm0T9/ZCi99sxdnr97A3neGGExP3ngsFz/su4gf9l2ULVZ75BbewoncYvRr3QgaR9TQ16PVCnyXcQE9W4SgTVigxe31a864o7NXS3A6z3ldZkcuFWL32Wt4rG8LeLI6MLmYUZ9WLcDaONAXD/ZoqnA0JBe2nDiY/mye8kotTl0pQaVWYG+NYmfFtxy7+F5xaQUyzl03eidu7LHYD1MxftEubDp+xaFxAcCqfRfx5vcHkDB7i1XbO3qGVE25hbewaOtpSWOO9Mk17shR7vn3Vkz7+TC+z3DOoOLcolv4ZMMJ5BZyLA7J59SVYqVDIBkxOZGg0EgCYek+8/ClQqv27ejL7chPtmH0/B34MVNay8wb3+1H0a2qi7IQAuuP5CDfxou0KfvP58u6P7k99vku/P3Xo/i/lZkAgBW7szBLQgvawwvSDOrkqNWRy9b9rdpr0tI9mPX7H3j6yz1OOV5do9UKPLZoF15VweB5IlsxOakjTl2p6ib4KfMSpAxjyCsuwyvLMwFUjZ+x94KiP87GVfyRU3VHtuHPBRKnfH8Qn2w8KamGSdY1ZWY2OdqVolI88UU6fj+cbfVr9l8oAFC1ZATJ70RuMbadzMOPmaxaTa6LyYnKaZxQhu26hZaQ6ovy3qx8u4+lP87G1RXedGxXnCv44Ncj2HT8Cp75yviyD3LbffYa9py95pRjuSotB1GTG2ByYif9rh5nzKzIKy7DD/vUV3CMHEPt15krTiwwd6OsAo8sSMPDC9J0M97k9M6Ph/DBr0dk3y8RScfkxAX93wr5C45ZaqFZuuOs7Me05EZZBb5MO4tL+Tedfmx7zN90Cm/9cNCOPdx+LwbO2oS//+IeF8ytJ67gze9s/9stLr19I3DTRLFEW13Kv4mvdp7Dwq1ncCzbOWNviMg0JicqZ+0sXmtbbQ5dLNStfSPFtJ8PO70M+we/HsXUnw7jgU+2Gzyu9smu/1hzTHKBOlOyrt3AIgVr38hVO+Kef23F45+nY+Uedbb6lVfeHrA8fO5WBSMhIoDJiVNlOmhWytXiUvRNScWHq49a3DavuBTrj+bYdJya058dbfMfVdOY84pLnbaA3v7z+Xhw3nbVj2vYeCwXw+duwaGLVYNKD10swKebThpcZIGqpHX/+XybSm///Zcj6P3Belmm/DprJlBdsv1kHobM3oytJxw/3Z/I2ZicONG0nw87ZL+Ltp1BTmEp/rPltO6xSq3Am9/txzfp1t3Br5Uw20IJ+q0ntt7LV1RqMWvtcew4lWdym7H/SUPm+Xw8vCDN4v6k/M7sHTsS+8F6g7gTl+zGsewi3PfxNqSfuYb7Pt6GmWuO1+p+2/zHFYyctx2DZm2y6jiHLhZg0pd7cCKnCIu2ncG1kjLM+l164UFyvPGLduFEbjEe/zzdqu0vF9xEym9HceG68ktgEFnC5ERG3aevs/m1prplrFk7xthL1x7Oxso9F5C86qDu7tqcPQ5oFblox1iRs3klKJK5MN2KPefxycaTeHThLoPHd5zKw4rdVUncrXLr65E4Yq0gU3KLSmvFXW3MZ7cTqb//ath6tvZwVSvZ1ZIyrNxz3mL334PztmPdkRw89vntY63ccwE3ypwzMynr6g1s+cN5LQGX8m/aXYPmWkmZSywz8PTSPfhs82mMX2T874icJ7vgFjKc3BLtapicyEh/wJ5cfj+Sg6M2NInrN+M/OG+7mS3V52RuEQbO2iS5K+KL7WeQvOqAyQtF1lXjd4yPLtyFKd8fdFrXkaOZuti++d0B/HbIfGtPdfXdnELD8UUXrtdONLefzMP2k1dtjNK4uz7aiAmL053SrZZ5Ph/9ZmzAyHnbba46vO5IDnq+vw5vGVk3SgiBk7nFqHRyRWNT49SqC0KeM/F/QM7TNyUVo+fvwEHW+jGJyYkLMFZWXMo4AGeXe5eiUlv1Aa7vx322FY96739H8E36eaSdsu2CedHIBVgJjlzK6MilQuw4lYfov/6Kf9rZXePIO3BHjc/St2pv1f/V0cuFGPzPzTbto7pS8H+NDIBesuMsEmZvxmt/VhZ2FZfyb7r94ppSOep/cs85dY9tUxKTE5WQWs1x0D83o+BGOTLOXYNwePH7Kseyi2Tf599WHUTaacNk4hOJKz/XVGJimqkj7mBdoSx9TdXdQx9vOInret2G9nZNOLprQ3+6uzP+4rVagTEL0jDJxqrIH2+o+jt2pUqtP++/hH4zNiDJxRIqcj9MTlxUcWkFEuZsxuj5aVhroalezVbsOW/wszUfiqev3C4Fb+mCWFahxdNL92DR1tMGU3JTfjtaa2aLLT7bfMrufQBVgxXHL9qJdUdsm0lV09rD2Zi38SRqXsZrJn5ler+DpWmmx9CUShiLIwdLXaQ5Tlg08HReCdLPXsO6IzlWtSQU3Ch3+a7Bj1NPALA9odp4LFc3fovIHkxOFCKltcNUnYsrRVXjAs7q9SFXyHDBdTQhBJ79ag9e/7Z2Qa5Ve6UtTGiMfjfR//ZfwvqjObUGin62+XStmUybbRiIuUmmwZvv/HgY209exaQv99g9s0dA4NmvMvDR2uPYddp8s7G1LT/3f7JNUsuIva0oc9efMPv8jN+O2bV/AFZ0/90+B2ua9WOm/44HPtmOHSdNzwaTKrfwFm6VV0KrFVienoXjDmi9lOqb9Cx8vct4Ipu4ZDemfH8QJ3KUj1MOt8orcTbPPdfFUjsmJwpxVAu4kgW7rHXu6g2sPZyD74yMpZHq3f/Vrp7626HLuu/3ZpkeEX/+muHAwOW7z5vY0n4vfbPP7IwX/QJ3/9tv/q51u4SL32kLH6z9Z260el/Wjl26UVaBu2dtwhIHVhW2t0JsbuEtnKgx1kku64/Ks7hl1tUbiP0wFf1nbsT/DlzCX1cdxLC5Wyy+rqRU/tL+t/ddgeRVB/HWD4fMDlivvnEy5fuMC1i09bTZbdTgwXnbMXDWJkmz+KRwgUleimFyohBjrQZykLI6rFLsGaB7+JL1o9sPXMg3W6m1QissTrO2VODK2haCY9lFeOm/+6y687V0jmqdClpeWfW7+OXAZYPWPEsqKrV4beV+p3YHZDuhW8iUM3klGPzPTUYHuuurXsH7SlGppFkd7xiZOSQX/a7Q0grbk6DXvt2Pv/96VPWtEo4YZ0fWYXKiEEsrAVNtt8orce+/t1ncrri0AkKIWmXva/pi+1nc97Hp/V24fsNigau9WflImL3ZqtL+qcdyMWzuFoOE5kZZBZ5eurvG7BS1F+g3rt3bv2HOuj8kj1b9ef8lfL/3AqZ8b/16ROkqr+BrTvKqAzh1pQSvOegGxRHVeG+UVWDl7vPIK7Zu6QutsC5xt7WWUUWl1mTrTElpBTYdz5VlTJk53+45j+RVB112ZtPvh7Ox/WQelmw/47Q6RlIwOSFZpJ26ig3H5BnMaYq1Tfmnr5SgZfJqu4935z+s6/I4mVuMTzZYP8PoVrkWJX8O+Px86xnZugGqKdlU/K9U82NFakpedQD5dSxRv+nkwcVymP6/I3jz+wN4dOFO3WPmxuw89vkuTPxit8PiGb0gDX0+WG+05fPppXvwxBe78c/f/3DY8QHgje8O4Jv0LKdW1844dw1XZVjjbPvJPDzzVQbGL9qFd/93BB/8annpE2djcuKGKiq1Nq2lYo9xC3fiySW2Tbl0B2US7tK6vrsWnaetRWlFJfKd/D45w24JrRrfpFs/zseR9V+cab+ZGi4bjuXg8c93IbtAuW4nY6qL9+XqtVa8sjzT7GscWem3+ndobAB9dWkCZ3UTOuuzdtuJPIyen4b4GRvs3lfNOkJbT8g3iFsuTE7czN6sfDzwyXZZZjM4yh8uPJJfjg+i6jE3GWell6/+cZ/9s5nsYc3snm9lGOgMACsdOEDZkscW7cLF/NoJwncZF/DcVxm4VW66Fe+KHXe2Ty7Zg60n8vD2jwcd3i1hjGt2UJgmZdbYN+lZNhdwtJWU3/fmP6paWO2trSSEwEdr1b9eFpMTN2Rvn/P9ZsZhyOGFr/c6dP+ONN3I7CCg6k7u/LUbuu4aazy6aJfZi5wxr67IlLS9nLadzEO7t39D7AfrZd2vqQ/oN78/YPe+yyq0Nk1r3nYyD8lGjv/6t/ux5nA2vkw7a/K1/9t/ye6m9/VHc2tNf7fVzbJKFN1yvxY6S67fKEfch6n4I6cIf+QUmR3cnnHuGpJXHcQ4vW4rd/Lz/ksYsyANuUW3dMsYqB2TE6rloBULBSpBDf9UpmbSHL5UiP4zN0qubvvLgcu1Hlt/1L6xOzWXA5DLG38O4MwtKpW19atY5gUeq13Mv4n27/yG11bWHniqsWLQsbnBn4U3q2I+buL3cPBigWoWA+w4dQ26vvu75NcpvaBhYY2E6lZ5JTYcy5E0jTy3qBRTvj+AoXO24PHP03EipwjHs4sw9adDyC263TJ2/lrtpSuesbEysBq9/M0+pJ+9htgPUiXdQCmJyQm5DP2VctVq/iZpFWMdMdLf3Awkucg5iHXOescMXPxyx1kIAaxSuCvMVfV8fx2m/nTY6cfVagW+3XMe3d79/c8qx1Xe+uEQnlyyR3Jpff1/sZO5xRj+ry34Mu0c/s9CK+TvMlVrVsKNsgq8unwf1rhw9XAmJ0QSsO4BKa3gZjnOXbWuPkjGOWnjmmq2J3210/SSBtV+3HcRz3y5R5Y7cq1W4L6Pt+GN76q61D5aexy3yisxZkEavv9zoUZLK2sb26e+6sYgW1tihRC4VmLdlGpr9mXOrfJKZJy7ZnRdsGU7z+Glb/YZHZv02ebT+DHzEp5bliFLnErwUjoAIiJbzN90Cs8PbO2w/TtrQU1r6EcS8571XTSj5++QP5gaqsdBfbbF/oqvV0vKao2ZW7nnvF11beTupv7r9wcN1gSbs/4PjOjaBMH+3jbtr1Ir8MQX6WjRKAB/f7CrwXOTvqwaID1leIdar3v7z2J7gzo0xkM9mhk8l2umQu8BCQX9lMSWEyIFFblI/6/Slu44iwk1CuL9Y416Z6S5G2vGnhTcsK01YcfJPKT8dtTkLBRrB41rhcCl/NpjR+RWc7HSnMJS3POvrUj57ajkAe5AVevW1hN5WLYzq1YxtOopvl+ZGYAtdczWB6vVV9PEGCYnRKR6034+7JDKp+bUvB5fl6kpXw3e+fEQjmcXGSQdt8orHbbekDmPLtqFzzafxjIrupDMmbfxFPrJUAPEFhfzb+KzzadrrRdUVqHFHzlFZpM7/S6bTlPXGt3mUsEtaMwU+jl0sQCLtp422v1jjaxrN5B+Rl1Vl5mcEJHL+uBX41O7Lck4J+2DeN/5fPR4f53u58+2nFL9ujDmfLXzHIbN3YKHPr3d7TNP4kwzW5kar5F17QZynLjmUf6NchzPLpL1faxZFO6ppbsxdM4WfG9mtXVruw//Y6bb7L6Pt+Hvvx7FCiO1gQ5cyLdq/2M+S7NqO2dhckJEVlHJzFgDC7feXoVbCGHV7Kfl6VlGV7M25/Maq32XVwrV3WnaQr9S6A4nFSCbs8747CytEE6ZaaZv2Nwtf646LM9KzqfzSpBbeAsj/rUVy3ae03XLLDWxQvfVkjI8urD2LERbC/Ady67dumhpjTG1YnJCRJJpVZaplFZUIj5lA1r9bbXZ1XJPXSnGX1dZt8CgpTOstPA7sGf1bXdmalbPl2n2devYQ85lJGauPY6jlwt1A1bNqZn0AsCnm06i7Vu/YbMDy/+7AiYnRGQV/UvtpKXqKVCl1QoMm7MF2X92CRgrulbN2im4NR01Mt4l2UySI2D7irvu5mRuMQZ8tBHf7qnd5eDs9ZJMpYtyLKZXzd5WmJlrqkrLT1xsfkV0d8fkhIgkc9YsI2tmifSfuRFnr97Q/Wys6q4t9AcXnr4iLakx1Tpw/toNo49bYumCdyKnyOgKvXL45cAlu14/5fsDOHf1hq52iRrbkxxdffpaSRkenr8DK40kaPY6f91whtLV4lJZky2lsM4JEamWEJbvri8amT6667RzF3CrydQgy/4zN9q0v693mV9hd8icLQCA5c/0tWn/5kz+7z4E+tl2qXj8812SC8FJ4cwBtNYqNTIl+mL+TVzMv4k9565jTO8ou49xRC+Z0h8oW6kV6PV3ede+UgpbTojIKsUuVJNl7H+UXcBt1u+OKclvSaqd6zLJrXpAqD3MNZ79elBaK5mx7jm5rZNS9t7GZqR7/r3V6ONS1h1SO7acEJFV7F2q3dn0K5bmFt1C7AepCkYDVBiZgXHwQgG6NgtWIJrazA0kNmXHKWnJh9KLCX68wfh0af2QKiq1ui4oUg5bTohIteS6jA2fa/xO01k2HMsxujLw/Z9UTZ0trag0O8DWWgV2zDo5dFF6q4KxabDm2DIW4l+pJyS/xh7Tfj6MH7hYpOKYnBCRag2ctRErjRSWkkquhdps9eSSPbhpZlDrV2nn8E26+XEl1li554Ld+zBGzplHUlf7veGEroo0vTFKlsb3kHMwOSEi1Tp/7Sbe/N79m9jVOLBTqqVW1ilxRrJBro/JCREROcW2k/YPkHV19+tVwVXTytdqw+SEiEhBm/+4YnQ6tLPsOJnntEGq79VYNqDMxjLtruygg+rRuBvO1iEiUpDSlUAfXbQL98dEKnLsZQqWrHdHq9xoIC9bToiI6rj/7bevCqytLijYYkTqxuSEiIhIAeWVHHNiCpMTIiJShCNL25NrY3JCRESK0F9ckUgfkxMiIiJSFSYnREREpCpMToiIiEhVmJwQERGRqjA5ISIiIlVhckJERESqwuSEiIiIVIXJCREREakKkxMiIiJSFSYnREREpCpMToiIiEhVmJwQERGRqjA5ISIiIlVhckJERESqwuSEiIiIVIXJCREREakKkxMiIiJSFSYnREREhHNXS5QOQcem5GTevHmIjo6Gn58f4uLikJ6ebtXrli9fDo1GgwcffNCWwxIREZGDFN2qUDoEHcnJyYoVK5CUlIRp06Zh7969iImJwbBhw5Cbm2v2dWfPnsXrr7+O/v372xwsEREROcbN8kqlQ9CRnJzMnj0bkyZNQmJiIjp16oQFCxYgICAAixcvNvmayspKjB8/Hu+99x5atWplV8BEREQkv6vFpUqHoCMpOSkrK0NGRgYSEhJu78DDAwkJCUhLSzP5uunTpyMsLAxPPfWUVccpLS1FYWGhwRcRERHVDZKSk7y8PFRWViI8PNzg8fDwcGRnZxt9zbZt2/D5559j4cKFVh8nJSUFwcHBuq+oqCgpYRIREZELc+hsnaKiIjz++ONYuHAhQkNDrX5dcnIyCgoKdF/nz593YJRERESkJl5SNg4NDYWnpydycnIMHs/JyUFERESt7U+dOoWzZ8/i/vvv1z2m1WqrDuzlhePHj6N169a1Xufr6wtfX18poREREZGbkNRy4uPjg169eiE1NVX3mFarRWpqKuLj42tt36FDBxw8eBCZmZm6rwceeAB33303MjMz2V1DRESkEkIoHcFtklpOACApKQkTJ05E7969ERsbi7lz56KkpASJiYkAgAkTJqBp06ZISUmBn58funTpYvD6kJAQAKj1OBERERFgQ3IyduxYXLlyBVOnTkV2dja6d++ONWvW6AbJZmVlwcODhWeJiIjINhoh1NSQY1xhYSGCg4NRUFCAoKAg2fYb/ddfZdsXERGRK5s/vidGdG0i6z5tvX6ziYOIiIig0SgdwW1MToiIiEhVmJwQERGRqjA5ISIiIlVhckJERESqwuSEiIiIVFWEjckJERERqQqTEyIiIlIVJidERETEOidEREREpjA5ISIiIg6IJSIiIjKFyQkRERGpCpMTIiIigop6dZicEBERkbowOSEiIiJVYXJCREREUFGZEyYnREREpC5MToiIiEhVmJwQERGRqjA5ISIiIlVhckJERETYdz5f6RB0mJwQERERDlzIVzoEHSYnREREpCpMToiIiAgaFVU6YXJCREREqsLkhIiIiFSFyQkRERGpCpMTIiIiUhUmJ0RERKQqTE6IiIgIAkLpEHSYnBARERGEenITJidERESkLkxOiIiICBr11GBjckJERETqwuSEiIiIVIXJCREREalKnU5OPD1U1MFGREREAOp4ctIlMkjpEIiIiKiGOp2cxLZsqHQIREREqsA6JyqhUdO8KSIiIgJQx5MTIiIiqqKm+3UmJ0RERKQqTE6IiIhIVep0cvJobHOlQyAiIqIa6nRyEh1aT+kQiIiIqIY6nZwQERGR+jA5ISIiIlVhckJEREQovFmhdAg6TE6IiIgIRy4XKh2CDpMTIiIiUhUmJ0RERKQqTE6IiIhIVZicEBERkaowOSEiIiJVYXJCREREqsLkhIiIiFSFyQkRERGpCpMTIiIiUhUmJ0RERKQqTE6IiIhIVZicEBERkaowOSEiIiJVYXJCREREqsLkhIiIiFSFyQkRERGpCpMTIiIiUhUmJ0RERKQqTE6IiIhIVZicEBERkarYlJzMmzcP0dHR8PPzQ1xcHNLT001uu2rVKvTu3RshISGoV68eunfvjq+++srmgImIiMi9SU5OVqxYgaSkJEybNg179+5FTEwMhg0bhtzcXKPbN2zYEG+99RbS0tJw4MABJCYmIjExEWvXrrU7eDk0quejdAhERESkR3JyMnv2bEyaNAmJiYno1KkTFixYgICAACxevNjo9gMHDsRDDz2Ejh07onXr1njllVfQrVs3bNu2ze7g5dC6cX2lQyAiIiI9kpKTsrIyZGRkICEh4fYOPDyQkJCAtLQ0i68XQiA1NRXHjx/HXXfdJT1aIiIicnteUjbOy8tDZWUlwsPDDR4PDw/HsWPHTL6uoKAATZs2RWlpKTw9PfHpp59iyJAhJrcvLS1FaWmp7ufCwkIpYRIREZELk5Sc2CowMBCZmZkoLi5GamoqkpKS0KpVKwwcONDo9ikpKXjvvfecERoRERGpjKTkJDQ0FJ6ensjJyTF4PCcnBxERESZf5+HhgTZt2gAAunfvjqNHjyIlJcVkcpKcnIykpCTdz4WFhYiKipISKhEREbkoSWNOfHx80KtXL6Smpuoe02q1SE1NRXx8vNX70Wq1Bt02Nfn6+iIoKMjgi4iIiOoGyd06SUlJmDhxInr37o3Y2FjMnTsXJSUlSExMBABMmDABTZs2RUpKCoCqLprevXujdevWKC0txerVq/HVV19h/vz58p6JrTRKB0BERET6JCcnY8eOxZUrVzB16lRkZ2eje/fuWLNmjW6QbFZWFjw8bjfIlJSU4IUXXsCFCxfg7++PDh06YNmyZRg7dqx8Z0FERERuQyOEEEoHYUlhYSGCg4NRUFAgexfPmAVpSD97TdZ9EhERuaKzM+6VdX+2Xr+5tg4RERGpCpMTIiIiUhUmJ0RERKQqTE6IiIhIVZicEBERkaowOWGdEyIiIlVhckJERESqwuSEiIiIVIXJCREREakKkxMiIiKCr5d6UgL1REJERESK0ahoggiTEyIiIlKVOp+cxLdqpHQIREREpMdL6QCU9sLdrREW5IvrJWWY9fsfSodDRERU59X5lhNfL0+Mj2uByYPaKh0KERGRYjQqqkpa55MTIiIiUhcmJ0RERKQqTE6IiIhIVZicEBERkaowOSEiIiJVYXJCREREEBBKh6DD5ISIiIgg1JObMDkhIiIidWFyQkRERKrC5MSEDx/qqnQIREREdRKTExPUNDCIiIioLmFyoifIr86vg0hERKQ4JidERESkKkxOiIiISFWYnBARERE0GqUjuI3JiZ76vhxzQkREpDQmJ3r+M6E32oTVx38e76V0KERERHUWkxM9XZoGY33SAAztHKF0KERERHUWkxMiIiJSFSYnREREpCpMTkwY0jFc6RCIiIjqJCYnJoQF+SEteRDuj4lUOhQiIqI6hcmJGU2C/fHxuB5Kh0FERORwGqin0AmTEyIiIlIVJidERESkKkxOiIiISFWYnBAREZGqMDkhIiIiVWFyYoVRPZsqHQIREZFD+XmrJyVQTyQq9uFDXQ1+bhLsp1AkREREjtG/bWOlQ9BhcmIFP29Pg58jQ/wVioSIiMgxPD1Y54SIiIjIKCYnNuAYFCIicjdCCKVD0GFyYoNxfZorHQIREZHbYnJiAw8PDVo1rqd0GERERG6JyQkRERGpCpMTIiIiUhUmJ0RERKQqTE4kmvlwNwDAqB6WZ+zc2SbU0eEQERG5HSYnEkUEVVWHfW5Aayx9MtbstsuejnNGSERERG6FyYmNvDw9MKCdfKV+x/aOkm1fRERErozJiQr8X0I7TOwXLdv+OjUJkm1fREREzsbkxEEeiIm0ettnB7SS9dgxUSGy7o+IiMiZvJQOwF01rOdjcZv0twYjxN8HPl7MEYmIiKoxOVFQWKCf0iEQERGpDm/Z7fT0nS1l2U/jQF9Z9kNEROTqmJzY6e37Osmyn8aBvhanJhMREdUFTE4cxMtDI/k1A9o1RpNg+7t62obVl/ya/m1ZMM4ZRvW0XLyPiKiuY3IikZ+3p8VtWjWuh+cHtnZCNMY9Ht9C8mvGx0l/jbWWPcVidEREZD0mJ1Z6fWg7jOrZFH2iG1jcdsNrA9GoftUYEiVaJLw9a7+te95OwBeJfRSJ5062yhARkQRMTqw0eVBbzB7THRqNtO4atXSXhNb3xd3tw4wmLmTZQ1aspWQVIc9uiIjcGa9UDqaB9LEn9hjSKdzs886Nxn0E+FjuziMiInkwOVGZGaO7mX2+aYg/3rqnI36efIfR5//zeC9HhCU7V5uZxAYPIiLnYRE2mQ3qEGbws8ReIAxo1xiZU4eg+/R1tZ57ZXBbvDSoDbzMdM1I7XZSihC83BMRkXFMTmT02yv90SEi0O79+HoZ70Lw8fIwm5goqW+rhth5+prSYTgMcykiIudR55XOxdzTNQJxLRuifXig7C0XP7zQT9b9Oco9XZsoHYJLYI5DRGSZTcnJvHnzEB0dDT8/P8TFxSE9Pd3ktgsXLkT//v3RoEEDNGjQAAkJCWa3d0Wfju+FFc/Gw8OGwmuWhNa3v6x958ggGSJxrvAglvOvNmdsjNIhEBE5leTkZMWKFUhKSsK0adOwd+9exMTEYNiwYcjNzTW6/aZNmzBu3Dhs3LgRaWlpiIqKwtChQ3Hx4kW7g3dXphpfbB2n8d9JfW0+pr6dyYNNPtesgb+UkMz64KEu+OnFO2Xbn5rY8h4+1KOZAyIhIlIvycnJ7NmzMWnSJCQmJqJTp05YsGABAgICsHjxYqPbf/3113jhhRfQvXt3dOjQAYsWLYJWq0VqaqrdwZNl7cMDEezvrftZPwmZP76n1fuJbhSACBOl9RvV88Hd7cOMPlft3q5NEBvd0KpjjY9rYfJYtnp5UBtZ90e20f9bJCJ1UVO3s6TkpKysDBkZGUhISLi9Aw8PJCQkIC0tzap93LhxA+Xl5WjY0PSFqrS0FIWFhQZfrkrNs2dGyDhOxNJ5zhvfEyufi9f9bOqfoGE9n1qPBfl5yTLQmJQnZwsbEbkvSclJXl4eKisrER5uWOgrPDwc2dnZVu1jypQpiIyMNEhwakpJSUFwcLDuKyoqSkqYLmHRhN5Kh1BL84YBSoeA0UYWxnukdxSevLOlfTuWmCT++vKdWJLYR+8RNd1TEBG5N6fO1pkxYwaWL1+OH374AX5+ppvtk5OTUVBQoPs6f/68E6N0vPS3BiPBQiVXJdiymrHcPKxMIoy1sMipc2QwBup1VSk9lfjhXhx3IpfeLYyvjzXBhgUzicgxJCUnoaGh8PT0RE5OjsHjOTk5iIiIMPvaWbNmYcaMGfj999/RrZv5Kqi+vr4ICgoy+HInXh7qm8Gt9MW32tDO5v+Oqg1s31jSftXbuWadWY/E4PjfhysdhllNQyx32Uj5O3ttSDs7ojGtk4nZa1Pv6+SQ4xGRdJKukj4+PujVq5fBYNbqwa3x8fEmXzdz5ky8//77WLNmDXr3Vl93hiN1jwqRbV+WPtjfvd9xH66tGzunVaWXibtae7VqXM/ufSg9XsJUcT4lJBlJHB6Na27xdVJy4MkcxCyZlwPKGRApQfItfFJSEhYuXIilS5fi6NGjeP7551FSUoLExEQAwIQJE5CcnKzb/h//+AfeeecdLF68GNHR0cjOzkZ2djaKi4vlOwsV69WiAb56KharbCymJuVOs0OT23eE1QNIH+geaf2xzDz3lI1jPlqbSQrGxVq+mJkksaUnwOd2MWQfL9taruaM7a63P08sf8byFO2a9MN+575OBi1Ad0tsDVKSHMmeJY4aTO6j0irLcjjxwQilQyCSheTy9WPHjsWVK1cwdepUZGdno3v37lizZo1ukGxWVhY89Lot5s+fj7KyMjz88MMG+5k2bRreffdd+6J3Ef3bNsb1kjLdz54O+tDV3+uKZ+OxL+s67mwTatc+t755N05dKUY/M/uRcjqdmgThRG4R+kQ3xLojOZZfUE0F3U7hgbfHSQX7e6Nvq0Z27a9xoC+WJMbih30X4OPpiXu7NcEdMzbgYv5Ne0O1Sq8WDZBx7rpTjlVNDff1DUyMV1LzzDprucM5uILBHcLg6+2B1QetmwhC0tm0ts7kyZMxefJko89t2rTJ4OezZ8/acgi306CeD57oFw0ACA5wTK0H/Q+mYH9vgwGdeltZvb8n+kUjqmEAomScxfPLS3eirFILP2/ruygaBypfLdaRY3KUKrL2/fP9MPWnQ/gy7Vyt5469Pxwd3lmjQFTKCvTzQtGtCqXDIBfQJJjT4h3Jfds3VejdBzrj3Qc6Kx2G1RwRq4eHxurEZP74nhjVs6kuqbOF759dOHKM/RF6zTfucn/aKtR494yft6es46VcRUyzEKVDIBfhLp8BtnZzO5o6oyKjmjU0zNTfGNYeIyWMKXE1I7o2wewx3SW1stS0b+oQ7H4rwWzry+iezeBvxzGM6dfavi4fZxnftwVeHtQGK5+tPaDdnf+2TFGiVySqIe/ASTnxdnZPOwqTExfw9dNxeGVwW4yMMSxQ9uLdbfCvv/Rwaix9oqXNpmlUz3RS4MjrwAMxkUga0g4BPl4Wu4X+OSYGB98dKuvxrVnPyNHahdfH98+bnkUHAN6eHkga2h6xLa1bWsAW+oOpOSSitv88XrdmMJJt7mjTCPV9bRqJYdaLd6tzVhyTExdwR5tQ/N+QdrKsemzvxeHLJ+OsnnnUt1VDzHrE+hV1n72rla1h1fLvcT3w8uC2Jp+v+XvwsjCDQ8g0ItfS2JVXE6pilqMlp1mDAPRq4bikw1oJHW8XHEwa0g6h9X3xfwnt0MjBhfRcRccm7lXHqS4IU2DV9Lvbh6FBPfnHK7JbhxzO3sTD14o/Un8fT/Rsbrn15ONxPbD8mXg0b2TdYNpfXroTU4Z3sGpbW1VfBEZ0iVCs6Jylwz7SOwrbptyNw+8NwwcPdcEvLxmuzrw+aYDjgpNAI6Hdq3Hg7SSkSbA/dr81GK8kmE4c9d3T9XZRPnNjj+6VcZ0ocg4pC48qLbBGi8Wbwxz7WWWKlP87V8fkhHTeurejbPu6P0baeIUuTYMltQz1aB4iMSJg2VOx+Mforvjgoa6SX+tMzRoEwMNDg/FxLdClabDBc22cuMRAbxtaXaJrJKMzR3dDmzDDRRurZ5VZk0z/85Huuu/NNT8P7ay+5SDc2V/6OHa9s++ft60ulKP8+nJ/3fcju0eanI5O8mFyQjrNGgRg5sPmlxZQg5RRXW0q4Naovi/G9mluU7+tqZaWmrNd3rqnI1Jfs651Q6hlzQATujYLtryRngAfT3TWS6bahwdijJ0XMX8fT/x1RAf8X0I7h04p9/f2hIemaiqxHO5oo65Bhm8Ma6/rMpSDHGOHWiuwlte3z5kfg2WKfguwo6pYkyEmJyrkrTf+IdjfMTVRHGn2GOvHmVirj96AzXGxzeHl6aG7exvd0zl1QozlEjUHvk66q5XTSv3X7OY4Mn0Ydr91e7VvZzYA928bal/FXzOeG9Da6m4gQPoyA0emD8O+qUOg0Wjwzn2dECNxCnX/toYFCjtEBCK0vvK1efQ9N6A1Xk1wzFpFtmoXHogwJ9YwCvb3tut/c+PrA/GP0V3xqIP+zmt6/8Eu2PLG3Ra302gMuz/dBZMTFfL00GDVC/2w/Jm+DivYZooco8GtvauScvfVMrQeNrw2APun3p5VM31kF/x3Uhw+HNVFYoT2+9ufXWARwX5Orwfy4t2tERvdEMn3GPZ7WzMzyVG+eirOqinf1g4sTv/bYHtDslqAj5cu9ibB/vjpxTskvb5BgGETvz1T353loR5NzT4/qmdTizO97OHtWfXP/8vLt8dU3dXOscs3/PZKf8sbmdEytB7G9mmuGzzv6Jaf2OiGtcbsGfvM3PHXQegQYd2g6vdcqM4WkxOV6tm8gd3l0W0xrHME7o+JVOUKra0a1zdI1ny8PNCvdagiC+Lpz0BxtjeGdcDK5+IVXQjQ0cvThAX5Wd5IBtaspGxJgI/6k5Ga2oabv7C++0Bns60/9vZIVrfw6a/Qbm4dLimaBBv/24mU4b3WN6a3Y8fdWEvKINmJdhS0dDYmJ27EVLVPKTw9NPh4XA88aeNCf0p7pFczrHnVtjukYX8OqpQ6mFdKC5Dax5kAwD8tTP/u1aIBBnVwXHLmyKJkNRMJaxZvtDROwRmLIJpq6Tg6fbhDjhfkZ32LrS2tdSF/tjb5ed++BMmV5En9F3ukl23dwp4OWAFaiVWl1fqZJH9FF3K69L8NRklZJRrZ0M8d1dAf56/dRJwdRbhs7cuWc1rcwgm9cSn/pl13BrPHdMfWE3kYIFPz8sfjeuDn/ZekLXCoApYutnLNpBjUIQwr91yo9bjcqwYPbN8Ym45fAQCM7N7UIJm0tG7U2/d2RJ9oaf8brRrXQ6VW3g/8Dx/qimGdw/Hcsr0Gj/sr1GojVzG9AB8vfPFEHwgIHL1cJM9OzdBPhoCqhU2bhvjj24zaf4fO8NKgNtiXlY9tJ/MAAPV8vVBws1yRWNSGLSduICzIDy1tbDXZ8NpAHHx3qM1T41o0CsBjfVsYPOanQHfDkE7hdjdZ1vP1wvAuEUY/8G0ZmHx/TCQWTmD1T1Om3d8Zf39QnvFC5u70lyTG2rzf3hITk0fjmtvVJTq8s/GBjf4+nhjexT1rudzdIcyhLXH6AnwM78ejGgbIUtzSVuYKRRqTNKTqRtDU34k7YXJSx9T8N/T29ECghCbcmvRnRiSP6IBhncMxpJP71Jz411+64+72jfGSxA8Rsqyer1etxNZWA9o1xpjezTB9ZGeMctLsLWM+fKirrsvCFr7e6v5IvqNNIzw/sLVD9q0/kNja6sH3dWuCY+8Px8D2jh1M6yjeElsJR3Zvip3Jg/GP0eov+WAvdf8nkEt5dkBrfPZ4b4ul4F3JyO5N8UVirF1TutVWUEpJjppN5OGhwcyHYzAhPhovDWpjU5E+Y6ROS66m9jqetrZufv1031qVnCfIlGA+GtscvVs0wJThHRAdWg8po7ris8d7mX2Nt6eHS8yOGtwhzKrtQutbTsoigv0M/sDcdb0q97mKkFXc9Q/ZGWwdRWCqaJPUuyZbSb1Qa/T+SD6f2Bux0Q3tXv9l1Qv98NVTsQgLdPwsHG9PDwxsZ93FwJQtb9yN317pr7p6JfZ6+96O6Ne6kaw1aV6QaeE4fx9PfPd8P13LzLjY5hhmQ/eFXOtgyWnRROu6dy0lY3Jb/IR6u52ZnBDZwJ4k77kBrRHXsqFLdH8N7hiOlc/Fo7mdM2h6Nm+A/m0d0/Q+71H512hp3ijA6oRMpZMdjHq6fyv8d1JfWQfSOmLWijlPyTST8B0nlkvQWPmB4awCjtWcNdbHFkxOiJzIz7uqHPuKZ+NlaTkZH2f5Drh6NpWtUybV7t5uzh0o+smjPQx+tic3MRf7hj+XQajnhBk5xqriKlXQzxK5kgpzK39HBPlh7tjushynmqnKw+8+0BkNArzxt3vkXUywUT0fo62mLaxcjFVpTE7IJuF/LhluS7NrXfTBQ13QqUkQ3hjWXtb9vja0Pfq2amh2yYAB7Rpj3ztDXGLdJFdwX7dIBNmxBs+HegtPtqgxlVm/FaZV4/rYP20o9ulVRZbT8b8Px8O9muGbSX3x04t31ColH+Djha1v3o205EH479NxWPVC3Rk79Xh8C5PF3Gz1vYl6OW3C6mPvO0PwzF0SBhqbyYj/Pa4HGgf64j8Tehtdad7DTCuO1GnzjsQ6J2ST3165C/vP5zu85LQrsKbBdnxcC4yPk2fgoL6G9Xyw/BnLZcYduYqqGu/E1Dq2qp6PJx6Na46//XDQqu0dubaWr5cnZlkouFddB6ZJsOMK40kxyIqBpWrtZjM3UcDabh+jr63x8wMxkbi/WxOr99lVb7HOe7qqZ7o6kxOyScN6PrjbyhHopqj1AkLW6duqIXaevib7QmjmPlQf6dVMsYJZUihVHM0e0iodOy4OU2aO7obRNbomu/x5YbUUun5rlTvyqdFCYu5/qOYzXp4eOJNyj10JkiOwW4fITTn6s+aLJ2Lx3XPxmNS/le6xO1qHmnmF/T54qKvNy97LqVOk+cGySUPao2OTIEUXWrN1GrRaNW3grxt8u/bVuzD1vk6YEG9da6QbVTcw6rG+LRDTLBivD7WxWrfKEhOALSdEVtOflKDWpmNHMbbUgL+PZ60KqmP7RFndZWELHy8PVfSLP9EvGjtPXzP5fONAX90quH7eHpjyvTy/k7G9o7Biz3mrtp0xqhu2nryCQF8vp8+ocbT2EYFoHxFo1z4a1nPuiu8AEBnsh0sFt2x6rbkEor6vF36afKfJ510RkxMiKzVvGIC72jVGoK+XSxR+spU9lzE5LoJqXYhMn5SZVmP7NJctOWkdZv0yFf4+Hkge0VGW46qBuT8LKXf+s8fEYP/5fAzt5PzB/OuSBmDF7vOY/ssRu/ajxloucnPzxi4i+Wg0Gnz5ZCzmjZe/rgZJE2jHbBkyrvpibes6Xea0DXNe/Y6/jqiakmuqy2dUz2Z4b2QXgzV1Fj/Ru9ZUX3tbZoyp5+uFtuEy/y7cq1FMh//hVCc1DfHHxfybSodBNnLTz2OLHNmo9Ld7OqJrs2Dc3d6+ge7GjOrZDP9Yc0z2/Zo61p1tQtE40Bdfpp2z6jWDOoRjUIdwpPx2FGsOZeOxvi0Q7O+N9LcGm62H4kgqHAbiVExO6pioBuqb9qmEJYl98P6vR/FqAhf0c0ft5L47rQP8fTwxpneULPtq3bgeTl0p0f2s33Iy3wktj2FBhjVKrB2nlDyiI/46vIOum8jScgu/vHQndpzKw+krJVi+27qxQGQdJid1zCsJbVF0qwL3x0QqHYqi2oYH4ssnY2XZVwM7VqEl2zSo54PCWxUmnx/WOQLvP9gFMc2Caz1nbHCvPVxgiIzTtW5cH/X9vLH/fH6t50Y4oJaGqXc0c+oQXC0pQysJZeGljF/p0jQYXZoG4y0HDgKXW3CA8wcC24JjTuqYQD9v/OPhbrizrWOnfFpDqeZSub3/YBfERjfEpxyL4jT/ebw3YpoFY6mJBFOj0eDxvi3QrVmIcwOTqPo6WD3tt0tT+xZYVJOWKijOFxLg4/T1apyhvu/tdgWpN0ezx3RHj+YhWDhBvYv+AWw5IQU9P7A1tp/Kw4Pdmyodil0iQ/yxUgW1N+qS9hGBTp062SasPk7mFqOhgyrtbnp9ICq0QlKBOUfFolYNFJj6ay1HDJ41x8vTA0emD4MQ0lc3bxlaDz+8cIeDIpMPkxNSTEiAD355qb/SYVANde2iZ40vnuiDTzedwtP9bVsRN65lQ+w6cw2PmBjT4eXpAS8rGhL1e5BcabykPT1f8x7tiQvXb6BzZO0uOrV4NLY5vkw7h5O5xU47ZoCPe1++3fvsiMjA8wNbY/6mU0ZXM140oTeu3yjTradCt0U1DEDKKNtLoC+a2Bu7Tl9D/3aO707tHhWCC9dv6kq7O4OHBtA6aOyNs1edtoWXpwfWvNIfbd76zeBxP28P2VZRrmuYnBDVIW8MbY/7u0UabYZO6BSuQER1Q6Cft1W/317NG9h9rFXP94NWCLMLzdlrUIcwLN99HoF/jn3Y985Q5JWUYvA/N+u2acaZgTj83vBahQl9rWkiIyYnRHWJh4fG4rowpJxOkUH48cU7EBFkfgqrvucGtMaCzafw9r1V1WA9PDTwcHCnz5BO4VjxTF+0Da9KcoMDvBEc4A0fTw+UVWrRv20oRvdqhqslpRjepQnKKrQOjUetNUGMVUzu3aIB7uvWBK0cUOzOnTA5ISKL2kUEYs3hbKXDqBO616hUasmU4e0xIb4FIkOct9CfRqNBXKtGtR7f/OZAZJy7jhFdmsDTQ4OUUd0AAOuO5Dg6IgfvXz4eHhp88ihn9lnC5ITIbcn3gf3CwNYQQiChI7t+AHXVNtFoNE5NTMxpEuyP+7qpIxZybUxOiMgiP29PvDa0vdJhqJJSi7D9JVaeaq7O4My1dcg9MDkhIlWJbWldqXG1UKoV5dWEdsoc2AbRofXw7XPxaMRp6jZznY4rebBCLBGpxpvD2+Otezn10hpSi28prU90Q0ll5F2Nl6cHJsa30M1gUgNHrDDtLK71101Ebu2FgW0MSnO7u+asKeNW3hvZBf8cE6N0GDp/Hd5R6RBsxuSEiNxe7xZV9UPG9lHHOI3/Ph2Hlwe1wcO9pMWT0DEMABAe5OuIsMjNBAd4414HLLToDHXnFoWI6qzlz/TFtZIyhAX5Ie3UVaXDQb82oejXRnq12DZhgdjx10FcYoDcHpMTInJ7Xp4eCPuzsJmrD8pUy7RhNVFrETayHZMTIqpTHurZFLvPXkO/NrWLiJFrYm7ifpicEFGd4u3pgY8eUc+gRSJj6nrCxQGxREREpCpMToiIiEhVmJwQERHJpLrCcVMOXLYLx5wQERHJJCTABwffHQo/b0+lQ3FpbDkhcjN9oqsKjj0ap46CY+6o95+/48aBLIamBmqbShzo5+1yywuoDVtOiNzMfyf1xaX8m2jRyHXX1VC7kAAfHHh3KPy8PLFw62mlwyFyO0ztiNyMt6eHoonJqB5NAQCTB7VRLAZnCPLzho8XP0LJObw8VdY85GD8zyIiWf1zTAx2Jg/G/TGRSodC5PIm390GXZsG4xGJ6zBVe7p/SwDAiC4RcoblcOzWISJZaTQaRAT7KR0G1SEaNy5Z9vqw9nh9WHubX9+jeQPsnzoUQf6udbl3rWiJiIhIkuAAb6VDkIzdOkREdgit79oLCRKpEVtOiIjsMLpnM+y/UIB+rbmQoFLUNpWY7MfkhIjIDl6eHvjwoa5Kh0HkVtitQ0RERKrC5ISIVIFN80RUjckJERERqQqTEyIiIlIVJidERESkKkxOiIjIpd3brQkAINKNKhP7eXsqHYKiOJWYiIhc2r1dm6DJ8/5oE1Zf6VDsotFokDyiAwpvlSOqYYDS4SiKyQkREbk0jUaDXi0aKB2GLJ4d0FrpEFSB3TpERESkKkxOiIiISFWYnBAREZGqMDkhIkV1bBIEABjWKULhSIhILWxKTubNm4fo6Gj4+fkhLi4O6enpJrc9fPgwRo8ejejoaGg0GsydO9fWWInIDX35ZCymj+yMmY90UzoUIlIJycnJihUrkJSUhGnTpmHv3r2IiYnBsGHDkJuba3T7GzduoFWrVpgxYwYiInhnRESGGgf6YkJ8NIL8vJUOhYhUQnJyMnv2bEyaNAmJiYno1KkTFixYgICAACxevNjo9n369MFHH32Ev/zlL/D19bU7YCIiInJvkpKTsrIyZGRkICEh4fYOPDyQkJCAtLQ02YIqLS1FYWGhwRcRERHVDZKSk7y8PFRWViI8PNzg8fDwcGRnZ8sWVEpKCoKDg3VfUVFRsu2biIiI1E2Vs3WSk5NRUFCg+zp//rzSIREREZGTSCpfHxoaCk9PT+Tk5Bg8npOTI+tgV19fX45PISIiqqMktZz4+PigV69eSE1N1T2m1WqRmpqK+Ph42YMjIiKiukfywn9JSUmYOHEievfujdjYWMydOxclJSVITEwEAEyYMAFNmzZFSkoKgKpBtEeOHNF9f/HiRWRmZqJ+/fpo06aNjKdCRERE7kBycjJ27FhcuXIFU6dORXZ2Nrp37441a9boBslmZWXBw+N2g8ylS5fQo0cP3c+zZs3CrFmzMGDAAGzatMn+MyAiIiK3ohFCCKWDsKSwsBDBwcEoKChAUFCQ0uEQERGRFWy9fqtytg4RERHVXUxOiIiISFWYnBAREZGqMDkhIiIiVZE8W0cJ1WN2ucYOERGR66i+bkude+MSyUlRUREAcI0dIiIiF1RUVITg4GCrt3eJqcRarRaXLl1CYGAgNBqNbPstLCxEVFQUzp8/77ZTlN39HHl+rs/dz5Hn5/rc/RwdeX5CCBQVFSEyMtKgBpolLtFy4uHhgWbNmjls/0FBQW75B6fP3c+R5+f63P0ceX6uz93P0VHnJ6XFpBoHxBIREZGqMDkhIiIiVanTyYmvry+mTZsGX19fpUNxGHc/R56f63P3c+T5uT53P0c1np9LDIglIiKiuqNOt5wQERGR+jA5ISIiIlVhckJERESqwuSEiIiIVKVOJyfz5s1DdHQ0/Pz8EBcXh/T0dKVDQkpKCvr06YPAwECEhYXhwQcfxPHjxw22GThwIDQajcHXc889Z7BNVlYW7r33XgQEBCAsLAxvvPEGKioqDLbZtGkTevbsCV9fX7Rp0wZLliypFY/cv6N33323VuwdOnTQPX/r1i28+OKLaNSoEerXr4/Ro0cjJyfHJc6tWnR0dK1z1Gg0ePHFFwG43vu3ZcsW3H///YiMjIRGo8GPP/5o8LwQAlOnTkWTJk3g7++PhIQEnDhxwmCba9euYfz48QgKCkJISAieeuopFBcXG2xz4MAB9O/fH35+foiKisLMmTNrxfLtt9+iQ4cO8PPzQ9euXbF69WrJsUg5v/LyckyZMgVdu3ZFvXr1EBkZiQkTJuDSpUsG+zD2ns+YMUMV52fpHAHgiSeeqBX/8OHDDbZx1fcQgNH/R41Gg48++ki3jZrfQ2uuC2r67LQmFotEHbV8+XLh4+MjFi9eLA4fPiwmTZokQkJCRE5OjqJxDRs2THzxxRfi0KFDIjMzU9xzzz2iefPmori4WLfNgAEDxKRJk8Tly5d1XwUFBbrnKyoqRJcuXURCQoLYt2+fWL16tQgNDRXJycm6bU6fPi0CAgJEUlKSOHLkiPj444+Fp6enWLNmjW4bR/yOpk2bJjp37mwQ+5UrV3TPP/fccyIqKkqkpqaKPXv2iL59+4p+/fq5xLlVy83NNTi/devWCQBi48aNQgjXe/9Wr14t3nrrLbFq1SoBQPzwww8Gz8+YMUMEBweLH3/8Uezfv1888MADomXLluLmzZu6bYYPHy5iYmLEzp07xdatW0WbNm3EuHHjdM8XFBSI8PBwMX78eHHo0CHxzTffCH9/f/HZZ5/pttm+fbvw9PQUM2fOFEeOHBFvv/228Pb2FgcPHpQUi5Tzy8/PFwkJCWLFihXi2LFjIi0tTcTGxopevXoZ7KNFixZi+vTpBu+p/v+skudn6RyFEGLixIli+PDhBvFfu3bNYBtXfQ+FEAbndfnyZbF48WKh0WjEqVOndNuo+T205rqgps9OS7FYo84mJ7GxseLFF1/U/VxZWSkiIyNFSkqKglHVlpubKwCIzZs36x4bMGCAeOWVV0y+ZvXq1cLDw0NkZ2frHps/f74ICgoSpaWlQggh3nzzTdG5c2eD140dO1YMGzZM97MjfkfTpk0TMTExRp/Lz88X3t7e4ttvv9U9dvToUQFApKWlqf7cTHnllVdE69athVarFUK49vtX84Nfq9WKiIgI8dFHH+key8/PF76+vuKbb74RQghx5MgRAUDs3r1bt81vv/0mNBqNuHjxohBCiE8//VQ0aNBAd35CCDFlyhTRvn173c9jxowR9957r0E8cXFx4tlnn7U6FqnnZ0x6eroAIM6dO6d7rEWLFmLOnDkmX6OW8xPC+DlOnDhRjBw50uRr3O09HDlypBg0aJDBY670Hta8Lqjps9OaWKxRJ7t1ysrKkJGRgYSEBN1jHh4eSEhIQFpamoKR1VZQUAAAaNiwocHjX3/9NUJDQ9GlSxckJyfjxo0buufS0tLQtWtXhIeH6x4bNmwYCgsLcfjwYd02+udfvU31+Tvyd3TixAlERkaiVatWGD9+PLKysgAAGRkZKC8vNzhmhw4d0Lx5c90x1X5uNZWVlWHZsmV48sknDRatdOX3T9+ZM2eQnZ1tcJzg4GDExcUZvGchISHo3bu3bpuEhAR4eHhg165dum3uuusu+Pj4GJzP8ePHcf36davO2ZpY5FBQUACNRoOQkBCDx2fMmIFGjRqhR48e+Oijjwyay13h/DZt2oSwsDC0b98ezz//PK5evWoQv7u8hzk5Ofj111/x1FNP1XrOVd7DmtcFNX12WhOLNVxi4T+55eXlobKy0uBNAoDw8HAcO3ZMoahq02q1ePXVV3HHHXegS5cuuscfffRRtGjRApGRkThw4ACmTJmC48ePY9WqVQCA7Oxso+dW/Zy5bQoLC3Hz5k1cv37dIb+juLg4LFmyBO3bt8fly5fx3nvvoX///jh06BCys7Ph4+NT60M/PDzcYtxqODdjfvzxR+Tn5+OJJ57QPebK719N1fEYO45+rGFhYQbPe3l5oWHDhgbbtGzZstY+qp9r0KCByXPW34elWOx169YtTJkyBePGjTNYIO3ll19Gz5490bBhQ+zYsQPJycm4fPkyZs+e7RLnN3z4cIwaNQotW7bEqVOn8Le//Q0jRoxAWloaPD093eo9XLp0KQIDAzFq1CiDx13lPTR2XVDTZ6c1sVijTiYnruLFF1/EoUOHsG3bNoPHn3nmGd33Xbt2RZMmTTB48GCcOnUKrVu3dnaYkowYMUL3fbdu3RAXF4cWLVpg5cqV8Pf3VzAyx/j8888xYsQIREZG6h5z5fevLisvL8eYMWMghMD8+fMNnktKStJ9361bN/j4+ODZZ59FSkqKqkqCm/KXv/xF933Xrl3RrVs3tG7dGps2bcLgwYMVjEx+ixcvxvjx4+Hn52fwuKu8h6auC+6mTnbrhIaGwtPTs9bo4ZycHERERCgUlaHJkyfjl19+wcaNG9GsWTOz28bFxQEATp48CQCIiIgwem7Vz5nbJigoCP7+/k77HYWEhKBdu3Y4efIkIiIiUFZWhvz8fJPHdKVzO3fuHNavX4+nn37a7Hau/P5V78vccSIiIpCbm2vwfEVFBa5duybL+6r/vKVYbFWdmJw7dw7r1q2zuKx8XFwcKioqcPbsWbOx68et5PnV1KpVK4SGhhr8Tbr6ewgAW7duxfHjxy3+TwLqfA9NXRfU9NlpTSzWqJPJiY+PD3r16oXU1FTdY1qtFqmpqYiPj1cwsqppZpMnT8YPP/yADRs21GpGNCYzMxMA0KRJEwBAfHw8Dh48aPBhUv2B2qlTJ902+udfvU31+Tvrd1RcXIxTp06hSZMm6NWrF7y9vQ2Oefz4cWRlZemO6Urn9sUXXyAsLAz33nuv2e1c+f1r2bIlIiIiDI5TWFiIXbt2Gbxn+fn5yMjI0G2zYcMGaLVaXWIWHx+PLVu2oLy83OB82rdvjwYNGlh1ztbEYovqxOTEiRNYv349GjVqZPE1mZmZ8PDw0HWFqPn8jLlw4QKuXr1q8Dfpyu9htc8//xy9evVCTEyMxW3V9B5aui6o6bPTmlisYvXQWTezfPly4evrK5YsWSKOHDkinnnmGRESEmIwklkJzz//vAgODhabNm0ymNJ248YNIYQQJ0+eFNOnTxd79uwRZ86cET/99JNo1aqVuOuuu3T7qJ4yNnToUJGZmSnWrFkjGjdubHTK2BtvvCGOHj0q5s2bZ3TKmNy/o9dee01s2rRJnDlzRmzfvl0kJCSI0NBQkZubK4SomoLWvHlzsWHDBrFnzx4RHx8v4uPjXeLc9FVWVormzZuLKVOmGDzuiu9fUVGR2Ldvn9i3b58AIGbPni327dunm60yY8YMERISIn766Sdx4MABMXLkSKNTiXv06CF27doltm3bJtq2bWswDTU/P1+Eh4eLxx9/XBw6dEgsX75cBAQE1Jqm6eXlJWbNmiWOHj0qpk2bZnSapqVYpJxfWVmZeOCBB0SzZs1EZmamwf9k9QyHHTt2iDlz5ojMzExx6tQpsWzZMtG4cWMxYcIEVZyfpXMsKioSr7/+ukhLSxNnzpwR69evFz179hRt27YVt27dcvn3sFpBQYEICAgQ8+fPr/V6tb+Hlq4LQqjrs9NSLNaos8mJEEJ8/PHHonnz5sLHx0fExsaKnTt3Kh2SAGD064svvhBCCJGVlSXuuusu0bBhQ+Hr6yvatGkj3njjDYM6GUIIcfbsWTFixAjh7+8vQkNDxWuvvSbKy8sNttm4caPo3r278PHxEa1atdIdQ5/cv6OxY8eKJk2aCB8fH9G0aVMxduxYcfLkSd3zN2/eFC+88IJo0KCBCAgIEA899JC4fPmyS5ybvrVr1woA4vjx4waPu+L7t3HjRqN/kxMnThRCVE2PfOedd0R4eLjw9fUVgwcPrnXeV69eFePGjRP169cXQUFBIjExURQVFRlss3//fnHnnXcKX19f0bRpUzFjxoxasaxcuVK0a9dO+Pj4iM6dO4tff/3V4HlrYpFyfmfOnDH5P1ldtyYjI0PExcWJ4OBg4efnJzp27Cg+/PBDgwu7kudn6Rxv3Lghhg4dKho3biy8vb1FixYtxKRJk2olsa76Hlb77LPPhL+/v8jPz6/1erW/h5auC0Ko67PTmlgs0fx54kRERESqUCfHnBAREZF6MTkhIiIiVWFyQkRERKrC5ISIiIhUhckJERERqQqTEyIiIlIVJidERESkKkxOiIiISFWYnBAREZGqMDkhIiIiVWFyQkRERKrC5ISIiIhU5f8BExYo3z/S1cQAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# calibrate the batch norm at the end of training\n",
        "\n",
        "# with torch.no_grad():\n",
        "#   # pass the training set through\n",
        "#   emb = C[Xtr]\n",
        "#   embcat = emb.view(emb.shape[0], -1)\n",
        "#   hpreact = embcat @ W1 # + b1\n",
        "#   # measure the mean/std over the entire training set\n",
        "#   bnmean = hpreact.mean(0, keepdim=True)\n",
        "#   bnstd = hpreact.std(0, keepdim=True)"
      ],
      "metadata": {
        "id": "FAlwBVXDaXjs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "##  https://pytorch.org/docs/stable/generated/torch.no_grad.html\n",
        "\n",
        "@torch.no_grad() # this decorator disables gradient tracking\n",
        "def split_loss(split):\n",
        "  x,y = {\n",
        "    'train': (Xtr, Ytr),\n",
        "    'val': (Xdev, Ydev),\n",
        "    'test': (Xte, Yte),\n",
        "  }[split]\n",
        "  emb = C[x] # (N, block_size, n_embd)\n",
        "  embcat = emb.view(emb.shape[0], -1) # concat into (N, block_size * n_embd)\n",
        "  hpreact = embcat @ W1\n",
        "  # hpreact = bngain * (hpreact - hpreact.mean(0, keepdim=True)) / hpreact.std(0, keepdim=True) + bnbias\n",
        "  hpreact = bngain * (hpreact - bnmean_running) / bnstd_running + bnbias\n",
        "  h = torch.tanh(hpreact) # (N, n_hidden)\n",
        "  logits = h @ W2 + b2 # (N, vocab_size)\n",
        "  loss = F.cross_entropy(logits, y)\n",
        "  print(split, loss.item())\n",
        "\n",
        "split_loss('train')\n",
        "split_loss('val')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "haGlYos-940x",
        "outputId": "2a9bf36b-806b-42e8-d015-4ca1b4ff7ad8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train 2.066537618637085\n",
            "val 2.1117324829101562\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# sample from the model\n",
        "g = torch.Generator().manual_seed(242222883 + 10)\n",
        "\n",
        "for _ in range(20):\n",
        "\n",
        "    out = []\n",
        "    context = [0] * block_size # initialize with all ...\n",
        "    while True:\n",
        "      emb = C[torch.tensor([context])] # (1,block_size,d)\n",
        "      h = torch.tanh(emb.view(1, -1) @ W1)\n",
        "      logits = h @ W2 + b2\n",
        "      probs = F.softmax(logits, dim=1)\n",
        "      ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
        "      context = context[1:] + [ix]\n",
        "      out.append(ix)\n",
        "      if ix == 0:\n",
        "        break\n",
        "\n",
        "    print(''.join(itos[i] for i in out))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AoxNiWv--i3p",
        "outputId": "f135906a-bdd2-403a-91c2-0cb150ba6faf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "qaibht.\n",
            "tygigraclyr.\n",
            "fazviviz.\n",
            "fatmu.\n",
            "phmbiq.\n",
            "gricelyssafrantim.\n",
            "kevanyah.\n",
            "jasbbiahmsttv.\n",
            "qulbeifatsh.\n",
            "britshquenvighl.\n",
            "feymastamris.\n",
            "sha.\n",
            "selssimredgrefttonsthugfklamsimdhvi.\n",
            "faidhkauriplrrang.\n",
            "kwexthal.\n",
            "fahdias.\n",
            "mckerranncharmaribfir.\n",
            "vnyyah.\n",
            "blestap.\n",
            "faghn.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "1:03"
      ],
      "metadata": {
        "id": "124Rzm7se7wu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}