From 7f52e1c6d5e37a142ca573dc50f259dcb34e350d Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Tue, 2 Apr 2024 19:48:02 -0700 Subject: [PATCH] save minor logging --- .gitignore | 1 + auglm/analyze_results.ipynb | 68 +++++++++++++++++++++++++++++++++++++ auglm/lm.py | 21 ++++++++---- 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 auglm/analyze_results.ipynb diff --git a/.gitignore b/.gitignore index efee252..22d6c66 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.zip babylm_data **cache* +**.joblib diff --git a/auglm/analyze_results.ipynb b/auglm/analyze_results.ipynb new file mode 100644 index 0000000..73d4746 --- /dev/null +++ b/auglm/analyze_results.ipynb @@ -0,0 +1,68 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The autoreload extension is already loaded. To reload it, use:\n", + " %reload_ext autoreload\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAHVCAYAAABxKsZjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABmP0lEQVR4nO3dd3hUVf7H8fedmUx6IQQSSgKR0HsRCDaUEoR1Qf1ZUbCsbcEVcS3sWlddXFdUXFlRV0F3ZbEsYkGFCIIKoRMJSJcqhAAhnSSTmfv7Y8hATBAISSYz+byeZ56ZuffMnW8OET7ee885hmmaJiIiIiLiMyzeLkBEREREzo4CnIiIiIiPUYATERER8TEKcCIiIiI+RgFORERExMcowImIiIj4GAU4ERERER+jAFdDTNMkLy8PTasnIiIitU0Brobk5+cTGRlJfn5+jR/b4XDwySef4HA4avzYUpH6um6on+uO+rruqK/rhvrZTQFORERExMcowImIiIj4GAU4ERERER+jACciIiLiYxTgRERERHyMApyIiIiIj1GAExEREfExCnAiIiIiPkYBTkRERMTHKMCJiIiI+BgFOBEREREfU28C3HPPPYdhGEyYMKHSPtM0ufzyyzEMg7lz51bYt2fPHkaMGEFISAhNmzblwQcfpKysrEKbxYsX06tXLwIDA0lKSmLmzJmVvmPatGm0bt2aoKAg+vXrx8qVK2vwpxMRERGpOfUiwK1atYrXX3+dbt26Vbn/5ZdfxjCMStudTicjRoygtLSUZcuW8c477zBz5kwef/xxT5udO3cyYsQILr30UtLT05kwYQK/+93vmD9/vqfN+++/z8SJE3niiSdYu3Yt3bt3JyUlhaysrJr/YUVERETOkc3bBRQUFDB69GjefPNNnnnmmUr709PTmTJlCqtXr6ZZs2YV9i1YsIAff/yRr7/+mtjYWHr06MHTTz/Nww8/zJNPPondbmf69OkkJiYyZcoUADp27Mj333/PSy+9REpKCgAvvvgid9xxB7feeisA06dPZ968ebz99ts88sgjVdZdUlJCSUmJ531eXh4ADocDh8Nx7h1zkvLj1fRxpTL1dd1QP9cd9XXdUV/XjYbQzwEBAadt4/UAN27cOEaMGMHgwYMrBbiioiJuvPFGpk2bRlxcXKXPpqWl0bVrV2JjYz3bUlJSuOeee9i4cSM9e/YkLS2NwYMHV/hcSkqK51JtaWkpa9asYdKkSZ79FouFwYMHk5aWdsq6J0+ezFNPPVVp+4IFCwgJCTmjn/1spaam1spxpTL1dd1QP9cd9XXdUV/XDX/u55EjR562jVcD3OzZs1m7di2rVq2qcv/999/PgAEDTvmDZGZmVghvgOd9Zmbmr7bJy8vj2LFjHD16FKfTWWWbzZs3n7L2SZMmMXHiRM/7vLw84uPjGTp0KBEREaf8XHU4HA5SU1MZMmTIGaVyqT71dd1QP9cd9XXdUV/XDfWzm9cC3N69e7nvvvtITU0lKCio0v5PP/2URYsWsW7dOi9Ud3qBgYEEBgZW2h4QEFBrv1CnOnZmbjH/Xr6LMpfJpMs71sp3NzS1+ecoJ6if6476uu6or+tGQ+9nrw1iWLNmDVlZWfTq1QubzYbNZmPJkiW88sor2Gw2UlNT2bFjB1FRUZ79AFdffTUDBw4EIC4ujoMHD1Y4bvn78kuup2oTERFBcHAwMTExWK3WKttUddm2PiosLWPaNzv4T9pub5ciIiIidcBrAW7QoEFkZGSQnp7uefTp04fRo0eTnp7On//8Z9avX19hP8BLL73EjBkzAEhOTiYjI6PCaNHU1FQiIiLo1KmTp83ChQsrfHdqairJyckA2O12evfuXaGNy+Vi4cKFnjb1XVyE+wxmYamT/GL/valTRERE3Lx2CTU8PJwuXbpU2BYaGkrjxo0926s6A5aQkEBiYiIAQ4cOpVOnTtx88808//zzZGZm8uijjzJu3DjP5c27776bV199lYceeojbbruNRYsW8cEHHzBv3jzPMSdOnMjYsWPp06cPffv25eWXX6awsNAzKrW+Cw20ERFkI6+4jMzcYsKDGu4pZRERkYbA66NQz4XVauXzzz/nnnvuITk5mdDQUMaOHctf/vIXT5vExETmzZvH/fffz9SpU2nZsiX/+te/PFOIAFx33XUcOnSIxx9/nMzMTHr06MFXX31VaWBDfRYXGURecQEHcotpGxvu7XJERESkFtWrALd48eJf3W+aZqVtrVq14osvvvjVzw0cOPC0gyHGjx/P+PHjT1tjfRUXGczWgwVk5hV7uxQRERGpZfViJQY5d82O3weXmasAJyIi4u8U4PxEbKQ7wB1QgBMREfF7CnB+otnxAHdQl1BFRET8ngKcn4jTGTgREZEGQwHOT8R57oE75uVKREREpLYpwPmJ8kuoR4scFDucXq5GREREapMCnJ+IDA4gKMD9x6n74ERERPybApyfMAzjpMuoCnAiIiL+TAHOj5QPZNBkviIiIv5NAc6PNIsMBjQSVURExN8pwPmRWF1CFRERaRAU4PxI+UhUBTgRERH/pgDnR8rPwB3QPXAiIiJ+TQHOj3iW09IZOBEREb+mAOdHygNcVn4xZU6Xl6sRERGR2qIA50cahwVitRi4TDhUUOLtckRERKSWKMD5EavFIDY8ENBABhEREX+mAOdn4jQSVURExO8pwPmZ8gCnyXxFRET8lwKcn4mLcK/GoAXtRURE/JcCnJ9ppjNwIiIifk8Bzs/EakF7ERERv6cA52e0nJaIiIj/U4DzM3ERJ87Amabp5WpERESkNijA+Zny9VBLy1wcLXJ4uRoRERGpDQpwfsZusxATZgfgQO4xL1cjIiIitUEBzg9pMl8RERH/pgDnh06+D05ERET8jwKcH9IZOBEREf+mAOeHmkW6V2PQZL4iIiL+SQHOD5WPRNVyWiIiIv5JAc4PaTktERER/6YA54fK74E7qAAnIiLilxTg/FD5KNT8kjLyizWZr4iIiL9RgPNDoYE2woNsgO6DExER8UcKcH7qxKL2JV6uRERERGqaApyfKh+JquW0RERE/I8CnJ9qpsl8RURE/JYCnJ+KOz6Zr5bTEhER8T8KcH7Ksx6qzsCJiIj4HQU4P6XJfEVERPyXApyf8kzmq0uoIiIifkcBzk+VX0I9UlhKscPp5WpERESkJinA+amokAACbe4/3qw8zQUnIiLiTxTg/JRhGCemEtFlVBEREb+iAOfHNJmviIiIf1KA82PNNJBBRETELynA+bHyyXw1lYiIiIh/UYDzY3ERgYAm8xUREfE3CnB+TMtpiYiI+CcFOD+mBe1FRET8kwKcHytfjSErvwSny/RyNSIiIlJTFOD8WExYIFaLgdNlcrhAk/mKiIj4CwU4P2a1GMSGuwcyaCSqiIiI/1CA83OxnvvgNJmviIiIv1CA83MayCAiIuJ/FOD8XFzE8cl8NZWIiIiI31CA83NxkZrMV0RExN8owPk5z2S+CnAiIiJ+QwHOz3nugdMlVBEREb+hAOfn4iJODGIwTU3mKyIi4g8U4Pxc0+ML2peUucgpcni5GhEREakJCnB+LtBmJSbMDmgyXxEREX+hANcAxB6/jHpQ98GJiIj4BQW4BqB8IIPOwImIiPgHBbgGIDZCy2mJiIj4EwW4BkBTiYiIiPgXBbgGoHwyX11CFRER8Q8KcA3AyXPBiYiIiO9TgGsA4nQJVURExK8owDUA5QEuv7iMgpIyL1cjIiIi50oBrgEIC7QRHmgDdBlVRETEHyjANRDlZ+E0ma+IiIjvU4BrIOI0ma+IiIjfUIBrIOK0nJaIiIjfUIBrIE4sp6XVGERERHydAlwDUT6ZrwYxiIiI+D4FuAYiLjIQ0FxwIiIi/qDeBLjnnnsOwzCYMGECANnZ2dx77720b9+e4OBgEhIS+MMf/kBubm6Fz+3Zs4cRI0YQEhJC06ZNefDBBykrqzjX2eLFi+nVqxeBgYEkJSUxc+bMSt8/bdo0WrduTVBQEP369WPlypW19aN6RVyEzsCJiIj4i3oR4FatWsXrr79Ot27dPNv279/P/v37eeGFF9iwYQMzZ87kq6++4vbbb/e0cTqdjBgxgtLSUpYtW8Y777zDzJkzefzxxz1tdu7cyYgRI7j00ktJT09nwoQJ/O53v2P+/PmeNu+//z4TJ07kiSeeYO3atXTv3p2UlBSysrLqpgPqQPMo9z1whwtKOVbq9HI1IiIici5s3i6goKCA0aNH8+abb/LMM894tnfp0oX//e9/nvdt2rTh2Wef5aabbqKsrAybzcaCBQv48ccf+frrr4mNjaVHjx48/fTTPPzwwzz55JPY7XamT59OYmIiU6ZMAaBjx458//33vPTSS6SkpADw4osvcscdd3DrrbcCMH36dObNm8fbb7/NI488UmXdJSUllJSUeN7n5eUB4HA4cDgcNdpH5cc7l+OGBhg0CgngaJGDbZm5dGwWXlPl+ZWa6Gs5PfVz3VFf1x31dd1oCP0cEBBw2jZeD3Djxo1jxIgRDB48uEKAq0pubi4RERHYbO6y09LS6Nq1K7GxsZ42KSkp3HPPPWzcuJGePXuSlpbG4MGDKxwnJSXFc6m2tLSUNWvWMGnSJM9+i8XC4MGDSUtLO2UtkydP5qmnnqq0fcGCBYSEhJz2566O1NTUc/p8lNXKUQz+l/o9vWLMGqrKP51rX8uZUT/XHfV13VFf1w1/7ueRI0eeto1XA9zs2bNZu3Ytq1atOm3bw4cP8/TTT3PnnXd6tmVmZlYIb4DnfWZm5q+2ycvL49ixYxw9ehSn01llm82bN5+ynkmTJjFx4kTP+7y8POLj4xk6dCgRERGn/XnOhsPhIDU1lSFDhpxRKj+VpaUb2bnmZyJatmX4ZUk1WKH/qKm+ll+nfq476uu6o76uG+pnN68FuL1793LfffeRmppKUFDQr7bNy8tjxIgRdOrUiSeffLJuCjyNwMBAAgMDK20PCAiotV+ocz1229gI4Gd2HjnWoH/pz0Rt/jnKCernuqO+rjvq67rR0PvZa4MY1qxZQ1ZWFr169cJms2Gz2ViyZAmvvPIKNpsNp9N9o31+fj7Dhg0jPDycjz/+uMIfVlxcHAcPHqxw3PL3cXFxv9omIiKC4OBgYmJisFqtVbYpP4a/aNM0FIDtWQVerkRERETOhdcC3KBBg8jIyCA9Pd3z6NOnD6NHjyY9PR2r1UpeXh5Dhw7Fbrfz6aefVjpTl5ycTEZGRoXRoqmpqURERNCpUydPm4ULF1b4XGpqKsnJyQDY7XZ69+5doY3L5WLhwoWeNv4iqYl74MLOw4U4XboHTkRExFd57RJqeHg4Xbp0qbAtNDSUxo0b06VLF094Kyoq4j//+Q95eXmekZ5NmjTBarUydOhQOnXqxM0338zzzz9PZmYmjz76KOPGjfNc3rz77rt59dVXeeihh7jttttYtGgRH3zwAfPmzfN878SJExk7dix9+vShb9++vPzyyxQWFnpGpfqLFo2CsdsslJS52J9zjPjo2hlsISIiIrXL66NQT2Xt2rWsWLECgKSkijfc79y5k9atW2O1Wvn888+55557SE5OJjQ0lLFjx/KXv/zF0zYxMZF58+Zx//33M3XqVFq2bMm//vUvzxQiANdddx2HDh3i8ccfJzMzkx49evDVV19VGtjg66wWg/NiQtmcmc/2rAIFOBERER9VrwLc4sWLPa8HDhyIaZ7+Ml+rVq344osvfrXNwIEDWbdu3a+2GT9+POPHjz+jOn1Zm6ZhbM7MZ8ehAi7t0NTb5YiIiEg11IuVGKTutGkSBsCOQxrIICIi4qsU4BqYNk00ElVERMTXKcA1MElNy8/AFXq5EhEREakuBbgG5rwYd4DLLiwlu7DUy9WIiIhIdSjANTDBdistooIB3QcnIiLiqxTgGiDPZVTdByciIuKTFOAaoPKRqBrIICIi4psU4Bqg8jVRdQlVRETENynANUBJTTQSVURExJcpwDVAbY7fA7f3aBHFDqeXqxEREZGzpQDXADUOtRMZHIBpws7DOgsnIiLiaxTgGiDDME6a0Ff3wYmIiPgaBbgGSktqiYiI+C4FuAZKS2qJiIj4LgW4Bqp8LjhN5isiIuJ7FOAaqPIA99PhAlwu08vViIiIyNlQgGug4qNDsFstFDtc/JxzzNvliIiIyFlQgGugrBaDxBityCAiIuKLFOAasPIltTQSVURExLcowDVgWlJLRETENynANWBtNJmviIiIT1KAa8A0lYiIiIhvUoBrwM47vhrDkcJSjhaWerkaEREROVMKcA1YiN1Gi6hgwD0fnIiIiPgGBbgG7jytiSoiIuJzFOAaOK2JKiIi4nsU4Bq48oEMOgMnIiLiOxTgGjjPSFRNJSIiIuIzFOAauPJLqHuziyh2OL1cjYiIiJwJBbgGLibMTkSQDZcJu47oPjgRERFfoADXwBmGcWJFhiwFOBEREV+gACcnrYmq++BERER8gQKceM7AaSSqiIiIb1CAE41EFRER8TEKcOIZifrToUJcLtPL1YiIiMjpKMAJ8Y2CCbAaHHM42Z97zNvliIiIyGkowAk2q4XWjd1rompJLRERkfpPAU6Ak9ZE1UAGERGRek8BToCT1kTVQAYREZF6TwFOAJ2BExER8SUKcAKcPJWI7oETERGp7xTgBIDzmrgHMRwuKCG3yOHlakREROTXKMAJAKGBNppHBgG6D05ERKS+U4ATjza6D05ERMQnKMCJh5bUEhER8Q0KcOLhOQOnACciIlKvKcCJR5vjAxm26xKqiIhIvaYAJx5Jxy+h7skuoqTM6eVqRERE5FQU4MSjSXggjUICcJmwJTPf2+WIiIjIKSjAiYdhGHSPjwIgfW+OV2sRERGRU1OAkwq6t4wCFOBERETqMwU4qaDH8TNwPyjAiYiI1FsKcFJBt5aRgHtN1LxiLaklIiJSHynASQWNwwKJjw4GIGNfrperERERkaoowEklug9ORESkflOAk0p0H5yIiEj9pgAnlZRPJfLDvhyv1iEiIiJVU4CTSjo3j8BqMTiYV0JmbrG3yxEREZFfUICTSkLsNtrFhgO6D05ERKQ+UoCTKvWId08nosuoIiIi9Y8CnFSpfCSqBjKIiIjUP9UKcDNmzKCoqKima5F6pEdCFADr9+XicpneLUZEREQqqFaAe+SRR4iLi+P2229n2bJlNV2T1ANtm4YTYrdSUFLGT4cLvF2OiIiInKRaAe7nn3/mnXfe4fDhwwwcOJAOHTrwt7/9jczMzJquT7zEajHo0sJ9H1z6Xq3IICIiUp9UK8DZbDauvPJKPvnkE/bu3csdd9zBe++9R0JCAr/97W/55JNPcLlcNV2r1DFN6CsiIlI/nfMghtjYWC688EKSk5OxWCxkZGQwduxY2rRpw+LFi2ugRPEWz0AGjUQVERGpV6od4A4ePMgLL7xA586dGThwIHl5eXz++efs3LmTn3/+mWuvvZaxY8fWZK1Sx7ofn0pk04E8ih1OL1cjIiIi5aoV4K644gri4+OZOXMmd9xxBz///DP//e9/GTx4MAChoaE88MAD7N27t0aLlbrVIiqYmDA7DqfJpgN53i5HREREjrNV50NNmzZlyZIlJCcnn7JNkyZN2LlzZ7ULE+8zDIPuLaNYuDmLH/bm0DOhkbdLEhEREap5Bu6SSy6hV69elbaXlpby7rvvAu5//Fu1anVu1YnXnVjYXiNRRURE6otqBbhbb72V3NzK/6Dn5+dz6623nnNRUn9010hUERGReqdaAc40TQzDqLR93759REZGnnNRUn90b+n+8/zpcCG5RQ4vVyMiIiJwlvfA9ezZE8MwMAyDQYMGYbOd+LjT6WTnzp0MGzasxosU74kKsdO6cQi7jhTxw74cLm7XxNsliYiINHhnFeBGjRoFQHp6OikpKYSFhXn22e12WrduzdVXX12jBYr3dY+Pcge4vQpwIiIi9cFZBbgnnngCgNatW3PdddcRFBRUK0VJ/dK9ZRSfpO/XhL4iIiL1RLWmEdEEvQ1L+UCG9L25p7z/UUREROrOGQe46Ohotm7dSkxMDI0aNfrVf8Szs7NrpDipHzo3j8BmMThcUML+3GJaRAV7uyQREZEG7YwD3EsvvUR4eLjntc7CNBxBAVY6NAtnw895/LA3RwFORETEy854GpGxY8cSGBgIwC233MLYsWNP+aiO5557DsMwmDBhgmdbcXEx48aNo3HjxoSFhXH11Vdz8ODBCp/bs2cPI0aMICQkhKZNm/Lggw9SVlZWoc3ixYvp1asXgYGBJCUlMXPmzErfP23aNFq3bk1QUBD9+vVj5cqV1fo5/JVnYXvNByciIuJ11ZoHrqoABFBWVsakSZPO+nirVq3i9ddfp1u3bhW233///Xz22Wd8+OGHLFmyhP3793PVVVd59judTkaMGEFpaSnLli3jnXfeYebMmTz++OOeNjt37mTEiBFceumlpKenM2HCBH73u98xf/58T5v333+fiRMn8sQTT7B27Vq6d+9OSkoKWVlZZ/2z+KsT98HleLUOERERqeYghj/84Q/MmzePN954g0aN3OtjbtmyhRtvvJEjR44wefLkMz5WQUEBo0eP5s033+SZZ57xbM/NzeWtt95i1qxZXHbZZQDMmDGDjh07snz5cvr378+CBQv48ccf+frrr4mNjaVHjx48/fTTPPzwwzz55JPY7XamT59OYmIiU6ZMAaBjx458//33vPTSS6SkpADw4osvcscdd3hWkZg+fTrz5s3j7bff5pFHHqmy7pKSEkpKSjzv8/Lci707HA4cjpqd8Lb8eDV93LPRJc49ZUzGz7kUl5RitfjnJfT60NcNgfq57qiv6476um40hH4OCAg4bZtqBbh169Zx00030bVrV2bMmMHWrVt56KGHGDVqFP/85z/P6ljjxo1jxIgRDB48uEKAW7NmDQ6Hg8GDB3u2dejQgYSEBNLS0ujfvz9paWl07dqV2NhYT5uUlBTuueceNm7cSM+ePUlLS6twjPI25ZdqS0tLWbNmTYUzhxaLhcGDB5OWlnbKuidPnsxTTz1VafuCBQsICQk5qz44U6mpqbVy3DPhMiHQYqWo1MmMOV/SvHZ+xHrDm33dkKif6476uu6or+uGP/fzyJEjT9umWgGuTZs2LF26lAkTJjBs2DCsVivvvPMON9xww1kdZ/bs2axdu5ZVq1ZV2peZmYndbicqKqrC9tjYWDIzMz1tTg5v5fvL9/1am7y8PI4dO8bRo0dxOp1Vttm8efMpa580aRITJ070vM/LyyM+Pp6hQ4cSERFxmp/87DgcDlJTUxkyZMgZpfLaMvvgKlbsPEpkYneG927htTpqU33pa3+nfq476uu6o76uG+pnt2oFOIB58+Yxe/ZskpOT2bp1K2+99RaXXHIJzZs3P6PP7927l/vuu4/U1FSfnBA4MDDQM6jjZAEBAbX2C1Wbxz4TPROiWbHzKBsO5HOjn/9H4+2+bijUz3VHfV131Nd1o6H3c7UGMdx1111cc801PPzww3z33XesX78eu91O165d+eCDD87oGGvWrCErK4tevXphs9mw2WwsWbKEV155BZvNRmxsLKWlpeTk5FT43MGDB4mLiwMgLi6u0qjU8venaxMREUFwcDAxMTFYrdYq25QfQ9x6xLsXttdIVBEREe+qVoBbunQpK1as4IEHHsAwDOLi4vjiiy/4y1/+wm233XZGxxg0aBAZGRmkp6d7Hn369GH06NGe1wEBASxcuNDzmS1btrBnzx6Sk5MBSE5OJiMjo8Jo0dTUVCIiIujUqZOnzcnHKG9Tfgy73U7v3r0rtHG5XCxcuNDTRtzKR6Juzsyn2OH0bjEiIiINWLUuoa5Zs6bKy4fjxo2rNGDgVMLDw+nSpUuFbaGhoTRu3Niz/fbbb2fixIlER0cTERHBvffeS3JyMv379wdg6NChdOrUiZtvvpnnn3+ezMxMHn30UcaNG+ep7+677+bVV1/loYce4rbbbmPRokV88MEHzJs3z/O9EydOZOzYsfTp04e+ffvy8ssvU1hY6BmVKm5xEUE0DQ8kK7+Ejftz6d0q2tsliYiINEjVOgMXGBjIjh07ePTRR7nhhhs8Z8C+/PLLSpPonouXXnqJ3/zmN1x99dVcfPHFxMXFMWfOHM9+q9XK559/jtVqJTk5mZtuuokxY8bwl7/8xdMmMTGRefPmkZqaSvfu3ZkyZQr/+te/PFOIAFx33XW88MILPP744/To0YP09HS++uqrSgMbGjrDMCqsiyoiIiLeUa0zcEuWLOHyyy/nggsu4Ntvv+XZZ5+ladOm/PDDD7z11lt89NFH1Spm8eLFFd4HBQUxbdo0pk2bdsrPtGrVii+++OJXjztw4EDWrVv3q23Gjx/P+PHjz7jWhqpHfBSpPx7UfXAiIiJeVK0zcI888gjPPPMMqamp2O12z/bLLruM5cuX11hxUv94ltTal+PVOkRERBqyagW4jIwMrrzyykrbmzZtyuHDh8+5KKm/urZ0j0TdfaSI7MJSL1cjIiLSMFUrwEVFRXHgwIFK29etW0eLFv45wau4RQYH0C7WvaxW2o4jXq5GRESkYapWgLv++ut5+OGHyczMxDAMXC4XS5cu5Y9//CNjxoyp6RqlnrmobRMAvt9+yMuViIiINEzVCnB//etf6dChA/Hx8RQUFNCpUycuvvhiBgwYwKOPPlrTNUo9c1HbGAC+3XoY0zS9XI2IiEjDU61RqHa7nTfffJPHHnuMDRs2UFBQQM+ePWnbtm1N1yf1UL/ExtitFn7OOcbOw4Wc1yTM2yWJiIg0KNVeCxUgISGBhISEmqpFfESw3Uqf1o1YtuMI328/rAAnIiJSx844wE2cOPGMD/riiy9WqxjxHRe2jWHZjiN8u/UwY5Jbe7scERGRBuWMA9zpJsItZxhGtYsR33Fx2yY8/9UWlv90BIfTRYC1WrdTioiISDWccYD75ptvarMO+TXFxwgpOebtKiro1CyC6FA72YWlpO/N4fzWWhdVRESkrpzzaZO9e/eyd+/emqhFquDavhlefIpeO3/0dikVWCwGFyS5R6N+t1XTiYiIiNSlagW4srIyHnvsMSIjI2ndujWtW7cmMjKSRx99FIfDUdM1NmhGo8YAhJQeA9Pl5Woq8kwnsk2rb4iIiNSlao1Cvffee5kzZw7PP/88ycnJAKSlpfHkk09y5MgRXnvttRotskGLigaLBZvLBfn50DjQ2xV5lAe49ftyyC1yEBkS4OWKREREGoZqBbhZs2Yxe/ZsLr/8cs+2bt26ER8fzw033KAAV4MMq9Ud4rIPQ/YhaBzj7ZI8mkUGk9Q0jO1ZBSzbcZjLuzbzdkkiIiINQrUuoQYGBtK6detK2xMTE7Hb7edak/xS9PHQll3/LlXqMqqIiEjdq1aAGz9+PE8//TQlJSWebSUlJTz77LOMHz++xoqT4+pxgLv4+Lqo3207pGW1RERE6ki1LqGuW7eOhQsX0rJlS7p37w7ADz/8QGlpKYMGDeKqq67ytJ0zZ07NVNqQ1eMA1++8aAKsBvuOHmP3kSJax4R6uyQRERG/V60AFxUVxdVXX11hW3x8fI0UJFWoxwEuxG6jd6tGLP8pm++2HVKAExERqQNnHeBM0+Spp56iSZMmBAcH10ZN8kvR7suUHM3GdDrdAxvqkYvaNjke4A5zs5bVEhERqXVnfQ+caZokJSWxb9++2qhHqhIeTpnF4p4HLifb29VUUj6QIW2He1ktERERqV1nHeAsFgtt27blyJEjtVGPVMWwUGR3n+00j9S/VQ86N4+kUUgA+SVl/LA3x9vliIiI+L1qjUJ97rnnePDBB9mwYUNN1yOnUBhUfwOc9aRltTSdiIiISO2rVoAbM2YMK1eupHv37gQHBxMdHV3hITWvMPD4/Yb1MMDBicuo32+rn/WJiIj4k2qNQn355ZdruAw5ncLAEADM7PoZkC48Ph9c+t4cco85iAzWsloiIiK1pVoBbuzYsTVdh5xG+Rm4+ngJFaBFVDBtmoSy41AhaTsOM6yLltUSERGpLdW6hAqwY8cOHn30UW644QaysrIA+PLLL9m4cWONFScneC6h5uVilpb8emMvucizKoPugxMREalN1QpwS5YsoWvXrqxYsYI5c+ZQUFAAuFdjeOKJJ2q0QHFz2AIg2H0Zlez6OQK4/D44BTgREZHaVa0A98gjj/DMM8+QmppaYfH6yy67jOXLl9dYcfILx1dkqK+XUfuf15gAq8Ge7CJ2Hyn0djkiIiJ+q1oBLiMjgyuvvLLS9qZNm3L4sM6+1Jp6HuBCA230TGgE6CyciIhIbapWgIuKiuLAgQOVtq9bt44WLVqcc1FyCuUBrp6ORAW42HMZtf7WKCIi4uuqFeCuv/56Hn74YTIzMzEMA5fLxdKlS/njH//ImDFjarpGKVe+qH09PQMHJwYyLNt+hDItqyUiIlIrqhXg/vrXv9KxY0cSEhIoKCigU6dOXHzxxQwYMIBHH320pmuUcvX8EipAlxaRRAYfX1ZrX663yxEREfFLZzUPnMvl4u9//zuffvoppaWl3HzzzVx99dUUFBTQs2dP2rZtW1t1CkCj42fgjhVhFhVihIR6t54qWC0GFybFMC/jAN9tO0TvVo28XZKIiIjfOaszcM8++yx/+tOfCAsLo0WLFsyaNYuPPvqIa6+9VuGtLtjtEBEJ1O+zcJpOREREpHadVYB79913+ec//8n8+fOZO3cun332Ge+99x4ul+51qitGY/c9ZmTX33B04fEAl743h7xih5erERER8T9nFeD27NnD8OHDPe8HDx6MYRjs37+/xguTqhnR7gBXn8/AtWwUwnlNQnG6TL7ZnOXtckRERPzOWQW4srIygoKCKmwLCAjA4dBZljrTuP4HOIARXd1roX72Q+XpZkREROTcnNUgBtM0ueWWWwgMDPRsKy4u5u677yY09MQN9XPmzKm5CqUCw0cC3BXdm/OPRdtZsjWL3CIHkSEB3i5JRETEb5xVgBs7dmylbTfddFONFSOnd/I9cKbpwjCqNRNMrWsXG0772HC2HMxn/o+ZXNsn3tsliYiI+I2zCnAzZsyorTrkTEVFg8UCjlLIz4OIKG9XdEq/6daMLan5fL7+gAKciIhIDaqfp2/klAyr1R3iqP+XUX/TvTkAS7cf5khBiZerERER8R8KcD7IV+6DS4wJpWuLSJwuky83ZHq7HBEREb+hAOeDPPfB1fMAB3BF9/LRqJpqRkREpKYowPkiHzkDBzCim/sy6spd2WTmFnu5GhEREf+gAOeDfOUSKkCLqGD6tGqEacK8DM0JJyIiUhMU4HyQ5xLq0WxMp9O7xZyB33RzX0b9fL0uo4qIiNQEBThfFB4BAXYwXZCT7e1qTmt4t2ZYDFi3J4e92UXeLkdERMTnKcD5IMOwQLR7wXhfuIzaNDyI/uc1BuDz9bqMKiIicq4U4HyUL90HB+6ltUCjUUVERGqCApyP8qWpRACGdY7DZjH48UAe27MKvF2OiIiIT1OA81GeM3DZvhHgGoXaubCt+7KvBjOIiIicGwU4X+Vjl1ABruh24jKqaZperkZERMR3KcD5KOP4IAbycjFLfWOd0SGdY7HbLOw4VMjmzHxvlyMiIuKzFOB8lBESCsEh7jfZR7xbzBmKCArg0vbuM4cazCAiIlJ9CnA+zNdGosJJo1HX6zKqiIhIdSnA+TIfDHCXdWhKiN3K3uxj/LAv19vliIiI+CQFOB9mRPvWSFSAELuNQR1jAfhcl1FFRESqRQHOh/naXHDlrvCsjXoAl0uXUUVERM6WApwP88V74AAuad+E8CAbmXnFrN591NvliIiI+BwFOF8W7V5flGNFmEWF3q3lLATarKR0jgM0GlVERKQ6FOB8mGEPhIhIAMzsw16u5uyUj0b9IuMAZU6Xl6sRERHxLQpwPs5X74Mb0KYx0aF2jhSW8t023wqfIiIi3qYA5+M8I1F9LMAFWC2M7OE+C/du2i7vFiMiIuJjFOB8nY8OZAAYk9wagMVbD7HrsO/cwyciIuJtCnA+zldHogIkxoQysH0TTBPeTdvt7XJERER8hgKcj/PcA5d9GNP0vcEAYwe0BuDD1XspLCnzbjEiIiI+QgHO10VFg2EBRynk53m7mrN2SdsmJMaEkl9Sxpx1P3u7HBEREZ+gAOfjDKsVGkUDvnkZ1WIxGJPcCoB3l+3SAvciIiJnQAHOD/jyfXAA/9e7JaF2K9uyCli244i3yxEREan3FOD8gK/OBVcuPCiAq3u3BGDmsl3eLUZERMQHKMD5g/IzcD62GsPJyqcUWbjpIHuzi7xbjIiISD2nAOcHPJdQD2d5uZLqS2oaxkVtY3CZ8J/lmlJERETk1yjA+QGjqXtheLKPYBbke7eYczD2+Fm42av2cqzU6d1iRERE6jEFOD9ghIZjNI8HTMxtP3q7nGq7tENT4qODyT3m4JN0TSkiIiJyKgpwfsJo1wkA15aNXq6k+qwWgzH9WwPuwQyaUkRERKRqCnB+wnI8wJk/bcMsc3i5muq7tk88wQFWNmfms3JntrfLERERqZcU4PxFXAuIiARHKebO7d6uptoiQwIY1bMFAO+k7fJuMSIiIvWUApyfMAwDS7vOAJg+fBkVYOwA98oM8zceZH/OMS9XIyIiUv94NcC99tprdOvWjYiICCIiIkhOTubLL7/07M/MzOTmm28mLi6O0NBQevXqxf/+978Kx8jOzmb06NFEREQQFRXF7bffTkFBQYU269ev56KLLiIoKIj4+Hief/75SrV8+OGHdOjQgaCgILp27coXX3xROz90LfLcB7f1R5++f6xDXAT9z4vG6TJ5b4WmFBEREfklrwa4li1b8txzz7FmzRpWr17NZZddxsiRI9m40X0GacyYMWzZsoVPP/2UjIwMrrrqKq699lrWrVvnOcbo0aPZuHEjqampfP7553z77bfceeednv15eXkMHTqUVq1asWbNGv7+97/z5JNP8sYbb3jaLFu2jBtuuIHbb7+ddevWMWrUKEaNGsWGDRvqrjNqgJGYBAF2yM+FTN8exXnLgNYA/HflXoodmlJERETkZDZvfvkVV1xR4f2zzz7La6+9xvLly+ncuTPLli3jtddeo2/fvgA8+uijvPTSS6xZs4aePXuyadMmvvrqK1atWkWfPn0A+Mc//sHw4cN54YUXaN68Oe+99x6lpaW8/fbb2O12OnfuTHp6Oi+++KIn6E2dOpVhw4bx4IMPAvD000+TmprKq6++yvTp06usvaSkhJKSEs/7vLw8ABwOBw5HzQ4iKD/eGR03sS1s3UjZpgyIia3ROurSJUnRNIsM4kBuMXPX7uXqXi3q5HvPqq+l2tTPdUd9XXfU13WjIfRzQEDAadt4NcCdzOl08uGHH1JYWEhycjIAAwYM4P3332fEiBFERUXxwQcfUFxczMCBAwFIS0sjKirKE94ABg8ejMViYcWKFVx55ZWkpaVx8cUXY7fbPW1SUlL429/+xtGjR2nUqBFpaWlMnDixQj0pKSnMnTv3lPVOnjyZp556qtL2BQsWEBIScg49cWqpqamnbdPymINuQO7q5SzNL62VOupK70iDz3OtvLpgA0EHfsAw6u67z6Sv5dypn+uO+rruqK/rhj/388iRI0/bxusBLiMjg+TkZIqLiwkLC+Pjjz+mUyf3vVwffPAB1113HY0bN8ZmsxESEsLHH39MUlIS4L5HrmnTphWOZ7PZiI6OJjMz09MmMTGxQpvY2FjPvkaNGpGZmenZdnKb8mNUZdKkSRVCX15eHvHx8QwdOpSIiIhq9kbVHA4HqampDBky5PSpvLAApj5L5LEChl90AYRH1mgtdal/YSmpL3zLvkIXsV2S6dOqUa1/51n1tVSb+rnuqK/rjvq6bqif3bwe4Nq3b096ejq5ubl89NFHjB07liVLltCpUycee+wxcnJy+Prrr4mJiWHu3Llce+21fPfdd3Tt2tWrdQcGBhIYGFhpe0BAQK39Qp3RsaMaUdYyAXPfbiw/bcPaJ7lWaqkLsVEBXNWrBf9duZd/fPMTs+7oX2ffXZt/jnKC+rnuqK/rjvq6bjT0fvb6NCJ2u52kpCR69+7N5MmT6d69O1OnTmXHjh28+uqrvP322wwaNIju3bvzxBNP0KdPH6ZNmwZAXFwcWVkVF3AvKysjOzubuLg4T5uDBw9WaFP+/nRtyvf7mvLRqOZW355OBGD8ZW2xWy0s23GEpdsPe7scERGResHrAe6XXC4XJSUlFBUVAWCxVCzRarXicrkASE5OJicnhzVr1nj2L1q0CJfLRb9+/Txtvv322wo3O6amptK+fXsaNWrkabNw4cIK35Oamuq5F8/XWNofnw9u5zZMh2/fB9ciKpgb+yUA8Pz8LT49PYqIiEhN8WqAmzRpEt9++y27du0iIyODSZMmsXjxYkaPHk2HDh1ISkrirrvuYuXKlezYsYMpU6aQmprKqFGjAOjYsSPDhg3jjjvuYOXKlSxdupTx48dz/fXX07x5cwBuvPFG7HY7t99+Oxs3buT9999n6tSpFe5fu++++/jqq6+YMmUKmzdv5sknn2T16tWMHz/eG91y7prEQVQ0lJVh/rTV29Wcs3GXJhEcYOWHvTmk/njw9B8QERHxc14NcFlZWYwZM4b27dszaNAgVq1axfz58z03Jn7xxRc0adKEK664gm7duvHuu+/yzjvvMHz4cM8x3nvvPTp06MCgQYMYPnw4F154YYU53iIjI1mwYAE7d+6kd+/ePPDAAzz++OMV5oobMGAAs2bN4o033qB79+589NFHzJ07ly5dutRpf9QU96oMvr+4fbkm4YHcdmFrAKYs2IrLpbNwIiLSsHl1EMNbb731q/vbtm1baeWFX4qOjmbWrFm/2qZbt2589913v9rmmmuu4ZprrvnVNr7EaNcJVn6PuXUTpunCMOrd1fKzcudFbfh32m62HMzns/X7GdmjbuaFExERqY98+191OSWjdRuwB0JhPub+fd4u55xFhgRw1yVtAHgxdSsOp8vLFYmIiHiPApyfMqw2jKQOgO8vbl/u1gtaExNmZ/eRIj5c7fuhVEREpLoU4PyY5z44P5hOBCDEbmPcpe5JnF9ZuE1rpIqISIOlAOfHjLYdwTDg4AHMnGxvl1MjbuyXQPPIIDLzivnP8t3eLkdERMQrFOD8mBESihHvXkbMtfVHL1dTMwJtViYMbgfAtG+2k1/sv4sZi4iInIoCnJ87sSqDfwQ4gKt6teC8mFCOFjl4+/td3i5HRESkzinA+TlL++MBbtd2zJJiL1dTM2xWCxOHus/C/eu7nzha6NurTYiIiJwtBTh/17gpRMeA04m5w/dXZSg3vEszOjWLIL+kjOnf7vB2OSIiInVKAc7PuVdlcK+N6i+jUQEsFoMHU9oD8M6yXRzM84+ziyIiImdCAa4BMMovo27bhOnynwlwB7ZvQu9WjSh2uHh10XZvlyMiIlJnFOAaACM+EYKCoagQ82f/mXrDME6chfvvyj3sOVLk5YpERETqhgJcA2BYrX63KkO5/uc15qK2MZS5TJ78bCOmqYXuRUTE/ynANRCW9sfvg9vyo9+FnCeu6ITdamHR5iw+X3/A2+WIiIjUOgW4BsJI6gBWGxw+iLn7J2+XU6OSmobz+0vdC90/9dlGcoo0rYiIiPg3BbgGwggKxtKzLwCu71K9XE3Nu2dgG5KahnG4oJS/frHJ2+WIiIjUKgW4BsRywaVgsWD+tA3XPv8ZzADuJbaeu6orAB+s3seyHYe9XJGIiEjtUYBrQIyoaIxufQBwffe1l6upeX1aR3NT/wQA/jQng2KH08sViYiI1A4FuAbGeuFlYBiYW3/EzPzZ2+XUuIeGdSA2IpBdR4p4ZeE2b5cjIiJSKxTgGhijcROMzj0AcH630LvF1IKIoAD+MrILAG98+xObDuR5uSIREZGapwDXAFkvGgSA+eN6zEMHvVxNzUvpHEdK51jKXCaPzMnA6fKvaVNEREQU4Bogo2kzjA5dARPn9/53Fg7gLyO7EB5o44e9Obybtsvb5YiIiNQoBbgGynMWLmMd5tEjXq6m5sVGBPHw5e7VJ/4+fws/5xzzckUiIiI1RwGugTKax7sn9zVduL5f5O1yasWNfRM4v3UjikqdPDZ3g9+tQCEiIg2XAlwDZrloMACu9FWYeTneLaYWWCwGk6/qqmW2RETE7yjANWCWhESM1m3A5cS1dLG3y6kVWmZLRET8kQJcA+c5C7c2DbMg38vV1I6Tl9l65H8ZupQqIiI+TwGugTMS22K0SICyMlxpS7xdTq0ItFmZck13AqwGX23M5K3vd3q7JBERkXOiANfAGYaB5eIhALhWL8M8VuTlimpH9/goHvtNJwAmf7mZVbuyvVyRiIhI9SnACUbbjhDbHEpLcK34ztvl1Jqb+7fit92b43SZjHtvLYfyS7xdkoiISLUowAmGYWC9+Pi9cCu+wywp9nJFtcMw3KNSk5qGkZVfwh/+u44yp8vbZYmIiJw1BTgBwOjYFWJiofgYrlVLvV1OrQkNtDH9pl6E2K2k/XSEF1O3erskERGRs6YAJwAYhgXrhZcB4Er71m/vhQP31CJ/u7obAP9cvIOvf/S/9WBFRMS/KcCJh9G1J8Q0haICnJ996NfTbVzRvTm3DGgNwMQP0tlzxH8Dq4iI+B8FOPEwLFZsV94IFivmpvWY6Su9XVKt+tPwjvRMiCKvuIzfz1pDscPp7ZJERETOiAKcVGA0j8dy2TAAnF/OxTxyyMsV1R67zcK0G3sRHWpnw895PPXZRm+XJCIickYU4KQSy4CBGK2TwFGKc857mE7/PTPVPCqYqdf3wDDgvyv3Mmfdz94uSURE5LQU4KQSw7BgvfIGCArG3L8X1+L53i6pVl3UtgkTBrUD4PFPN7Gv0MsFiYiInIYCnFTJiIjCesW1ALi+X4Rr1w4vV1S77r0siYvbNaGkzMXrm6zsydagBhERqb8U4OSULJ26YfToC5g4P57l11OLWCwG/7i+J+1jw8hzGNz6zhqy8v1zQmMREfF9CnDyq6yXj4LoGMjLwTnvI7+eWiQyJIC3x/amcaDJnuxjjHlrJbnHHN4uS0REpBIFOPlVhj0Q61WjwWLB3PgD5g+rvV1SrWoaHsjvOzlpEmZnc2Y+t89cxbFS/x3EISIivkkBTk7L0iIBy8DyqUU+xsw+7OWKaldMELw9tjcRQTZW7z7KPe+tobRMa6aKiEj9oQAnZ8RywaUYrc6D0hK/n1oEoENcOG/fcj5BARYWbznEHz/8AZfLfy8fi4iIb1GAkzNiWCxYr7zRPbXIz3twLVng7ZJqXZ/W0Uy/qTc2i8GnP+zniU83+vU9gCIi4jsU4OSMGZGNsP7m/wBwfbcQl5/fDwcwsH1TXrzOPdHvv5fv5qXUrd4uSURERAFOzo6lcw8s518AmDjnzsaVvsrbJdW633Zvzl9GdgHglUXbefv7nV6uSEREGjoFODlrlstHYemdDJg4P3kf1zr/XvQe4Ob+rXhgiHu1hr98/iOzV+7xckUiItKQKcDJWTMMC5YRV2PpMwAwcX76Aa51K7xdVq0bf1kSt1+YCMAjczKY9s123RMnIiJeoQAn1WIYBpbhV524nPrpB7jW+neIMwyDR0d05J6BbQD4+/wtPPnpRpwanSoiInVMAU6qzTAMLJdfiaXvhQA4P/sA15rlXq6qdhmGwcPDOvDEFZ0wDHgnbTd/+O86Ssr8e1oVERGpXxTg5JwYhoFl2Cgs/S4CwPn5h7jWpHm5qtp36wWJvHJ9TwKsBvMyDjD27ZXkFWvZLRERqRsKcHLODMPAkjISS7+LAXB+/hHO1f4f4q7o3pyZt/YlLNDG8p+yue715WTlFXu7LBERaQAU4KRGuEPcb7H0d4c417yPcK5a5uWqat8FSTHMvrM/MWGBbDqQx1WvLeOnQwXeLktERPycApzUGMMwsAz9LZbkSwBwffE/nJ9/hOko9XJltatLi0jm3DOA1o1D2Hf0GP83PY30vTneLktERPyYApzUKMMwsAy5AsvFQwBwrUmj7M2XMQ/u93JltSuhcQgf3TOAri0iyS4s5YY3lvP1jwe9XZaIiPgpBTipcYZhYL10GNab7oSwcDh0kLI3p+Jc8Z1fz5sWExbI7Dv7c1HbGI45nPzu3dVM/nITDqfL26WJiIifUYCTWmNp0x7b3X/EaNsRnGW4vpqL879vYRb67z1ioYE23hp7PmOTWwHw+pKfuPHN5WTmanCDiIjUHAU4qVVGaBjWG27HMmwUWG2Y2zZRNv0FXDu2eLu0WmO3WXhqZBem3diLsEAbq3YdZfgr37Fk6yFvlyYiIn5CAU5qnWEYWPtdhO2O+6BJLBTk4/zPGzgXfIbpLPN2ebVmRLdmfH7vhXRqFkF2YSm3zFjJlAVbtHKDiIicMwU4qTNGbHNsd9x/fA1VcKUtpuytf+Da85OXK6s9rWNCmfP7AYzul4Bpwj8WbWf0vzRfnIiInBsFOKlTRkAA1hFXY73uVggOgQP7cM6YRtl/38I8eMDb5dWKoAArz17ZlanX9yDUbmX5T9kMf+V7lm4/7O3SRET8npmTjeOpByib+c86/27Xru3u75773xo/tgKceIWlQxds9zyIpVd/MCyYW3+kbPoUyj6ehXn0iLfLqxUje7Tg03svpENcOIcLSrjprRU8/9Vmih1aR1VERM6OApx4jREegfWKa7CNewijc3fAxFy/hrJX/4bzy7mYhfneLrHGtWkSxse/v4Dr+sRjmvDPxTu4fOp3pO3wz9AqIiK1w+btAkSMxk2w/d8YXAP24lr4BeZPW3Gt/A7XuhVYki/BMmAgRmCQt8usMcF2K3/7v25c2qEJj3+ykZ2HC7nhzeVc26clfxrekagQu7dLFBHxC87F83EtWQCAuXsHjqce8OwzuvfBNuoG975jRbiWfoNry0bIOQJWG0azlu5/g9p1qnRcM+sAzu8XYe7dBfl5EBgI4ZFYWrfBcsFlGOERlM39L+YPq93tf1iN4/hrAMslQ7EOTDmnn00BTuoNS/N4LDffheunre4gt38vrm9Tca1aiqVnXyy9+mM0buLtMmvMsC7NGJAUw/NfbeY/y/fwwep9LNyUxeNXdOK33ZtjGIa3SxQR8WlGXHOMjt0wN62H0HCMpPaefZaERADMI4coe3c65OVAVDRGm/ZQWoK5b4977tIhv8E64FLP58z9eymb8SqUlUFsM4wWncHhwDx6BNeK7zA6dMEIj8CSkIirIB9zxxZo1Bjj+PeV13WuFOCk3rGc1w4jsS3mpgyci76AI4dwLVuMa9lijNZJWHr3x+jQFcPm+7++EUEBPDOqK6N6tGDSnAy2ZRVw3+x05qz9mWdGdSE+OsTbJYqI+CxLh64YcS0o27QeI6ap54xbOdPlouyDdyAvB8vg32AZcAmG4b67zMw+TNm/X8f19RdYkjpgNG0GgHPl91BWhmXIFVgHDKx4vMMHITDY/d29+kN0DM4dWzASEit99zn/bDV6NJEaYhgGlk7dsP3+QazX3+pezQEDc9d2nP/7D2Uv/cU9j9wR/5gct0/raOb94SIeGNIOu9XCkq2HGPrSt7z57U+UaSkuEZFaYW7dCFkHMDp2w3rBpZ7wBmBEx2Ad+lswXbjWrDjxoeOrCVnOa1fpeEZMLEZ4RK3XDQpwUs8ZFiuW9l2w3fg7bBP+jOXiIRAeCUWF7nnkXn2Osnf+iStjLWaJb8+tZrdZuHdQW76ccBH9EqM55nDy7BebGPHK93z940G/XkdWRMQbzB1bAbB07FrlfqPV8cus+/ec2NasJQDOL/6Ha9d2TJd3ZhLw/WtQ0mAYkY2wXjoMyyVDMLdtxrV2Oea2TZi7duDctQMsVozENhjtOmNp1wkjKtrbJVdLmyZhzL6zPx+u3sezX2xiy8F8fvfuavq0asRDwzrQN9E3fy4RkfrGzMkGwDnnPZxz3jt1u6JCz2vLBZdi7t3p/rfnndfAHojRshVG245YepyPERRc63WDApz4IMNixWjfGUv7zpi5R3GtW4krYy1kH8bcsRVzx1ZcX34MTZu5g1z7zhgt4iucGq/vDMPg2vPjSekcx/RvdzBj6U5W7z7Kta+ncWn7Jjw0rAMdm9XNaXoREb91/MqGkdQBQsNO2cwICT3xOjAI65h73CFuy4+Yu3dg7tzunkHh+0XYbh1XJwPuFODEpxmRjbAOTME6MAXzcBaurRvd/0Ht3QlZB3BlHYDvF7pHH7XtgKV1EkZ8a/eIIB8Y5RkZEsDDwzpwy4DWvLJwG7NX7eWbLYdYvPUQI7s3Z+KQ9iQ01kAHEZFqiYgEwNKzH5ZO3c74Y4ZhYCScBwnnAWAW5uP86hPMDetwLvoS2zVjaqXckynAid8wYppijWkKAy7FLCrE3L7ZHei2bYbCfMz0VTjTV7kbh4ZjxLd2PxISMZq1wLDW3/8cYiOCePbKrvzuovOYsmALn68/wNz0/czLOMANfRMYf1kSTcP9Z648EZEaY7W6n6u4V81yXjuc61bi2pxxVgHul4zQcKwDUyjbsA4zK7OK7675wWj1918skXNghIRidOuNpVtvTGcZ5u6fMLdvcZ/yPrDPHeg2Z2BuznB/wGrDaBEPLRJodjQLDh3EjG2GUf4fXz2RGBPKqzf24u5Lcnl+/ha+3XqId9N2M3vlXkb1bM7vLjqPdrHh3i5TRKT+CAkFixXz6BFMlwvDctJI007d4NtYzIy1OBs3xXLBpRWmqDJN0z1ZLyfmjXOuXoalTXuMRo0rfI1r2yb3MSMjTxw//PjrWpgxQQFO/J5htWGc1w6OD/k2yxyY+/e5w9zeXe7/OIsKMffshD076Qnw5mbKrFaIaYrRtJn7EdsMo2kcRER5/fJrlxaRvHtbX5btOMyUBVtZs/soH6zexwer93FJuyb87qJELkyK8XqdIiLeZlhtGEntPWtuG81agNWKJT4RS8++2K67lbL/vIFr8Ve4Vn2PEdvMfT9cUSFm5n4oLMCSMhKOBzjX6jRc8/4HTWIxYmLBYsE8nAUH94PNhuXioSe+OyoaYpu5J/9982VoGgeGgaV9Zyztu5zTz6UAJw2OYQtwz4hdPgu3aboHQOzdiXPPLo5u2USjsmIoLYWDBzAPHqDCBB5BwRjRMRDdGKNRDEajaGgUgxHdGMIj6nSwxIA2MQy4J4Y1u7P513c7mb8xkyVbD7Fk6yE6xIVz+4WJ/LZHcwJt9etMoohIXbL+9lr33KE/bcXMWOee283lwtKzr3s5x7sm4lq5FNfmDMx9e9yXW8MiMOJauAfNde5+4liXDsO1eQPmz3swd24DpxMiIjF69sM6YCBGTNMK32279hacqZ9h7v4JDuwD08SMiAIFOJFzYxgGNG6C0bgJzs49STMDGX75MGyFBZhZBzAPZrqfsw7A4UNQfAxz/17Yv5dKM7PZbBDVGCO6MUZEFIRHYkREHH+OdM9hFxhU42fGereKpneraPYcKeLtpTv5YPVeNmfm8+BH63l+/hbGJrdidL9WNArVOqsi0vAYoeHYrrzx1PuDgrFePBjrxYNPeyzL8VkQzvi7o2OwXXfrGbc/UwpwIlUxLBiNGrvvcTjp/5LMsjI4koWZfRjz6BHIPuK+r+LoEcg56l4b7/BBzMMHK4e7cgF295m6iEj3YIqQUPc9GqFhGCFhEBrqfg4JhZAQDMuZnz1LaBzCk7/tzP2D2zFr5R5mLtvJwbwSXliwlVcWbmdIp1j+r3dLLmobg83qO9OqiIhIRQpwImfBsNkgtjlGbOWFiE2XE3Jzjoe7bMjLwczPhfxczPw8yMuF4mPgKHVfss0+7P7cr38jBAZCcAgEBbkniDz+qPA6MBDsQWC3Q2AgEfYg7u4exe29+vHl1iO8uWwPG/bnMy/jAPMyDtA0PJAre7Xgmt4tSWqqQQ8iIr7GqwHutdde47XXXmPXrl0AdO7cmccff5zLL7/c0yYtLY0///nPrFixAqvVSo8ePZg/fz7Bwe6ZjrOzs7n33nv57LPPsFgsXH311UydOpWwsBMT8q1fv55x48axatUqmjRpwr333stDDz1UoZYPP/yQxx57jF27dtG2bVv+9re/MXz48NrvBPEbhsXqnl/uFyOTTmY6SiE/DzPveLArLICiQigscM/0XVRwYtuxY4AJJcXuBxXD3pksrGUAw4HhFgvOyACKXBZyyqCo1ELxcisH0iwcCQqkSXQYzWIiCAgMdIdUq819Ofik14bNBrYA97B4q9W9vfy1xf1sWG1gtbj3WSzHH9YTr63HX5ta31WkLp1Yis+s8FThb5Jfbqvwl4xZxf5f/C10quX+qtr+q589xfeXPzkc2B2l4HBAQEDV39kAeDXAtWzZkueee462bdtimibvvPMOI0eOZN26dXTu3Jm0tDSGDRvGpEmT+Mc//oHNZuOHH37ActIQ4NGjR3PgwAFSU1NxOBzceuut3HnnncyaNQuAvLw8hg4dyuDBg5k+fToZGRncdtttREVFceeddwKwbNkybrjhBiZPnsxvfvMbZs2axahRo1i7di1dupzbTYYiJzMC7BAd4x4EcRqmywnHiuDYMcziY+6zd8UVX1NcjFlcBCUlUFqCefyZ0hL3tjKH+2AuF9bSEsKBcKi4CrIDOHgQDrr/bqyrFVcvB1j/PQ6LBQzLiZBnWMBinNhmGO7Hya+NE689UwJ49h1/cPJ7jr+n4vbyz538XOXr8jaceP2Ll6feXoUz6uSq/mE9g+2//IfR5aL3wYPwQRZlFsuv/8N5un/kzbN9XdWxf/21WWF73X1v5de/CCynDDcn7TNNBpWWwpbVOM7mO3+thl9+l+fp1/ZVeOGXBgO0bgl9kr1ditcYZj1bITs6Opq///3v3H777fTv358hQ4bw9NNPV9l206ZNdOrUiVWrVtGnTx8AvvrqK4YPH86+ffto3rw5r732Gn/+85/JzMzEbnffwP3II48wd+5cNm/eDMB1111HYWEhn3/+uefY/fv3p0ePHkyfPv2M6s7LyyMyMpLc3FwiImp2iSOHw8EXX3zB8OHDCWjA/7dRF/ytr02X60SgczjAUeo+C+hwgMNBXn4R63ZksX7XIfLziwjEhR0XwRaTVpF2EqMCaRFhJxAXOMvcgbDM6R6h5XRiOt3POMvcz67y9073xJUuZ61MYCkiwoirCegzwNtVeE29uQfO6XTy4YcfUlhYSHJyMllZWaxYsYLRo0czYMAAduzYQYcOHXj22We58MILAffl1aioKE94Axg8eDAWi4UVK1Zw5ZVXkpaWxsUXX+wJbwApKSn87W9/4+jRozRq1Ii0tDQmTpxYoZ6UlBTmzp17ynpLSkooKSnxvM/LywPcAcDhcJzqY9VSfryaPq5U5pd9bbVBsA2qWF85FLiwO1xgmmzcn89XGw/yxY8H2XmkCA4Bh9wnw3olRDG0UyxDOjalZaOzXKjZNN2P8jDncuEoKeHbxd9w8UUXEWC1era727ncl1g9r82THi5wmSfOzpy8n5PbUXlbeS0n1+Q5e/IrZ1uqfH38fRUvz+jMh2lWPMvnYfzK27M583fijGGZ08mPP/5Ip06dsVmtVbapcICT95/qrGSlz1bx/qxf/9oZ0NPVUNVxz+TMKqffd9rXJ7Y5nGWkpaWRnDyg8v8Anuvxq+zrX/mzr+qYZ/X+FL8fp6ylBtqcclvFGh0OB6mpqQzp1MP9P6N+6ExOIHg9wGVkZJCcnExxcTFhYWF8/PHHdOrUieXLlwPw5JNP8sILL9CjRw/effddBg0axIYNG2jbti2ZmZk0bfqL+VZsNqKjo8nMdC9lkZmZSWJiYoU2sbGxnn2NGjUiMzPTs+3kNuXHqMrkyZN56qmnKm1fsGABISG1szZlampqrRxXKmuofd0J6NgWDraE9dkG67Mt7C00WL07h9W7c/jrl1toEWLSPtKkXZRJm3ATe3WnmAsIJHX5yposX06lcTP2Hcz2dhUNQ1Aoqet+8HYVDYI//z09cuTI07bxeoBr37496enp5Obm8tFHHzF27FiWLFmC6/hll7vuuotbb3XPn9KzZ08WLlzI22+/zeTJk71ZNpMmTapw1i4vL4/4+HiGDh1aK5dQU1NTGTJkiF9c1qvP1NeV/ZxzjNRNWaT+mMXq3Uf5ucjg5yKDRQcgwGrQOyGKC9o0ZkCbxnRuHoHVcvo57tTPdUd9XXfU13VD/ezm9QBnt9tJSkoCoHfv3qxatYqpU6fyyCOPANCpU6cK7Tt27MiePXsAiIuLIysrq8L+srIysrOziYuL87Q5ePBghTbl70/Xpnx/VQIDAwkMDKy0PSAgoNZ+oWrz2FKR+vqE1k0CuKNJBHdcnER2YSnfbTvE0u2H+X7bYfbnFrN851GW7zzKlK+3ExkcwIA2jRmQFEPvhEa0jwv/1UCnfq476uu6o76uGw29n70e4H7J5XJRUlJC69atad68OVu2bKmwf+vWrZ5pRpKTk8nJyWHNmjX07t0bgEWLFuFyuejXr5+nzZ///GccDofnDzo1NZX27dvTqFEjT5uFCxcyYcIEz/ekpqaSnNxwR7eIVCU61M7IHi0Y2aMFpmmy83Ah3x8Pc2k7jpB7zMGXGzL5coP79oOwQBs94qPolRBFr1aN6JnQiMjghvsXrohITfFqgJs0aRKXX345CQkJ5OfnM2vWLBYvXsz8+fMxDIMHH3yQJ554gu7du9OjRw/eeecdNm/ezEcffQS4z8YNGzaMO+64g+nTp+NwOBg/fjzXX389zZu7J1q98cYbeeqpp7j99tt5+OGH2bBhA1OnTuWll17y1HHfffdxySWXMGXKFEaMGMHs2bNZvXo1b7zxhlf6RcQXGIbBeU3COK9JGGOSW1PmdLH+51y+33aYVbuyWbcnh4KSMnfA237Y87m2TcPoGR+J5ahBq/15dGwRpbVaRUTOklcDXFZWFmPGjOHAgQNERkbSrVs35s+fz5AhQwCYMGECxcXF3H///WRnZ9O9e3dSU1Np06aN5xjvvfce48ePZ9CgQZ6JfF955RXP/sjISBYsWMC4cePo3bs3MTExPP7445454AAGDBjArFmzePTRR/nTn/5E27ZtmTt3ruaAEzkLNquFXgmN6JXgPrPtdJlsPZjPmt1HWbv7KGv3HGXXkSK2ZRWwLasAsDL7teUEWA2SmobTuXkEnZpFuJ+bRxAepDN1IiKnUu/mgfNVmgfOP6iva9fhghLW7j7Kqp1H+Gb9T2SV2skrLquybUJ0CJ2aRdA2NoykpmG0aeJ+BFd72GvDpN/puqO+rhvqZ7d6dw+ciPivmLBAhnaO49J2jeni3M7ll19KVmEZG/fn8eP+PDbuz2PTgTx+zjnGnuwi9mQX8dXGisdoERVMUtOwCo9W0SE0CQ/EqHJuNRHxJS+lbmXqwm38/f+6cU2feG+XU28pwImI1xiGQctGIbRsFEJK5xOjvo8WlrLpQB6bMvPZnlXAjqwCth8qILuwlJ9zjvFzzjGWbD1U4VhBARYSokOOP0JJiA6mVeNQ4qNDiI8O1n12ItWwN7uIi57/hn6J0bx/lwb21ScKcCJS7zQKtTMgKYYBSRXXjM0uLGV7VsGJx6ECfjpUwP6cYxQ7XGw9WMDWgwVVHjMmLJDmUUE0iwyiWWTw8dfu5+ZRwTQNDzqjOexEpHaNHdCaK7o3p2lE5am65AQFOBHxGdGhdvomRtM3MbrC9tIyF/tzjrE7u4g9RwrZk13E7iNFnsuwRaVODheUcLighPX7cqs8ttVi0DjUTpPwQJqGBx5/Dqr0PjrMTqjdqsu1IrUkOtROdKj99A0bOAU4EfF5dpuF1jGhtI4JBZpU2GeaJtmFpRzILWZ/zjHP8/7cYg4cf5+ZV4zTZZKVX0JWfgkbq/6aCt8XHWL3/ENz8qNRqJ1GIQFEBrsfUcF2IoMDCA+yYdEZPvEh5feiAazYmU3rR+Z59l3dqyVTru1O60fm0SIqmG/+OJDXFu/gkx9+Zl/2MS5p34Q3x/Sh2OHkk/SfSf0xiy0H88jKK8Fus9AxLoKbklvx2+7NT/m9v7wH7rrX01ixM5tvJl7E+myDt19fwdaDBdhtFi5u14Q/De9As8izXKvZhynAiYhfMwyDxmGBNA4LpEuLyCrbOF0mhwtKOJRfQlZ+sfs5r4RDBSeey/cVO1yUlrnIzHMHvzOvAyKCTgS7iGAbYYE2wgLd4a78Uf4+LMi9P8RuPf5sIzTQSnCAzv5J3ejUPILLu8Tx5YZMYsICuaTdif85Or91I89r0zS589+rWbkzm36J0XSMiyAqxD06dN/RYzz8vwxiIwI5LyaM7i2jOJRfwto9R1m5K5sdWQXcP6TdWdX13sq9zNhi4fxEC5d2aEL6nhw++2E/G37O5cv7LiIooGHc76oAJyINntViEBsRRGxEEFB1yCtXVFpGdmHpKR9HCkvJPeYg75iD3GMOcoocHHM4MU3IPb7tXBgGhARYCQm0EWq3Emx3h7zgACtBAVaC7VZCjj8HBVgJtMKu/QZHlu8hNCiAQJuVoAALgTYrgcef3e8t2K1W7DYLdtvx9zYLNouhwNhApXSOo1OzCL7ckEmbJqFMubZ7le325xZjt1lY9MBA4iKDKuxrHGrnP7f344KkxhV+j/ZmF3HDm8v5x6Jt/F/vlsRHh5xxXe+t2MMfujgZd935BAQEcKzUyU1vrWDN7qN8mr6fa89vGCNXFeBERM5CiN19NqxlozP/B6e0zHU8vJV6Qlx+cRl5xWUUFJeRX+ygoKSM/OLyh3t/UWkZhaVOikrczwCmCYWlTgpLnRw6zfeeYOWT3ZvP+mcFd2C0W0+EugBr+cMg4Pj2k98HWN2hL8BqwWY1sFnc723H97tfu9tbLQY2i4H1eBvr8Xbl2y2G+7XncdJ7y0nvT7Rzn3E9ebvFAlbDHUItBp7txi9eW4zj7Q0wMDAsVHx/vI3n+XjfKNy6PTSsQ6XwBu4BSRe2jam0PT46hPGXJvHInAwWbjrILRcknvF33TKgFYml2zzvg+1WfndhImt2H2XFzmwFOBERqRl2m4UmxwdCVJfLZVJc5qSwxElRaRkFJWUUlTopKnVyrNRJscPJMYf7fbHDve2Yw0lBsYOfdu0hJrYZpU6TkjIXxQ5nlc+lZS5KnS6crhPzu5smlJS5KClzkV8TneGHLCeFO5fLyoOrvq4Q/Nxhr+rXlL/HHQjhl/sqHqe8PZ72Jz3zi+1UDJjGL1788niV2gFlLhcA6/flkPLStxXbndTw5a+38srCbZxKUan797XMaVL+21XmdB976sLtvL96n6ftoXz3rQkvpW5lxtJdnu27jhQC8EVGJp8WW7EnZnF5txYAJDYJBSAr/8xva/B1CnAiIj7AYjE8Z//gzIOge9b6XQwf3v2MZ613ukxKy1yUlDmPP7uDXWmZizKnSanTheP445fvHWUmDpd7u8PposxlUuZ04XCalHm2m7hM93uny6TMabqfXeXP7nZO0/2+/OE6+b1p4nS5g63TNHGV7zdNXC487d0PPJ81q3ptul9XV/nx3QycZa7qH6yeOuZwseXgqSP8qabvORNHi0o5WlRaafv+3GL251YOZLuzjwEGOSfdjhBqd8eZUj/s+1NRgBMRkQqsFoNgu7XBLVtmnhT2XOaJgFe+zTTdbUwTTH6xDSgpdbBo0SIuvfRSrDbb8X1gUvkznLStQpvjrzm+/URtFbebJ9VsntSGKtqcfKzy1TPNX2yv0A8nffJQfgn3zU6nY1w4j/2m00lt3Eb/awUxYXamXt+z0rFMTGYs3cmizYfoEBfOVb1a0CIqmFC7e0R2xs+5/H3+Fi5MiuHOi8/zfG7O2n3MTd/P7y5K5KK2JwZOTP5iE5sz83n+qs78tGk9FyY1rlx8A6IAJyIiQvn9c2Cleve1ORxWGgVC86hgv1mjc292EQARwQGVJtYuF2izcsEp9t3/fjpWi8GHdycTHlSxT/YcP3ZsRBAXnzTCdc3uowC0jw2vMPL1n99sB+D81tEEHjCJi6h8z11DYvF2ASIiIlI/2W3umHDyfZFnI/eYg7BAW6XwBjBv/YFzqq2hU4ATERGRKjUKsRNgNdidXVStEJcYE0ruMQef/bC/wvZ/ffcTaT8dqakyGyRdQhUREZEq2W0WLmnXhK83ZXH51G/p0jySAKuF3q0bcW2f00/X8fuBSUx4P517/7uOf6ftJi4yiE0H8thxqIDbL0zkre931sFP4Z90Bk5ERERO6bmru3FVzxYcLXLwyQ/7eX/1Xlb8lH1Gnx3VswUzbjmfnglR/Hggj8VbsoiNCGLWHf0Z3DG2liv3bzoDJyIiIqcUExbIi9f1qHLfrudGnPbzl3ZoyqUdmp7x5+8f0q7K5bXevysZcE+Ns/4X++KjQ86oFn+iM3AiIiIiPkYBTkRERMTHKMCJiIiI+BgFOBEREREfowAnIiIi4mMU4ERERER8jAKciIiIiI9RgBMRERHxMQpwIiIiIj5GAU5ERETEx2gprRpimiYAeXl5NX5sh8NBUVEReXl5BAQE1Pjx5QT1dd1QP9cd9XXdUV/XjYbSz+Hh4RiGccr9CnA1JD8/H4D4+HgvVyIiIiK+Ljc3l4iIiFPuN8zyU0dyTlwuF/v37z9tYq6OvLw84uPj2bt376/+Ycq5U1/XDfVz3VFf1x31dd1oKP2sM3B1xGKx0LJly1r9joiICL/+Za1P1Nd1Q/1cd9TXdUd9XTcaej9rEIOIiIiIj1GAExEREfExCnA+IDAwkCeeeILAwEBvl+L31Nd1Q/1cd9TXdUd9XTfUz24axCAiIiLiY3QGTkRERMTHKMCJiIiI+BgFOBEREREfowAnIiIi4mMU4Oq5adOm0bp1a4KCgujXrx8rV670dkk+79tvv+WKK66gefPmGIbB3LlzK+w3TZPHH3+cZs2aERwczODBg9m2bZt3ivVhkydP5vzzzyc8PJymTZsyatQotmzZUqFNcXEx48aNo3HjxoSFhXH11Vdz8OBBL1Xsu1577TW6devmmdg0OTmZL7/80rNf/Vx7nnvuOQzDYMKECZ5t6u+a8eSTT2IYRoVHhw4dPPsbej8rwNVj77//PhMnTuSJJ55g7dq1dO/enZSUFLKysrxdmk8rLCyke/fuTJs2rcr9zz//PK+88grTp09nxYoVhIaGkpKSQnFxcR1X6tuWLFnCuHHjWL58OampqTgcDoYOHUphYaGnzf33389nn33Ghx9+yJIlS9i/fz9XXXWVF6v2TS1btuS5555jzZo1rF69mssuu4yRI0eyceNGQP1cW1atWsXrr79Ot27dKmxXf9eczp07c+DAAc/j+++/9+xr8P1sSr3Vt29fc9y4cZ73TqfTbN68uTl58mQvVuVfAPPjjz/2vHe5XGZcXJz597//3bMtJyfHDAwMNP/73/96oUL/kZWVZQLmkiVLTNN092tAQID54Ycfetps2rTJBMy0tDRvlek3GjVqZP7rX/9SP9eS/Px8s23btmZqaqp5ySWXmPfdd59pmvq9rklPPPGE2b179yr3qZ9NU2fg6qnS0lLWrFnD4MGDPdssFguDBw8mLS3Ni5X5t507d5KZmVmh3yMjI+nXr5/6/Rzl5uYCEB0dDcCaNWtwOBwV+rpDhw4kJCSor8+B0+lk9uzZFBYWkpycrH6uJePGjWPEiBEV+hX0e13Ttm3bRvPmzTnvvPMYPXo0e/bsAdTPoMXs663Dhw/jdDqJjY2tsD02NpbNmzd7qSr/l5mZCVBlv5fvk7PncrmYMGECF1xwAV26dAHcfW2324mKiqrQVn1dPRkZGSQnJ1NcXExYWBgff/wxnTp1Ij09Xf1cw2bPns3atWtZtWpVpX36va45/fr1Y+bMmbRv354DBw7w1FNPcdFFF7Fhwwb1MwpwIlIHxo0bx4YNGyrcvyI1q3379qSnp5Obm8tHH33E2LFjWbJkibfL8jt79+7lvvvuIzU1laCgIG+X49cuv/xyz+tu3brRr18/WrVqxQcffEBwcLAXK6sfdAm1noqJicFqtVYaUXPw4EHi4uK8VJX/K+9b9XvNGT9+PJ9//jnffPMNLVu29GyPi4ujtLSUnJycCu3V19Vjt9tJSkqid+/eTJ48me7duzN16lT1cw1bs2YNWVlZ9OrVC5vNhs1mY8mSJbzyyivYbDZiY2PV37UkKiqKdu3asX37dv1eowBXb9ntdnr37s3ChQs921wuFwsXLiQ5OdmLlfm3xMRE4uLiKvR7Xl4eK1asUL+fJdM0GT9+PB9//DGLFi0iMTGxwv7evXsTEBBQoa+3bNnCnj171Nc1wOVyUVJSon6uYYMGDSIjI4P09HTPo0+fPowePdrzWv1dOwoKCtixYwfNmjXT7zW6hFqvTZw4kbFjx9KnTx/69u3Lyy+/TGFhIbfeequ3S/NpBQUFbN++3fN+586dpKenEx0dTUJCAhMmTOCZZ56hbdu2JCYm8thjj9G8eXNGjRrlvaJ90Lhx45g1axaffPIJ4eHhnvtSIiMjCQ4OJjIykttvv52JEycSHR1NREQE9957L8nJyfTv39/L1fuWSZMmcfnll5OQkEB+fj6zZs1i8eLFzJ8/X/1cw8LDwz33cZYLDQ2lcePGnu3q75rxxz/+kSuuuIJWrVqxf/9+nnjiCaxWKzfccIN+r0HTiNR3//jHP8yEhATTbrebffv2NZcvX+7tknzeN998YwKVHmPHjjVN0z2VyGOPPWbGxsaagYGB5qBBg8wtW7Z4t2gfVFUfA+aMGTM8bY4dO2b+/ve/Nxs1amSGhISYV155pXngwAHvFe2jbrvtNrNVq1am3W43mzRpYg4aNMhcsGCBZ7/6uXadPI2Iaaq/a8p1111nNmvWzLTb7WaLFi3M6667zty+fbtnf0PvZ8M0TdNL2VFEREREqkH3wImIiIj4GAU4ERERER+jACciIiLiYxTgRERERHyMApyIiIiIj1GAExEREfExCnAiIiIiPkYBTkRERMTHKMCJSIM2cOBAJkyYcMbtd+3ahWEYpKen11pNNWHmzJlERUV5uwwRqSUKcCLiV2655RYMw+Duu++utG/cuHEYhsEtt9zi2TZnzhyefvrpMz5+fHw8Bw4cqLQepohIXVKAExG/Ex8fz+zZszl27JhnW3FxMbNmzSIhIaFC2+joaMLDw8/42Farlbi4OGw2W43VKyJythTgRMTv9OrVi/j4eObMmePZNmfOHBISEujZs2eFtr+8hNq6dWv++te/cttttxEeHk5CQgJvvPGGZ/+ZXEItKSnhj3/8Iy1atCA0NJR+/fqxePFiz/7yy5tz586lbdu2BAUFkZKSwt69eysc57XXXqNNmzbY7Xbat2/Pv//97wr7c3JyuOuuu4iNjSUoKIguXbrw+eefV2gzf/58OnbsSFhYGMOGDePAgQOefYsXL6Zv376EhoYSFRXFBRdcwO7du0/5c4lI/aEAJyJ+6bbbbmPGjBme92+//Ta33nrrGX12ypQp9OnTh3Xr1vH73/+ee+65hy1btpzxd48fP560tDRmz57N+vXrueaaaxg2bBjbtm3ztCkqKuLZZ5/l3XffZenSpeTk5HD99dd79n/88cfcd999PPDAA2zYsIG77rqLW2+9lW+++QYAl8vF5ZdfztKlS/nPf/7Djz/+yHPPPYfVaq3wHS+88AL//ve/+fbbb9mzZw9//OMfASgrK2PUqFFccsklrF+/nrS0NO68804Mwzjjn1NEvMgUEfEjY8eONUeOHGlmZWWZgYGB5q5du8xdu3aZQUFB5qFDh8yRI0eaY8eO9bS/5JJLzPvuu8/zvlWrVuZNN93kee9yucymTZuar732mmmaprlz504TMNetW1fl9+/evdu0Wq3mzz//XGH7oEGDzEmTJpmmaZozZswwAXP58uWe/Zs2bTIBc8WKFaZpmuaAAQPMO+64o8IxrrnmGnP48OGmaZrm/PnzTYvFYm7ZsqXKOsq/Y/v27Z5t06ZNM2NjY03TNM0jR46YgLl48eIqPy8i9Ztu4hARv9SkSRNGjBjBzJkzMU2TESNGEBMTc0af7datm+e1YRjExcWRlZV1Rp/NyMjA6XTSrl27CttLSkpo3Lix573NZuP888/3vO/QoQNRUVFs2rSJvn37smnTJu68884Kx7jggguYOnUqAOnp6bRs2bLS95wsJCSENm3aeN43a9bM83NER0dzyy23kJKSwpAhQxg8eDDXXnstzZo1O6OfU0S8SwFORPzWbbfdxvjx4wGYNm3aGX8uICCgwnvDMHC5XGf02YKCAqxWK2vWrKlwORMgLCzsjGs4neDg4NO2qernME3T837GjBn84Q9/4KuvvuL999/n0UcfJTU1lf79+9dYnSJSO3QPnIj4rWHDhlFaWorD4SAlJaVOvrNnz544nU6ysrJISkqq8IiLi/O0KysrY/Xq1Z73W7ZsIScnh44dOwLQsWNHli5dWuHYS5cupVOnToD7LOG+ffvYunXrOdc7adIkli1bRpcuXZg1a9Y5HU9E6obOwImI37JarWzatMnzui60a9eO0aNHM2bMGKZMmULPnj05dOgQCxcupFu3bowYMQJwnx279957eeWVV7DZbIwfP57+/fvTt29fAB588EGuvfZaevbsyeDBg/nss8+YM2cOX3/9NQCXXHIJF198MVdffTUvvvgiSUlJbN68GcMwGDZs2Gnr3LlzJ2+88Qa//e1vad68OVu2bGHbtm2MGTOm9jpHRGqMzsCJiF+LiIggIiKiTr9zxowZjBkzhgceeID27dszatQoVq1aVWEOupCQEB5++GFuvPFGLrjgAsLCwnj//fc9+0eNGsXUqVN54YUX6Ny5M6+//jozZsxg4MCBnjb/+9//OP/887nhhhvo1KkTDz30EE6n84xqDAkJYfPmzVx99dW0a9eOO++8k3HjxnHXXXfVWD+ISO0xzJNviBARkVo3c+ZMJkyYQE5OjrdLEREfpTNwIiIiIj5GAU5ERETEx+gSqoiIiIiP0Rk4ERERER+jACciIiLiYxTgRERERHyMApyIiIiIj1GAExEREfExCnAiIiIiPkYBTkRERMTHKMCJiIiI+Jj/BzKeFZDPBAS8AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "import joblib\n", + "import matplotlib.pyplot as plt\n", + "import dvu\n", + "dvu.set_style()\n", + "r = joblib.load('results.joblib')\n", + "r.keys()\n", + "\n", + "# plt.figure(figsize=(4, 3), dpi=300)\n", + "plt.plot(r['perplexity_train'], label='train')\n", + "plt.plot(r['perplexity_test'], label='test', color='salmon')\n", + "plt.ylabel('Perplexity')\n", + "plt.xlabel('Mini epochs')\n", + "plt.grid()\n", + "dvu.line_legend(adjust_text_labels=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/auglm/lm.py b/auglm/lm.py index 9de656e..1da6ce2 100644 --- a/auglm/lm.py +++ b/auglm/lm.py @@ -192,12 +192,13 @@ def _update_vocab_emb(self, predicted_emb, next_token_correct_id): max_n_tokens = 3 model_kwargs = dict( device='cuda', - emb_size=1000, - learning_rate=0.1, + emb_size=5000, + learning_rate=0.01, context_length=max_n_tokens, similarity_function='cosine', random_state=42, ) + r = defaultdict(list) # set up data ###################### tokenizer = AutoTokenizer.from_pretrained(tokenizer_checkpoint) @@ -220,25 +221,33 @@ def _update_vocab_emb(self, predicted_emb, next_token_correct_id): **model_kwargs ) - for i in tqdm(range(100)): + for i in tqdm(range(1000)): # fit (calculates perplexity while updating, so technically should rerun to get a frozen estimate) ans_dict = lm.fit_and_calc_perplexity( dset, fit=True, eval_perfect_match=True, - n_examples=10000, + n_examples=100000, seed=42, ) print( f'train perplexity {ans_dict["perplexity"]:.3E} frac-Perfect_match {ans_dict["perfect_match"]:.3f}') + r['perplexity_train'].append(ans_dict['perplexity']) + r['perfect_match_train'].append(ans_dict['perfect_match']) # evaluate on test data ans_dict = lm.fit_and_calc_perplexity( dset_test, fit=False, eval_perfect_match=True, - n_examples=2000, + n_examples=5000, seed=42, ) print( - f'eval perplexity {ans_dict["perplexity"]:.3E} frac-Perfect_match {ans_dict["perfect_match"]:.3f}') + f'test perplexity {ans_dict["perplexity"]:.3E} frac-Perfect_match {ans_dict["perfect_match"]:.3f}') + r['perplexity_test'].append(ans_dict['perplexity']) + r['perfect_match_test'].append(ans_dict['perfect_match']) + + # save results + if i % 10 == 0: + joblib.dump(r, 'results.joblib')