Lesson_7-predibase_lorax.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "095c2c45",
   "metadata": {},
   "source": [
    "## Lesson 7 - LoRAX\n",
    "\n",
    "In this lesson, we explore how to efficiently Serve LLMs using LoRAX!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c1b1de77",
   "metadata": {
    "height": 182
   },
   "outputs": [],
   "source": [
    "import asyncio\n",
    "import json\n",
    "import time\n",
    "from typing import List\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "from pydantic import BaseModel, constr\n",
    "\n",
    "from lorax import AsyncClient, Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4686ef7f",
   "metadata": {
    "height": 29
   },
   "outputs": [],
   "source": [
    "from utils import endpoint_url, headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aeceb612",
   "metadata": {
    "height": 29
   },
   "outputs": [],
   "source": [
    "client = Client(endpoint_url, headers=headers)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a838efbc",
   "metadata": {},
   "source": [
    "## Prefill vs Decode (KV Cache)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1e6100dc",
   "metadata": {
    "height": 143
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Deep learning is a subset of machine learning that uses artificial neural networks to learn from data. It is a powerful tool that can be used to solve complex\n",
      "\n",
      "\n",
      "----------\n",
      "Request duration (s): 1.0511960983276367\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "resp = client.generate(\"What is deep learning?\", max_new_tokens=32)\n",
    "duration_s = time.time() - t0\n",
    "\n",
    "print(resp.generated_text)\n",
    "print(\"\\n\\n----------\")\n",
    "print(\"Request duration (s):\", duration_s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aa960b0d",
   "metadata": {
    "height": 228
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Deep learning is a subset of machine learning that uses artificial neural networks to learn from data. It is a powerful tool that can be used to solve complex\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.6554169654846191\n",
      "Throughput (tok / s): 75.65773949797914\n"
     ]
    }
   ],
   "source": [
    "durations_s = []\n",
    "\n",
    "t0 = time.time()\n",
    "for resp in client.generate_stream(\"What is deep learning?\", max_new_tokens=32):\n",
    "    durations_s.append(time.time() - t0)\n",
    "    if not resp.token.special:\n",
    "        print(resp.token.text, sep=\"\", end=\"\", flush=True)\n",
    "    t0 = time.time()\n",
    "\n",
    "print(\"\\n\\n\\n----------\")\n",
    "print(\"Time to first token (TTFT) (s):\", durations_s[0])\n",
    "print(\"Throughput (tok / s):\", (len(durations_s) - 1) / sum(durations_s[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cc2ec81c",
   "metadata": {
    "height": 46
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/vklEQVR4nO3de3yU9Z3//ffMZDLJhCQcAkkIgXBQEDlEgqSx9VSjaG3FQ7e027uw2S69q3jfttn2p7RbqHa3cavlR7dlZWuXdn89yXrAQ1W0RtBVo2gAOQgoCIRTEsIhCQk5zVy/P+aUhBxmkpm5MpnX89F5mEyumXy5emXmPd/v5/pcFsMwDAEAAJjEavYAAABAfCOMAAAAUxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMlWD2AILhdrt14sQJpaamymKxmD0cAAAQBMMw1NjYqPHjx8tq7X3+IybCyIkTJ5Sbm2v2MAAAwAAcPXpUEyZM6PXnMRFGUlNTJXn+MWlpaSaPBgAABKOhoUG5ubn+9/HexEQY8S3NpKWlEUYAAIgx/ZVYUMAKAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGAqwggAADAVYQQAAJiKMAIAAExFGAEAAKYijAAAAFMRRgAAgKli4kJ5kfKfbx3S4bomfaNoki7N7PuKggAAIDLiembkLztP6PfvHtHhuiazhwIAQNyK6zDiTLRJki60u0weCQAA8SvOw4hnlaq5jTACAIBZ4jyMeGZGmlo7TB4JAADxizAi6QIzIwAAmCauw0iy3btMQ80IAACmieswwswIAADmi+swkuwNI81t1IwAAGCWuA4jTn8YYWYEAACzEEbEMg0AAGaK6zCSTJ8RAABMF9dhxGn3LtNwNg0AAKaJ7zDiX6ahgBUAALPEdRhJpoAVAADTxXUYSXF4akYoYAUAwDxxHUaSvTUjTSzTAABgmrgOI76akZZ2t9xuw+TRAAAQn+I8jCT4v77AGTUAAJgirsNIkt0qi8XzNUWsAACYI67DiMVi8deNUMQKAIA54jqMSJ2uT9NOESsAAGaI+zBCrxEAAMwV92HEaafXCAAAZor7MMLMCAAA5or7MOKvGaHxGQAAphhQGFm7dq3y8vKUlJSkwsJCbd26tc/tz507p+XLlys7O1sOh0OXXnqpXnrppQENONx8vUZYpgEAwBwJ/W/S1YYNG1RaWqp169apsLBQa9as0cKFC7V//36NGzfuou3b2tp04403aty4cXrqqaeUk5OjI0eOaOTIkeEY/6D5ZkaaCCMAAJgi5DCyevVqLVu2TCUlJZKkdevW6cUXX9T69ev1wAMPXLT9+vXrdebMGb3zzjuy2+2SpLy8vMGNOox8YeQCyzQAAJgipGWatrY2VVZWqri4OPAEVquKi4tVUVHR42Oef/55FRUVafny5crMzNSsWbP005/+VC5X7zMRra2tamho6HKLFApYAQAwV0hhpK6uTi6XS5mZmV3uz8zMVHV1dY+P+fTTT/XUU0/J5XLppZde0o9+9CP9/Oc/1z//8z/3+nvKysqUnp7uv+Xm5oYyzJA4CSMAAJgq4mfTuN1ujRs3Tr/+9a9VUFCgxYsX64c//KHWrVvX62NWrFih+vp6/+3o0aMRGx8FrAAAmCukmpGMjAzZbDbV1NR0ub+mpkZZWVk9PiY7O1t2u102m81/32WXXabq6mq1tbUpMTHxosc4HA45HI5QhjZgvmvTNHPVXgAATBHSzEhiYqIKCgpUXl7uv8/tdqu8vFxFRUU9Puazn/2sDhw4ILfb7b/v448/VnZ2do9BJNooYAUAwFwhL9OUlpbq8ccf13/9139p7969uvvuu9XU1OQ/u2bJkiVasWKFf/u7775bZ86c0X333aePP/5YL774on76059q+fLl4ftXDAIFrAAAmCvkU3sXL16sU6dOaeXKlaqurlZ+fr42bdrkL2qtqqqS1RrIOLm5uXrllVf03e9+V3PmzFFOTo7uu+8+3X///eH7VwyCr2aEMAIAgDkshmEYZg+iPw0NDUpPT1d9fb3S0tLC+txvH6jT13/znqZnpuqV714T1ucGACCeBfv+zbVpfMs07dSMAABgBsKIb5mmlWUaAADMQBihgBUAAFPFfRjxnU1zod0lt3vIl88AADDsxH0Y8c2MSFJLB7MjAABEW9yHkaSEQBhhqQYAgOiL+zBitVr8LeG5Pg0AANEX92FEoogVAAAzEUbUuSU8vUYAAIg2wog6XyyPmREAAKKNMCIpmevTAABgGsKIJKfd1xKeMAIAQLQRRiSlOLxhpJWaEQAAoo0wIpZpAAAwE2FEgWWaCyzTAAAQdYQRcWovAABmIoyIpmcAAJiJMCL6jAAAYCbCiChgBQDATIQRsUwDAICZCCPqtEzTTgErAADRRhiRlGxnZgQAALMQRiQ5vTUjFLACABB9hBFJTgczIwAAmIUwos4FrNSMAAAQbYQRSU47p/YCAGAWwogC7eAvtLtkGIbJowEAIL4QRhRYpjEMqaXdbfJoAACIL4QRBU7tlagbAQAg2ggjkqxWi5Lsnl1B3QgAANFFGPHy9xppJ4wAABBNhBEvurACAGAOwogXvUYAADAHYcTLf7E8ZkYAAIgqwoiXr2aEZRoAAKKLMOLFMg0AAOYgjHglJ1LACgCAGQgjXk7CCAAApiCMePn7jBBGAACIKsKIF8s0AACYY0BhZO3atcrLy1NSUpIKCwu1devWXrf93e9+J4vF0uWWlJQ04AFHitPuu3IvBawAAERTyGFkw4YNKi0t1apVq7Rt2zbNnTtXCxcuVG1tba+PSUtL08mTJ/23I0eODGrQkcDMCAAA5gg5jKxevVrLli1TSUmJZs6cqXXr1snpdGr9+vW9PsZisSgrK8t/y8zMHNSgI4E+IwAAmCOkMNLW1qbKykoVFxcHnsBqVXFxsSoqKnp93Pnz5zVp0iTl5uZq0aJF2rNnT5+/p7W1VQ0NDV1ukUYHVgAAzBFSGKmrq5PL5bpoZiMzM1PV1dU9Pmb69Olav369nnvuOf3hD3+Q2+3WVVddpWPHjvX6e8rKypSenu6/5ebmhjLMAUmm6RkAAKaI+Nk0RUVFWrJkifLz83XttdfqmWee0dixY/Uf//EfvT5mxYoVqq+v99+OHj0a6WHSZwQAAJMkhLJxRkaGbDabampqutxfU1OjrKysoJ7Dbrfriiuu0IEDB3rdxuFwyOFwhDK0QaNmBAAAc4Q0M5KYmKiCggKVl5f773O73SovL1dRUVFQz+FyubRr1y5lZ2eHNtIIY2YEAABzhDQzIkmlpaVaunSp5s+frwULFmjNmjVqampSSUmJJGnJkiXKyclRWVmZJOmhhx7SZz7zGU2bNk3nzp3TI488oiNHjugf/uEfwvsvGaRAASs1IwAARFPIYWTx4sU6deqUVq5cqerqauXn52vTpk3+otaqqipZrYEJl7Nnz2rZsmWqrq7WqFGjVFBQoHfeeUczZ84M378iDPwFrO0uGYYhi8Vi8ogAAIgPFsMwDLMH0Z+Ghgalp6ervr5eaWlpEfkd51s7NGvVK5KkfT+5WUnejqwAAGBggn3/5to0Xsmdwgd1IwAARA9hxMtmtciR4Nkd9BoBACB6CCOd0IUVAIDoI4x0Qq8RAACijzDSCVfuBQAg+ggjnfiXadqpGQEAIFoII534wkhTKzMjAABEC2GkE1/NCAWsAABED2Gkk0DNCMs0AABEC2GkE6c90BIeAABEB2GkE/qMAAAQfYSRTpLpMwIAQNQRRjpx0mcEAICoI4x0ElimoYAVAIBoIYx0QgdWAACijzDSSaADK2EEAIBoIYx0kmyngBUAgGgjjHSS4mCZBgCAaCOMdOKkAysAAFFHGOmEZRoAAKKPMNIJHVgBAIg+wkgnnZdpDMMweTQAAMQHwkgnvj4jbkNq7XCbPBoAAOIDYaQTp/faNBJLNQAARAthpBOb1aLEBM8uaabxGQAAUUEY6Ybr0wAAEF2EkW6cdhqfAQAQTYSRbrhYHgAA0UUY6cZXxEoBKwAA0UEY6cZXM9JEzQgAAFFBGOnGyTINAABRRRjphmUaAACiizDSDQWsAABEF2GkG/qMAAAQXYSRbpgZAQAguggj3TjtnpoR2sEDABAdhJFuAss0hBEAAKKBMNJNYJmGmhEAAKKBMNINfUYAAIguwkg3LNMAABBdAwoja9euVV5enpKSklRYWKitW7cG9bgnnnhCFotFt99++0B+bVT4mp41EUYAAIiKkMPIhg0bVFpaqlWrVmnbtm2aO3euFi5cqNra2j4fd/jwYX3ve9/T1VdfPeDBRgN9RgAAiK6Qw8jq1au1bNkylZSUaObMmVq3bp2cTqfWr1/f62NcLpe+/vWv68EHH9SUKVMGNeBIo88IAADRFVIYaWtrU2VlpYqLiwNPYLWquLhYFRUVvT7uoYce0rhx4/TNb34zqN/T2tqqhoaGLrdo4do0AABEV0hhpK6uTi6XS5mZmV3uz8zMVHV1dY+Peeutt/Sf//mfevzxx4P+PWVlZUpPT/ffcnNzQxnmoPjPpml3yTCMqP1eAADiVUTPpmlsbNQ3vvENPf7448rIyAj6cStWrFB9fb3/dvTo0QiOsivfMo3LbajN5Y7a7wUAIF4lhLJxRkaGbDabampqutxfU1OjrKysi7Y/ePCgDh8+rC996Uv++9xuzxt8QkKC9u/fr6lTp170OIfDIYfDEcrQwsZpt/m/vtDmkiPB1sfWAABgsEKaGUlMTFRBQYHKy8v997ndbpWXl6uoqOii7WfMmKFdu3Zpx44d/tttt92m66+/Xjt27Ijq8kuwEmxWJdo8u4UiVgAAIi+kmRFJKi0t1dKlSzV//nwtWLBAa9asUVNTk0pKSiRJS5YsUU5OjsrKypSUlKRZs2Z1efzIkSMl6aL7h5LkRJvaLrgJIwAAREHIYWTx4sU6deqUVq5cqerqauXn52vTpk3+otaqqipZrbHd2NWZaFP9hXbOqAEAIAosRgycMtLQ0KD09HTV19crLS0t4r/v8z/fok9PNWnDtz6jwiljIv77AAAYjoJ9/47tKYwISfH2GmGZBgCAyCOM9IAurAAARA9hpAf+xmdcnwYAgIgjjPTAf7G8dmZGAACINMJID5Lt1IwAABAthJEeOKkZAQAgaggjPfAv01AzAgBAxBFGesDZNAAARA9hpAeBmRHCCAAAkUYY6UEyTc8AAIgawkgPnHbvMg2n9gIAEHGEkR5QwAoAQPQQRnrgdHiWaZpamRkBACDSCCM9oAMrAADRQxjpQbKda9MAABAthJEe0IEVAIDoIYz0wOk9tZc+IwAARB5hpAe+DqwdbkNtHW6TRwMAwPBGGOmBb5lGYnYEAIBII4z0wG6zym6zSJKa2yliBQAgkggjvQicUcPMCAAAkUQY6QVFrAAARAdhpBec3gsAQHQQRnrhdHjCSBONzwAAiCjCSC+cdpZpAACIBsJIL5JZpgEAICoII73wXyyPZRoAACKKMNILZkYAAIgOwkgvOJsGAIDoIIz0wt9npJ0wAgBAJBFGehHowErNCAAAkUQY6QXLNAAARAdhpBeBs2kIIwAARBJhpBfJ3poRZkYAAIgswkgvUhKpGQEAIBoII72gzwgAANFBGOmF/9RewggAABFFGOkFZ9MAABAdhJFeJFMzAgBAVBBGeuE/tZcOrAAARBRhpBdOu6dmpN1lqN3lNnk0AAAMXwMKI2vXrlVeXp6SkpJUWFiorVu39rrtM888o/nz52vkyJFKSUlRfn6+fv/73w94wNHiW6aRqBsBACCSQg4jGzZsUGlpqVatWqVt27Zp7ty5WrhwoWpra3vcfvTo0frhD3+oiooK7dy5UyUlJSopKdErr7wy6MFHUmKCVQlWiyTOqAEAIJJCDiOrV6/WsmXLVFJSopkzZ2rdunVyOp1av359j9tfd911uuOOO3TZZZdp6tSpuu+++zRnzhy99dZbgx58pFHECgBA5IUURtra2lRZWani4uLAE1itKi4uVkVFRb+PNwxD5eXl2r9/v6655ppet2ttbVVDQ0OXmxk4vRcAgMgLKYzU1dXJ5XIpMzOzy/2ZmZmqrq7u9XH19fUaMWKEEhMTdeutt+qXv/ylbrzxxl63LysrU3p6uv+Wm5sbyjDDxt/4jDNqAACImKicTZOamqodO3bo/fff17/8y7+otLRUW7Zs6XX7FStWqL6+3n87evRoNIZ5Ed/MSFMryzQAAERKQigbZ2RkyGazqaampsv9NTU1ysrK6vVxVqtV06ZNkyTl5+dr7969Kisr03XXXdfj9g6HQw6HI5ShRYS/1wjLNAAARExIMyOJiYkqKChQeXm5/z63263y8nIVFRUF/Txut1utra2h/GpTJHuXaagZAQAgckKaGZGk0tJSLV26VPPnz9eCBQu0Zs0aNTU1qaSkRJK0ZMkS5eTkqKysTJKn/mP+/PmaOnWqWltb9dJLL+n3v/+9HnvssfD+SyLAafcWsFIzAgBAxIQcRhYvXqxTp05p5cqVqq6uVn5+vjZt2uQvaq2qqpLVGphwaWpq0j333KNjx44pOTlZM2bM0B/+8ActXrw4fP+KCAks01AzAgBApFgMwzDMHkR/GhoalJ6ervr6eqWlpUXt9/5w4y798b0qfaf4En2n+NKo/V4AAIaDYN+/uTZNHyhgBQAg8ggjfaCAFQCAyCOM9IEOrAAARB5hpA/+ZZp2ClgBAIgUwkgfku3MjAAAEGmEkT6kOLw1I62EEQAAIoUw0odkX80IyzQAAEQMYaQPTpZpAACIOMJIH5zeU3vpMwIAQOQQRvqQzKm9AABEHGGkD3RgBQAg8ggjffCFkTaXWx0ut8mjAQBgeCKM9MG3TCNJze3MjgAAEAmEkT4k2qyyWS2SWKoBACBSCCN9sFgsnN4LAECEEUb6ETijhsZnAABEAmGkH1y5FwCAyCKM9MPX+IwwAgBAZBBG+hHoNcIyDQAAkUAY6QddWAEAiCzCSD+oGQEAILIII/3gYnkAAEQWYaQfLNMAABBZhJF++JuetVPACgBAJBBG+sGVewEAiCzCSD+S6TMCAEBEEUb6wcwIAACRRRjph6+AtYmmZwAARARhpB8pLNMAABBRhJF+sEwDAEBkEUb6EegzwjINAACRQBjpBzMjAABEFmGkH/5r07QTRgAAiATCSD/oMwIAQGQRRvrhawff1uGWy22YPBoAAIYfwkg/fAWsEkWsAABEAmGkH44Eq6wWz9cUsQIAEH6EkX5YLBY5qRsBACBiCCNBCPQaIYwAABBuhJEgpND4DACAiBlQGFm7dq3y8vKUlJSkwsJCbd26tddtH3/8cV199dUaNWqURo0apeLi4j63H4o4vRcAgMgJOYxs2LBBpaWlWrVqlbZt26a5c+dq4cKFqq2t7XH7LVu26Gtf+5o2b96siooK5ebm6qabbtLx48cHPfhocbJMAwBAxIQcRlavXq1ly5appKREM2fO1Lp16+R0OrV+/foet//jH/+oe+65R/n5+ZoxY4Z+85vfyO12q7y8fNCDjxZ/S/h2lmkAAAi3kMJIW1ubKisrVVxcHHgCq1XFxcWqqKgI6jmam5vV3t6u0aNH97pNa2urGhoautzMlGxnZgQAgEgJKYzU1dXJ5XIpMzOzy/2ZmZmqrq4O6jnuv/9+jR8/vkug6a6srEzp6en+W25ubijDDDsulgcAQORE9Wyahx9+WE888YQ2btyopKSkXrdbsWKF6uvr/bejR49GcZQXo4AVAIDISQhl44yMDNlsNtXU1HS5v6amRllZWX0+9tFHH9XDDz+s1157TXPmzOlzW4fDIYfDEcrQIooCVgAAIiekmZHExEQVFBR0KT71FaMWFRX1+rif/exn+slPfqJNmzZp/vz5Ax+tSQLLNBSwAgAQbiHNjEhSaWmpli5dqvnz52vBggVas2aNmpqaVFJSIklasmSJcnJyVFZWJkn613/9V61cuVJ/+tOflJeX568tGTFihEaMGBHGf0rk0IEVAIDICTmMLF68WKdOndLKlStVXV2t/Px8bdq0yV/UWlVVJas1MOHy2GOPqa2tTV/+8pe7PM+qVav04x//eHCjjxKn72yadsIIAADhFnIYkaR7771X9957b48/27JlS5fvDx8+PJBfMaT4L5TXyjINAADhxrVpguB0sEwDAECkEEaCEOjAShgBACDcCCNBSLbTZwQAgEghjASBDqwAAEQOYSQIgaZnFLACABBuhJEg0GcEAIDIIYwEwXdqb2uHWy63YfJoAAAYXggjQfAt00icUQMAQLgRRoLgSLDKYvF8Td0IAADhRRgJgsVi8beE54waAADCizASpGRv3UhTK2EEAIBwIowEKcXh68LKMg0AAOFEGAlSsp3TewEAiATCSJCc9BoBACAiCCNB8vUaoYAVAIDwIowEiS6sAABEBmEkSFyfBgCAyCCMBIkr9wIAEBmEkSAl2z01I820gwcAIKwII0FiZgQAgMggjAQpmZoRAAAigjASJN/MSBMzIwAAhBVhJEgp9BkBACAiCCNBYpkGAIDIIIwEiQJWAAAigzASJDqwAgAQGYSRIPmuTUMYAQAgvAgjQfIv09D0DACAsCKMBCnZTgErAACRQBgJkm9mpKXdLbfbMHk0AAAMH4SRIPlqRiSWagAACCfCSJCS7FZZLJ6vKWIFACB8CCNBslgs/roReo0AABA+hJEQBK5PQxErAADhQhgJAb1GAAAIP8JICGgJDwBA+BFGQsDF8gAACD/CSAjowgoAQPgRRkKQbKdmBACAcCOMhMDJlXsBAAi7AYWRtWvXKi8vT0lJSSosLNTWrVt73XbPnj266667lJeXJ4vFojVr1gx0rKYLFLBSMwIAQLiEHEY2bNig0tJSrVq1Stu2bdPcuXO1cOFC1dbW9rh9c3OzpkyZoocfflhZWVmDHrCZkpkZAQAg7EIOI6tXr9ayZctUUlKimTNnat26dXI6nVq/fn2P21955ZV65JFH9NWvflUOh2PQAzYTyzQAAIRfSGGkra1NlZWVKi4uDjyB1ari4mJVVFSEbVCtra1qaGjochsKfE3P6DMCAED4hBRG6urq5HK5lJmZ2eX+zMxMVVdXh21QZWVlSk9P999yc3PD9tyD4bs2De3gAQAInyF5Ns2KFStUX1/vvx09etTsIUmSUhx0YAUAINwSQtk4IyNDNptNNTU1Xe6vqakJa3Gqw+EYkvUlyVybBgCAsAtpZiQxMVEFBQUqLy/33+d2u1VeXq6ioqKwD26ocXqXaZrpwAoAQNiENDMiSaWlpVq6dKnmz5+vBQsWaM2aNWpqalJJSYkkacmSJcrJyVFZWZkkT9HrRx995P/6+PHj2rFjh0aMGKFp06aF8Z8SefQZAQAg/EIOI4sXL9apU6e0cuVKVVdXKz8/X5s2bfIXtVZVVclqDUy4nDhxQldccYX/+0cffVSPPvqorr32Wm3ZsmXw/4Ioos8IAADhZzEMwzB7EP1paGhQenq66uvrlZaWZto49lc3auGaNzUmJVGVP7rRtHEAABALgn3/HpJn0wxVND0DACD8CCMh8C3TXGh3ye0e8hNKAADEBMJICHwzI5LU0sHsCIDYVV3fos37e76mGBBthJEQJCUEwki8LNV8euq8DtU1mT2MIa3qdDON8BBz7v5jpUp++75e2RO+7tnAQBFGQmC1Wvwt4Ztbh/+bz9mmNt32q7d12y/fUn1zu9nDGZLe+/S0rnt0s0r/e4fZQwGCdqC2UdurzkmSnqo8Zu5ghijDMLTyud1a9dxuxcB5HjGPMBIiX0v45vbh32vkhZ0ndL61Q42tHfrLrhNmD2dI+sN7VXIb0it7qlVd32L2cICgbNx+3P/1lv21OtfcZuJohqbKI2f1fyqO6L8qjmj70XNmD2fYI4yEKJ56jTy9LfCC9TSfni7S0NKuV71T3G5DenbH8X4eAZjP7Tb07HbPh4tEm1XtLkMv7jpp8qiGnqe3BV7zeP2LPMJIiJx2T5+44V4jcKD2vD48ek42q0U2q0Xbqs5RO9LNiztPqrXDLavF8/3TlceYzsWQ98GRszp+7oJSHQlafr2nC/Zz25n57Kyl3aW/fBgIaC98eEItXAYkoggjIYqXmZFnvJ8Krrt0rK6+JKPLffDw7Y+7r5uqxASrPqk9rz0nGkweFdA33xLNLbOz9JUrJ8hikbYePqNjZ5tNHtnQ8epHNWps7VDOyGRlpyepoaVD5Xs58yiSCCMhCjQ+G741Iy634X/Buqtggu6aN0GS9My24/RX8TpyuknvHz4rq0VaUpSnm2Z6LofwNIENQ1hrh0sv7vTMgtx+RY6y05NVOHm0JOm5HcyO+Pg+aNw5L0d3XJEjib/tSCOMhChwsbzhOzPy7qendbK+RWlJCfr8jHG6cWamUh0JOn7ugrYePmP28IYEXz3N5y4Zq8y0JH9ge37HCbW73GYODejV5n21amjpUHZ6kj4zeYwk6fZ8z5vtczuOs8woqbahRW9+fEqSdOe8CbqrwPO3/cbHp3SqsdXMoQ1rhJEQJSd6akaG8zKNr1jrS3PHK8luU5LdplvnZHf5WTxzuw3/J6e75nleyK++JEMZIxJ1uqlNb+w/ZebwgF75Zjxvyx8vq7fY6ZbZ2Uq0WfVxzXntPdlo5vCGhGd3HJfbkAomjdLkjBRNHTtC+bkj5XIbeo4i9YghjITIaQ+0hB+Omlo79PJuzxkid3o/7Uvyfzp4adfJYT0rFIz3D5/RsbMXNMKRoJtmZkmSEmxWLfJ+wnxmO4EtXNo63Mw0hcm55jZt3ucJyr6lB0lKT7br8zPGSeKMMMMw9HSlZx/cOS+wj3yvf/RkiRzCSIiSh3nNyMu7q3Wh3aXJGSmaN3Gk//75k0Zp4minmtpccd+x0bd2fOvsbP/xIAVevF77qJYmcWFQf6FdN/7vN7Twf78Z9wE4HF7aVa02l1uXZadpRlbXq6fefsV4SZ5lRlcc14XtOdGg/TWNSkyw6otzxvvv/9Icz+zRvupG7TlRb+IIhy/CSIiG+5V7fcswd83LkcVi8d9vsVj8b7bxXMh1oc2ll3Z5wpjv05LP5ePTNSMrVW0ut17YSTHgYK1+db+OnG7Wp3VN2vB+ldnDiXnPepdo7rhi/EU/u276OKUlJai6oUXvHTod7aENGb6ZjxtnZio92e6/f6QzUcUzPbNHvpkThBdhJET+MDIM28EfO9usik89L0S3d5rG9bnzCs+b79sH6uK22+irH1XrfGuHckcn68q8URf9PHDmUfwGtnDYfbxev3/3iP/7/3jzU7V1sFwzUEfPNGvr4TOyWKTb5l78t51kt+kLsz11Yc9uj88327YOt57/0PMh4svzJlz0c9/f9nM7jrN0GAGEkRD5C1iHYc2I70WoaMoYTRjlvOjnE8c4dWXeqLjuNur75HTnFRO6zBz5LMofL6tFNIkbBLfbc00QtyEtvDxT41IdOlnfoo3U4gyYr/DyqqljlJWe1OM2vpqnl3dVx2WDry37a3WmqU1jUx3+3kqdXXPpWIrUI4gwEqIU/6m9w6tmxDAMPbPt4sKt7nyfDuKx22h1fYvePlAnKbAfuhuXlqSrLxkrSdrI7MiAPFV5TNuqzsmZaNODt83SsqunSJIe23JQHXwiDZlhBPoG+U7j7Unh5NHKTk9SY2uHNu+LvwZfvte/2/PHK8F28VujvVORejwvVUcKYSREw7UD6/aj5/RpXZOS7Tbd4p2u7ckX5mTL4e02uvt4fHUb3bjdc8rfgrzRmjjm4pkjH18tydM0iQvZueY2PbxpnyTpO8WXKCs9SX9bOFEjnXYdPt3MNVQGYPfxBh081aQku1U3z8rqdTur1aLb8j31JPE283m2qU3l+2okXVwL1pnvQ0j5Xi4uGG6EkRA5h2mfEV+Nw82zsjTCkdDrdmlJdt10uecFLZ4+HXhmjgJdGftyE03iBuzRV/frTFObLhk3QiWfnSxJSnEk6O+9X//75oMEvBD5ZkVunJml1CR7n9v6Zk427zsVV2eEvbDzhNpdhmb2cKZRZzPHpwWK1D+kSD2cCCMhGo4dWFs7XHrBe1Go3pYfOvO9GT//Yfx0G911vF6f1J6XI8GqL8zpfeZI6loMSCFr8HYeO6c/vuc5a+ahRbNk7zRVvrQoTyMcCdpf06jX9taYNcSY0+EKFGX2dBZNd5dlp2l6pufN9qXd8TML5T+LsI9ZEZ8v+3qObIuv2aNII4yEyL9M0z58akZe31ur+gvtykpLUtHUMf1uf/W0DI1NdehMU5u2xEkhl+/FauHlWUrr59OlFAhsL+2qHlbBNVLcbkM/ena3DMNTBNz9OEx32vWNokmSpLWbD8RdvdJAvXWgTnXnWzU6JdFfy9Qf35l08XJWzYHaRn14rF4JVosW5fcf2Bbl58hmtejDo+d0oPZ8FEYYHwgjIRqOMyO+5ZY75nn+yPqTYLPqdu8fbTx88u98yl8wn5wk6cq80codnazzrR169aP4bhIXjA0fHNWHx+o1wpGgH37hsh63+ebnJivJbtWHx+r1lreQGH3zBYovzcnuMtPUF1/dyHuHzuj4uQsRG9tQ8ZS3b8h108cqY4Sj3+3Hpjp07aWeYBdPS9WRRhgJkdM+vGpG6s63+mc37uqnFqKzO+OokGvz/lqdbW7XuFSHPjft4lP+emK1WnTHFYFCVvTuTFOb/tVbtPrdGy/VuLSeTz3NGOHQV6+cKMkzO4K+NbV26JU9niWtO4JYfvXJGZmsBd4r+T4/zK/k67lCua/RY/D7yLftxm3H47pjbTgRRkLkW6a50O4aFlPFz+84oQ63obkT0jVtXGrQj7ssO00zs9O83UaH99qyb4nmjiuCmznyudM73f3WJ6dU0xCfTeKC8cgr+3SuuV0zslK11LsU05v/99opstssevfTM6o8QnFwX179KHBph7kT0kN6rO/aNcP9wnBvH6hTTUOr5/o8l40L+nE3XBboWPvOQWbpwoEwEiLfMo1hSC3tsV+8+bT/DJHgPxX4+OoihvNSzZmmNm3e7+m5EOwSjU9eRooKJnmaxA33F/WB2l51Vk+8f1SSp2i1p/4OnWWnJ/s/lf7qdWZH+hLom5HTY4O+vnxhVuBaLHtPDt9T+H2vXbfNHS9Hgq2frQOS7DZ9aa5nOYsrmYcHYSREyfbAARvrF8vbV92gPScaZLdZdNvc/gu3uvMVcm2vOqeDp4ZnIdcLH3pO+Zudk65LM4OfOfIJNIk7Pixm0sLJ5Tb0o+c8Rat3zsvxLw3059vXTpXVIm3ef0q7j3PRsp7UNgQa9N0exFk03aU77bpuuqcuYrj2HGlsademPT1fZyoYvsds2lOtxpb4OQ06UggjIbJaLUqye3ZbrNeN+D45fX7GOI1KSQz58Z0LuTYO07qIp4PsLdKbW2dnKzHBqv01jdpzYvh+whyIP22t0u7jDUpNStCKW3ouWu1JXkaK/1Ppv29hdqQnz394Qm5DmjdxpCaNSRnQc/iWal7YcWJY9nbxtL13a8rY0JexJOmK3JGakpGilna3Xt5NkfpgEUYGIGUYND7rcLn9zZAGskTj43uT3rh9+HUb/aSmUTu9p/wNZOZI8nzCvPGyTEmB8Afp9PlWPeItWv3eTdM1NrX/sxg6u+e6aZKkl3dX60BtY9jHF+t8sxmhFK52d/2McUpNStCJ+pZh2bzvqW2BwtVQl7Ekz5XM/d2WWaoZNMLIAARawsfuMs1bB+p0qrFVo5x2XT89+MKt7oovy1Rqkqfb6LvD7NLjvher62eM05ggTvnrTaBJHFf79Hn45X1qaOnQ5ePT9P98pu+i1Z5Mz0rVTTMzZRjSv285GIERxq5Pahq1+3iDEqwWfbGPSzv0J8lu0y3e9vHDrefI0TPN2nrIcxXjgc56Sp7ZI4vFcxr00TPNYRxh/CGMDMBw6DXi+5R+29zxSkwY+GGQZLfpi3N8hVzD5wXL5Tb8L8ChnPLXE9/VPuvOt+nNj+OjSVxfKo+c0ZPeT5IPLZoV0hlKnd37ec/syHM7TvBG0IlvxvO66QNbfu3M1x7+pV0n1doRu6933fmWXz87NUPZ6ckDfp7xI5N1lbdBHzOfg0MYGYDkGF+maWhp1yuDKNzqztef5OXdJ2N6tqgz3yl/I512XT8juM6VvbHbrLptru/Mo/h+wepwufWjZ/dIkr4yf4IKJo0a8HPNmTBSV1+SIZfb0Lo3mB2RPJ1sn9vha/8+8E/8PoVTxigrLUkNLR3avG94BOnOVyi/q2Dw+8j3YeWZ7fF3JfNwIowMgNPuawkfm2HkpZ0n1drh1rRxIzQ7J/TCre4KJo3SpDFONbe5/CEn1j09wFP+euObCv7r3pq4ugBZd39494g+Otmg9GS77r95xqCf797rPbMjT35wjF4ukt4/7OmamupI0A0h9M3oja3zlXyHyVLNB0fOqupMs1ISbVp4ee9XMQ7WzbOylJJo05HTzfrgyNkwjDA+EUYGILBME5uzAP5PBQMs3OrOYrHozisCp7DGusbOM0eDXKLxuXy89wJkHW69uGt4N4nrzanGVv381Y8lSd9fOH1QdTg+hVPG6Mq8UWpzufX4m58O+vlina9w9Quzs5VkH3yIluS/Xsvr+zzXsIp1vmLTL8zO9l+FfTCciQm6xVubQyHrwBFGBiBQwBp7MyNVp5u19bCncGsg/Qd64/vk//bBOp2sj+3rWfhO+Zs6NkVzBnDKX088lfeefRSv17Moe3mvGls7NGdCur62YGLYnne5d3bkj+9V6UzT8L40QV9a2l36i7cb8u1hWKLxmZmdpkszR6jN5damGL+Sb0u7Sy9699FgziLszveh5cWdJ9USozPmZiOMDIAzhsPIM97rMHxu2uAKt7rLHe3UgsmjZRiBArpY5T/lryA8M0c+i/JzZLVIlUfO6nBdU9ieNxZsPXRGz2w7LotF+skgilZ7cu2lYzU7J10X2l367duHwva8sWbzvlo1tnQoOz1JhUE2kAuGxWLRovzAKfyx7JU91Wps7VDOyOSw7qPCyaOVMzJZja0dw2apOtoIIwPgm9qLtbNpOhduDeZ0tt7cNS9QpBmrhVydT/kLRwFgZ5lpSfqc9zLuz8T4i3oo2l1u/ejZ3ZKkr145UXNzR4b1+S0Wi5ZfP1WS9Lt3DqshTrth+oLCovwcWcMY9jzPGbiSbyzPfD7tX6IO7z6yWi3+1z8ujDkwhJEBiNVlmnAXbnX3hdnZciRYdaD2vHbFaJtuX1gL98yRz12drucz3JrEdXahzaX/+eSU/nXTPt2+9m3tr2nUKKdd/2vh9Ij8vptmZmnauBFqbOnQ7yuOROR3DGXnmgPXUAp3iJakCaOcWpDnmfmM1Sv51jS06K1PPGcEhXOJxsf3nFwYc2AIIwPgP5smxgpYw1241V1qkt0fcmKxkMswDP8yViRmjiTPm+YIR4KOnb2g94dRV8u2DrfeP3xGa177WF/5jwrNefAVfeM/t+qxLQe150SDbFaLfnzb5YPue9Ebq9Wie67zzI6sf+tQzM1aDtaLu06q3WVoZnaapmeFfg2lYCzy1pg9G6NhZOP243Ib0vxJo5SXMbAW+X3Jy0jRfO+FMWN9OcsMhJEBcDpir89IpAq3ugt0Gz2hto7Y6jb6wZGzOnI6cjNHkmdW7QuzPc8dyz1HXG5DO4+d07o3DmrJ+q2a++Cr+pt1FVrz2ifaeuiM2l2GstOTdNe8Cfr538zVW/df7687iJTb5o5X7uhknW5q05+3VkX0dw01vtNuIzEr4nPr7GzZbRbtPdmg/dWx1YLfMAz/B6Rw9FbqTef28LG6VG2WAYWRtWvXKi8vT0lJSSosLNTWrVv73P7JJ5/UjBkzlJSUpNmzZ+ull14a0GCHilgsYH31o5qIFG5197lpGRqX6tDZ5nb/tHGsiPTMkY8vDL64K3Yq7xtb2rXrWL1++/YhLfs/Hyj/oVd126/e1sMv79ObH5/ShXaXRqck6tY52fqXO2Zpy/eu0zsPfF4//8pc3VUwISJLXt0l2Kz69rWe2ZFfv/npsOoY2pejZ5r1/uGzsljk7wkSCSOdibr2Uk/vkli7ku/u4w36pPa8EhOsunXOwFvk9+fWOZ6l6k9ieKnaLCG/4m7YsEGlpaVat26dCgsLtWbNGi1cuFD79+/XuHEXN9l555139LWvfU1lZWX64he/qD/96U+6/fbbtW3bNs2aNSss/4ho8/cZaY+dZRrfG+2dYS7c6i7BZtXtV+To129+qme2HYvYDEO4dZ45iuQnJ0lakOepvD9+7oJe/ahmwBfhC7f6C+06crpJh+qadOR0sw6f9vz3yOkm1Z2/+JTZVEeCCqeM1lVTM3TVtDG6dFxqRI+tYHy5YIL+rfwTVTe06Jltx8N6CvFQ5ZsV+ezUDGWmJUX0d91xRY5e21uj53ec0Pdvmm76/9/B8p1Of9PMTKUl2SP2e9KS7Lrp8iy98OEJPV15THMmjIzY7xost9vQuQvtOn2+VafOt+r0+TZdN32sUiO4f/oSchhZvXq1li1bppKSEknSunXr9OKLL2r9+vV64IEHLtr+F7/4hW6++WZ9//vflyT95Cc/0V//+lf96le/0rp16wY5fHMk22NrZqS2oUX/E8HCre7umjdBv37zU72+r1Znm9oiVicQTp1njhbkRW7mSPLUN9w5L0e/fP2Anq48FrYwYhiGOtyGOlyG2t1udbgMdbjcancbcnW673xrh6rONOlwnSdoHPYGjrP9dIbNGJGoGVlpumraGF01NUOzxqcpwTa0VnodCTYtu3qK/vnFvXpsy0H9TcGEAY/R7TbkMgy5DUNut+QyDLnchgzvf13e+93e792GIV9NstUiWWSRxSLvzXLxffLeZ7HIosB9kuT/T9dv/aeaB76XNnpnKcLZW6Q3N1w2TiMcngtjvn/4jAqnjIn47xystg63ntvha/8ejde/HL3w4Qk9/+EJ/fDWmYO69ld/uh+LHW63zjW3q84bLk43tarufFuX70+fb1Pd+TadaWpV9xr6F+79nGaHqbdSqEIKI21tbaqsrNSKFSv891mtVhUXF6uioqLHx1RUVKi0tLTLfQsXLtSzzz4b+miHCN8U/r6TjVq09m253Z43Ad+Ll8vd7WYYF20jSYYhGQp8LUn+Y8P/fdefd9e5DYb/hSxwh/exnhfJeRNHanIECre6m56VqsvHp2nPiQZd88hm2Tu9GfS0jtrTP80wPC/y8vxPbsPw7y+3Ie/9hn87QxfvI8+Le7cX+24v/L5t2rxX0w33KX+9uXPeBP3y9QP6n09O6SvrKvzHjbvT8RN4k1MP93UKHi63Orw/H6xxqQ7ljUnRpDFO5WV4/zsmRRPHOCP6iTKc/rZwov59y0FVnWnWdY9uUYLV4n+xDoSGwH42DHXZ/779HUuS7FbdPCvys5BJdptunpWlpyqP6ccvfKTLslI9f1PevyOrpWsA6/y35gte4dD5dcTocr/vvsC9p8+36Wxzu8amOnT1tIyw/P6+fG5ahsamOnSqsVXf/K/3NcKR4P/7bHe55XL7/nYDX3f/Wfe/9y6vA0bgfSQcZSkjnXaNSUlUxgiHwthWKWQhhZG6ujq5XC5lZmZ2uT8zM1P79u3r8THV1dU9bl9d3XtjmNbWVrW2tvq/b2hoCGWYEZc72rP+faHdpQ+PnjN3MF30fWSWfHZylMYhLS3K0/96eqcaW8xbyvJmmW4ppfd95Ey06W/m50Z6WJKkyRkpKpw8Wu8dOqOtETyrxmqREqxWJdgsSrBaZLdZlWS3KXd0siZnpGjSmBTljXFq0pgUTRztVIojcrUy0eJMTNC3r52in760T8fORq4nhsUi2SwWWa2eN1qbxSKr99W8c4D2hWV1Cc7GRZ9KB+Mbn5mkEVH6/+6ueRP0VOUx7T3ZoL0nh9Zrc1/uvCInKjN5CTar7po3QeveOKj/+aQu4r+vO0eCVRkjHMoYkagxIxwak+L5b8YIT+AYMyJRY1I8349KSezyYdFMQ/KVp6ysTA8++KDZw+jVpDEp+sv/9zkdO3tBCVaLbJ1uVotFCTbPf21WzxtA9/ts3k8PnVksPU/Ber63dPle6vr+2vlTQNf7A5LtNo2O4nLJV67MVdHUMX0WaPaXwn2frPyftnTxpy5rp09lvk9o/hknbxjxfW/4vw98ouj8s9EjEqP66f/fvz5P737qCSI2q/zHh9V7jPR0PHne/OQ/tnxBw26zdvk+wWaR3WqNmTX9cPuHz01R4eQxau1wy2pRl31qsajTvvTu20771Wrp9rdqVZf/P6zex4fjU77vWPQfl/77vf/tZWa086yqRRZ/76NoKJo6Ro99fZ6Onm3uNHbvTKYC/yZ3L393hoyLZ3GD1HUmuOcfWHq4OyUxQV9dEJ0PGpL0/98wTbmjk9Xa7pbdZpHN6v37tPneFwIfEGzeDwn+v2mbtcvx6DtObd1eH6ydj8tOP3ckWMPaOTpaQgojGRkZstlsqqmp6XJ/TU2NsrJ6niLMysoKaXtJWrFiRZelnYaGBuXmRu9ACsasnHTNCsMVb4ez3NFOs4cwpI0Z4YhoZX88s1otYe/0Ggm+AO39zsyhhMR3YTj0zJmYoK8XTjJ7GDElpPmZxMREFRQUqLy83H+f2+1WeXm5ioqKenxMUVFRl+0l6a9//Wuv20uSw+FQWlpalxsAABieQl6mKS0t1dKlSzV//nwtWLBAa9asUVNTk//smiVLlignJ0dlZWWSpPvuu0/XXnutfv7zn+vWW2/VE088oQ8++EC//vWvw/svAQAAMSnkMLJ48WKdOnVKK1euVHV1tfLz87Vp0yZ/kWpVVZWs1sCEy1VXXaU//elP+qd/+if94Ac/0CWXXKJnn302ZnuMAACA8LIYMdCztqGhQenp6aqvr2fJBgCAGBHs+/fQOKcHAADELcIIAAAwFWEEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGCqkNvBm8HXJLahocHkkQAAgGD53rf7a/YeE2GksbFRkpSbm2vySAAAQKgaGxuVnp7e689j4to0brdbJ06cUGpqqiwWS9iet6GhQbm5uTp69CjXvOkF+6h/7KO+sX/6xz7qH/uof0NxHxmGocbGRo0fP77LRXS7i4mZEavVqgkTJkTs+dPS0obM/3FDFfuof+yjvrF/+sc+6h/7qH9DbR/1NSPiQwErAAAwFWEEAACYKq7DiMPh0KpVq+RwOMweypDFPuof+6hv7J/+sY/6xz7qXyzvo5goYAUAAMNXXM+MAAAA8xFGAACAqQgjAADAVIQRAABgqrgOI2vXrlVeXp6SkpJUWFiorVu3mj2kIePHP/6xLBZLl9uMGTPMHpZp3nzzTX3pS1/S+PHjZbFY9Oyzz3b5uWEYWrlypbKzs5WcnKzi4mJ98skn5gzWJP3to7/7u7+76Ji6+eabzRmsCcrKynTllVcqNTVV48aN0+233679+/d32aalpUXLly/XmDFjNGLECN11112qqakxacTRF8w+uu666y46jr797W+bNOLoe+yxxzRnzhx/Y7OioiK9/PLL/p/H6jEUt2Fkw4YNKi0t1apVq7Rt2zbNnTtXCxcuVG1trdlDGzIuv/xynTx50n976623zB6SaZqamjR37lytXbu2x5//7Gc/07/9279p3bp1eu+995SSkqKFCxeqpaUlyiM1T3/7SJJuvvnmLsfUn//85yiO0FxvvPGGli9frnfffVd//etf1d7erptuuklNTU3+bb773e/qhRde0JNPPqk33nhDJ06c0J133mniqKMrmH0kScuWLetyHP3sZz8zacTRN2HCBD388MOqrKzUBx98oM9//vNatGiR9uzZIymGjyEjTi1YsMBYvny5/3uXy2WMHz/eKCsrM3FUQ8eqVauMuXPnmj2MIUmSsXHjRv/3brfbyMrKMh555BH/fefOnTMcDofx5z//2YQRmq/7PjIMw1i6dKmxaNEiU8YzFNXW1hqSjDfeeMMwDM8xY7fbjSeffNK/zd69ew1JRkVFhVnDNFX3fWQYhnHttdca9913n3mDGoJGjRpl/OY3v4npYyguZ0ba2tpUWVmp4uJi/31Wq1XFxcWqqKgwcWRDyyeffKLx48drypQp+vrXv66qqiqzhzQkHTp0SNXV1V2Op/T0dBUWFnI8dbNlyxaNGzdO06dP1913363Tp0+bPSTT1NfXS5JGjx4tSaqsrFR7e3uX42jGjBmaOHFi3B5H3feRzx//+EdlZGRo1qxZWrFihZqbm80YnulcLpeeeOIJNTU1qaioKKaPoZi4UF641dXVyeVyKTMzs8v9mZmZ2rdvn0mjGloKCwv1u9/9TtOnT9fJkyf14IMP6uqrr9bu3buVmppq9vCGlOrqaknq8Xjy/QyeJZo777xTkydP1sGDB/WDH/xAt9xyiyoqKmSz2cweXlS53W595zvf0Wc/+1nNmjVLkuc4SkxM1MiRI7tsG6/HUU/7SJL+9m//VpMmTdL48eO1c+dO3X///dq/f7+eeeYZE0cbXbt27VJRUZFaWlo0YsQIbdy4UTNnztSOHTti9hiKyzCC/t1yyy3+r+fMmaPCwkJNmjRJ//3f/61vfvObJo4MseqrX/2q/+vZs2drzpw5mjp1qrZs2aIbbrjBxJFF3/Lly7V79+64rsPqT2/76Fvf+pb/69mzZys7O1s33HCDDh48qKlTp0Z7mKaYPn26duzYofr6ej311FNaunSp3njjDbOHNShxuUyTkZEhm812UYVxTU2NsrKyTBrV0DZy5EhdeumlOnDggNlDGXJ8xwzHU2imTJmijIyMuDum7r33Xv3lL3/R5s2bNWHCBP/9WVlZamtr07lz57psH4/HUW/7qCeFhYWSFFfHUWJioqZNm6aCggKVlZVp7ty5+sUvfhHTx1BchpHExEQVFBSovLzcf5/b7VZ5ebmKiopMHNnQdf78eR08eFDZ2dlmD2XImTx5srKysrocTw0NDXrvvfc4nvpw7NgxnT59Om6OKcMwdO+992rjxo16/fXXNXny5C4/LygokN1u73Ic7d+/X1VVVXFzHPW3j3qyY8cOSYqb46gnbrdbra2tsX0MmV1Ba5YnnnjCcDgcxu9+9zvjo48+Mr71rW8ZI0eONKqrq80e2pDwj//4j8aWLVuMQ4cOGW+//bZRXFxsZGRkGLW1tWYPzRSNjY3G9u3bje3btxuSjNWrVxvbt283jhw5YhiGYTz88MPGyJEjjeeee87YuXOnsWjRImPy5MnGhQsXTB559PS1jxobG43vfe97RkVFhXHo0CHjtddeM+bNm2dccsklRktLi9lDj4q7777bSE9PN7Zs2WKcPHnSf2tubvZv8+1vf9uYOHGi8frrrxsffPCBUVRUZBQVFZk46ujqbx8dOHDAeOihh4wPPvjAOHTokPHcc88ZU6ZMMa655hqTRx49DzzwgPHGG28Yhw4dMnbu3Gk88MADhsViMV599VXDMGL3GIrbMGIYhvHLX/7SmDhxopGYmGgsWLDAePfdd80e0pCxePFiIzs720hMTDRycnKMxYsXGwcOHDB7WKbZvHmzIemi29KlSw3D8Jze+6Mf/cjIzMw0HA6HccMNNxj79+83d9BR1tc+am5uNm666SZj7Nixht1uNyZNmmQsW7YsrsJ/T/tGkvHb3/7Wv82FCxeMe+65xxg1apThdDqNO+64wzh58qR5g46y/vZRVVWVcc011xijR482HA6HMW3aNOP73/++UV9fb+7Ao+jv//7vjUmTJhmJiYnG2LFjjRtuuMEfRAwjdo8hi2EYRvTmYQAAALqKy5oRAAAwdBBGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGCq/wvZJJe+pwoaYgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(durations_s)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af60d44b",
   "metadata": {},
   "source": [
    "## Continuous Batching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9477d603",
   "metadata": {
    "height": 165
   },
   "outputs": [],
   "source": [
    "color_codes = [\n",
    "    \"31\",  # red\n",
    "    \"32\",  # green\n",
    "    \"34\",  # blue\n",
    "]\n",
    "\n",
    "\n",
    "def format_text(text, i):\n",
    "    return f\"\\x1b[{color_codes[i]}m{text}\\x1b[0m\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a9cdf83c",
   "metadata": {
    "height": 398
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\n",
      "\u001b[0m\u001b[32m\n",
      "\u001b[0m\u001b[32mDeep\u001b[0m\u001b[32m learning\u001b[0m\u001b[32m is\u001b[0m\u001b[32m a\u001b[0m\u001b[32m subset\u001b[0m\u001b[32m of\u001b[0m\u001b[32m machine\u001b[0m\u001b[32m learning\u001b[0m\u001b[34m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[34m\n",
      "\u001b[0m\u001b[31mDeep\u001b[0m\u001b[31m learning\u001b[0m\u001b[34mDeep\u001b[0m\u001b[34m learning\u001b[0m\u001b[31m is\u001b[0m\u001b[31m a\u001b[0m\u001b[31m subset\u001b[0m\u001b[31m of\u001b[0m\u001b[31m machine\u001b[0m\u001b[34m is\u001b[0m\u001b[34m a\u001b[0m\u001b[34m subset\u001b[0m\u001b[34m of\u001b[0m\u001b[34m machine\u001b[0m\u001b[31m learning\u001b[0m\u001b[31m that\u001b[0m\u001b[31m uses\u001b[0m\u001b[34m learning\u001b[0m\u001b[31m artificial\u001b[0m\u001b[31m neural\u001b[0m\u001b[31m networks\u001b[0m\u001b[31m to\u001b[0m\u001b[31m learn\u001b[0m\u001b[31m from\u001b[0m\u001b[31m data\u001b[0m\u001b[31m.\u001b[0m\u001b[31m It\u001b[0m\u001b[31m is\u001b[0m\u001b[31m a\u001b[0m\u001b[31m powerful\u001b[0m\u001b[31m tool\u001b[0m\u001b[31m that\u001b[0m\u001b[31m can\u001b[0m\u001b[31m be\u001b[0m\u001b[31m used\u001b[0m\u001b[31m to\u001b[0m\u001b[31m solve\u001b[0m\u001b[31m complex\u001b[0m\u001b[31m problems\u001b[0m\u001b[31m and\u001b[0m\u001b[31m make\u001b[0m\u001b[31m predictions\u001b[0m\u001b[31m.\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mWhat\u001b[0m\u001b[31m are\u001b[0m\u001b[31m the\u001b[0m\u001b[31m benefits\u001b[0m\u001b[31m of\u001b[0m\u001b[31m deep\u001b[0m\u001b[31m learning\u001b[0m\u001b[31m?\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mDeep\u001b[0m\u001b[31m learning\u001b[0m\u001b[31m has\u001b[0m\u001b[31m many\u001b[0m\u001b[31m benefits\u001b[0m\u001b[31m,\u001b[0m\u001b[31m including\u001b[0m\u001b[31m the\u001b[0m\u001b[31m ability\u001b[0m\u001b[31m to\u001b[0m\u001b[31m:\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mMake\u001b[0m\u001b[31m predictions\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mS\u001b[0m\u001b[31molve\u001b[0m\u001b[31m complex\u001b[0m\u001b[31m problems\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mL\u001b[0m\u001b[31mear\u001b[0m\u001b[31mn\u001b[0m\u001b[31m from\u001b[0m\u001b[31m data\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mIm\u001b[0m\u001b[31mpro\u001b[0m\u001b[31mve\u001b[0m\u001b[31m accuracy\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mRed\u001b[0m\u001b[31muce\u001b[0m\u001b[31m bias\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31m-\u001b[0m\u001b[31mIn\u001b[0m\u001b[31mcre\u001b[0m\u001b[31mase\u001b[0m\u001b[31m efficiency\u001b[0m\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): [0.6526806354522705, 0.45714545249938965, 0.6489715576171875]\n",
      "Throughput (tok / s): [70.77113136533289, 114.47093250689123, 62.925362187175466]\n",
      "Total duration (s): 2.094090700149536\n"
     ]
    }
   ],
   "source": [
    "async_client = AsyncClient(endpoint_url, headers=headers)\n",
    "\n",
    "\n",
    "durations_s = [[], [], []]\n",
    "\n",
    "\n",
    "async def run(max_new_tokens, i):\n",
    "    t0 = time.time()\n",
    "    async for resp in async_client.generate_stream(\"What is deep learning?\", max_new_tokens=max_new_tokens):\n",
    "        durations_s[i].append(time.time() - t0)\n",
    "        print(format_text(resp.token.text, i), sep=\"\", end=\"\", flush=True)\n",
    "        t0 = time.time()\n",
    "\n",
    "\n",
    "t0 = time.time()\n",
    "all_max_new_tokens = [100, 10, 10]\n",
    "await asyncio.gather(*[run(max_new_tokens, i) for i, max_new_tokens in enumerate(all_max_new_tokens)]) \n",
    "\n",
    "print(\"\\n\\n\\n----------\")\n",
    "print(\"Time to first token (TTFT) (s):\", [s[0] for s in durations_s])\n",
    "print(\"Throughput (tok / s):\", [(len(s) - 1) / sum(s[1:]) for s in durations_s])\n",
    "print(\"Total duration (s):\", time.time() - t0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e2f37bf",
   "metadata": {},
   "source": [
    "## Multi-LoRA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f09a9304",
   "metadata": {
    "height": 330
   },
   "outputs": [],
   "source": [
    "def run_with_adapter(prompt, adapter_id):\n",
    "    durations_s = []\n",
    "\n",
    "    t0 = time.time()\n",
    "    for resp in client.generate_stream(\n",
    "        prompt, \n",
    "        adapter_id=adapter_id,\n",
    "        adapter_source=\"hub\",\n",
    "        max_new_tokens=64,\n",
    "    ):\n",
    "        durations_s.append(time.time() - t0)\n",
    "        if not resp.token.special:\n",
    "            print(resp.token.text, sep=\"\", end=\"\", flush=True)\n",
    "        t0 = time.time()\n",
    "\n",
    "    print(\"\\n\\n\\n----------\")\n",
    "    print(\"Time to first token (TTFT) (s):\", durations_s[0])\n",
    "    print(\"Throughput (tok / s):\", (len(durations_s) - 1) / sum(durations_s[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e191cbf",
   "metadata": {
    "height": 313
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " are running around the field.\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.5081028938293457\n",
      "Throughput (tok / s): 44.631175713206424\n"
     ]
    }
   ],
   "source": [
    "pt_hellaswag_processed = \\\n",
    "\"\"\"You are provided with an incomplete passage below. Please read the passage and then finish it with an appropriate response. For example:\n",
    "\n",
    "### Passage: My friend and I think alike. We\n",
    "\n",
    "### Ending: often finish each other's sentences.\n",
    "\n",
    "Now please finish the following passage:\n",
    "\n",
    "### Passage: {ctx}\n",
    "\n",
    "### Ending: \"\"\"\n",
    "\n",
    "ctx = \"Numerous people are watching others on a field. Trainers are playing frisbee with their dogs. the dogs\"\n",
    "\n",
    "\n",
    "run_with_adapter(pt_hellaswag_processed.format(ctx=ctx), adapter_id=\"predibase/hellaswag_processed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bb0ad965",
   "metadata": {
    "height": 211
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Mondale was admitted to the Mayo Clinic on Friday . He was diagnosed with influenza after a routine checkup . He is expected to return home soon .\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.4702792167663574\n",
      "Throughput (tok / s): 59.360910371089105\n"
     ]
    }
   ],
   "source": [
    "pt_cnn = \\\n",
    "\"\"\"You are given a news article below. Please summarize the article, including only its highlights.\n",
    "\n",
    "### Article: {article}\n",
    "\n",
    "### Summary: \"\"\"\n",
    "\n",
    "article = \"(CNN)Former Vice President Walter Mondale was released from the Mayo Clinic on Saturday after being admitted with influenza, hospital spokeswoman Kelley Luckstein said. \\\"He's doing well. We treated him for flu and cold symptoms and he was released today,\\\" she said. Mondale, 87, was diagnosed after he went to the hospital for a routine checkup following a fever, former President Jimmy Carter said Friday. \\\"He is in the bed right this moment, but looking forward to come back home,\\\" Carter said during a speech at a Nobel Peace Prize Forum in Minneapolis. \\\"He said tell everybody he is doing well.\\\" Mondale underwent treatment at the Mayo Clinic in Rochester, Minnesota. The 42nd vice president served under Carter between 1977 and 1981, and later ran for President, but lost to Ronald Reagan. But not before he made history by naming a woman, U.S. Rep. Geraldine A. Ferraro of New York, as his running mate. Before that, the former lawyer was  a U.S. senator from Minnesota. His wife, Joan Mondale, died last year.\"\n",
    "\n",
    "\n",
    "run_with_adapter(pt_cnn.format(article=article), adapter_id=\"predibase/cnn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c908562c",
   "metadata": {
    "height": 313
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " {\"person\": [\"Fischler\"], \"organization\": [], \"location\": [\"France\", \"Britain\"], \"miscellaneous\": []}\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): 0.3575615882873535\n",
      "Throughput (tok / s): 60.23366582867029\n"
     ]
    }
   ],
   "source": [
    "pt_conllpp = \"\"\"\n",
    "Your task is a Named Entity Recognition (NER) task. Predict the category of\n",
    "each entity, then place the entity into the list associated with the \n",
    "category in an output JSON payload. Below is an example:\n",
    "\n",
    "Input: EU rejects German call to boycott British lamb . Output: {{\"person\":\n",
    "[], \"organization\": [\"EU\"], \"location\": [], \"miscellaneous\": [\"German\",\n",
    "\"British\"]}}\n",
    "\n",
    "Now, complete the task.\n",
    "\n",
    "Input: {inpt} Output:\"\"\"\n",
    "\n",
    "inpt = \"Only France and Britain backed Fischler 's proposal .\"\n",
    "\n",
    "\n",
    "run_with_adapter(pt_conllpp.format(inpt=inpt), adapter_id=\"predibase/conllpp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e4cc6add",
   "metadata": {
    "height": 551
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m are\u001b[0m\u001b[31m running\u001b[0m\u001b[31m around\u001b[0m\u001b[32m Mond\u001b[0m\u001b[32male\u001b[0m\u001b[32m was\u001b[0m\u001b[32m admitted\u001b[0m\u001b[31m the\u001b[0m\u001b[31m field\u001b[0m\u001b[31m.\u001b[0m\u001b[32m to\u001b[0m\u001b[32m the\u001b[0m\u001b[32m May\u001b[0m\u001b[32mo\u001b[0m\u001b[32m Clin\u001b[0m\u001b[32mic\u001b[0m\u001b[32m on\u001b[0m\u001b[34m {\"\u001b[0m\u001b[34mperson\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m [\"\u001b[0m\u001b[34mF\u001b[0m\u001b[34misch\u001b[0m\u001b[32m Friday\u001b[0m\u001b[32m .\u001b[0m\u001b[32m He\u001b[0m\u001b[32m was\u001b[0m\u001b[34mler\u001b[0m\u001b[34m\"],\u001b[0m\u001b[34m \"\u001b[0m\u001b[34morgan\u001b[0m\u001b[34mization\u001b[0m\u001b[32m diagnosed\u001b[0m\u001b[32m with\u001b[0m\u001b[32m influ\u001b[0m\u001b[32menza\u001b[0m\u001b[32m after\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m [],\u001b[0m\u001b[34m \"\u001b[0m\u001b[34mlocation\u001b[0m\u001b[32m a\u001b[0m\u001b[32m routine\u001b[0m\u001b[32m check\u001b[0m\u001b[32mup\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m [\"\u001b[0m\u001b[34mFr\u001b[0m\u001b[34mance\u001b[0m\u001b[32m .\u001b[0m\u001b[32m He\u001b[0m\u001b[32m is\u001b[0m\u001b[32m expected\u001b[0m\u001b[32m to\u001b[0m\u001b[34m\",\u001b[0m\u001b[34m \"\u001b[0m\u001b[34mB\u001b[0m\u001b[34mrit\u001b[0m\u001b[34main\u001b[0m\u001b[32m return\u001b[0m\u001b[32m home\u001b[0m\u001b[32m soon\u001b[0m\u001b[32m .\u001b[0m\u001b[34m\"],\u001b[0m\u001b[34m \"\u001b[0m\u001b[34mm\u001b[0m\u001b[34mis\u001b[0m\u001b[34mcell\u001b[0m\u001b[34maneous\u001b[0m\u001b[34m\":\u001b[0m\u001b[34m []\u001b[0m\u001b[34m}\u001b[0m\n",
      "\n",
      "\n",
      "----------\n",
      "Time to first token (TTFT) (s): [0.34526586532592773, 0.36800146102905273, 0.5030913352966309]\n",
      "Throughput (tok / s): [42.86800915759473, 49.713458222679904, 52.55369230406377]\n",
      "Total duration (s): 1.1703855991363525\n"
     ]
    }
   ],
   "source": [
    "durations_s = [[], [], []]\n",
    "\n",
    "\n",
    "async def run(prompt, adapter_id, i):\n",
    "    t0 = time.time()\n",
    "    async for resp in async_client.generate_stream(\n",
    "        prompt, \n",
    "        adapter_id=adapter_id,\n",
    "        adapter_source=\"hub\",\n",
    "        max_new_tokens=64,\n",
    "    ):\n",
    "        durations_s[i].append(time.time() - t0)\n",
    "        if not resp.token.special:\n",
    "            print(format_text(resp.token.text, i), sep=\"\", end=\"\", flush=True)\n",
    "        t0 = time.time()\n",
    "\n",
    "\n",
    "t0 = time.time()\n",
    "prompts = [\n",
    "    pt_hellaswag_processed.format(ctx=ctx),\n",
    "    pt_cnn.format(article=article),\n",
    "    pt_conllpp.format(inpt=inpt),\n",
    "]\n",
    "adapter_ids = [\"predibase/hellaswag_processed\", \"predibase/cnn\", \"predibase/conllpp\"]\n",
    "await asyncio.gather(*[run(prompt, adapter_id, i) \n",
    "                       for i, (prompt, adapter_id) in enumerate(zip(prompts, adapter_ids))]) \n",
    "\n",
    "print(\"\\n\\n\\n----------\")\n",
    "print(\"Time to first token (TTFT) (s):\", [s[0] for s in durations_s])\n",
    "print(\"Throughput (tok / s):\", [(len(s) - 1) / sum(s[1:]) for s in durations_s])\n",
    "print(\"Total duration (s):\", time.time() - t0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed936723",
   "metadata": {},
   "source": [
    "## Bonus: Structured Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "00431323",
   "metadata": {
    "height": 148
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Person',\n",
       " 'type': 'object',\n",
       " 'properties': {'name': {'title': 'Name', 'maxLength': 10, 'type': 'string'},\n",
       "  'age': {'title': 'Age', 'type': 'integer'}},\n",
       " 'required': ['name', 'age']}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pydantic import BaseModel, constr\n",
    "\n",
    "class Person(BaseModel):\n",
    "    name: constr(max_length=10)\n",
    "    age: int\n",
    "\n",
    "schema = Person.schema()\n",
    "schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "08b706aa",
   "metadata": {
    "height": 97
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'John Doe', 'age': 30}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = client.generate(\n",
    "    \"Create a person description for me\", \n",
    "    response_format={\"type\": \"json_object\", \"schema\": schema}\n",
    ")\n",
    "json.loads(resp.generated_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "15870540",
   "metadata": {
    "height": 347
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"person\":\\n[\"Fischler\"], \"organization\": [\"France\", \"Britain\"], \"location\": [],\\n\"miscellaneous\": []}\\n\\nInput: The EU has agreed to a 10-year extension of the ban on the use of\\nhormones in beef . Output: {\"person\": [], \"organization\": [\"EU\"],\\n\"location\": [], \"miscellaneous\": [\"hormones\", \"beef\"]}\\n\\nInput: The EU has agreed to a 10-year extension of the ban on the use of\\nhormones in'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt_template = \"\"\"\n",
    "Your task is a Named Entity Recognition (NER) task. Predict the category of\n",
    "each entity, then place the entity into the list associated with the \n",
    "category in an output JSON payload. Below is an example:\n",
    "\n",
    "Input: EU rejects German call to boycott British lamb . Output: {{\"person\":\n",
    "[], \"organization\": [\"EU\"], \"location\": [], \"miscellaneous\": [\"German\",\n",
    "\"British\"]}}\n",
    "\n",
    "Now, complete the task.\n",
    "\n",
    "Input: {input} Output:\"\"\"\n",
    "\n",
    "# Base Mistral-7B\n",
    "resp = client.generate(\n",
    "    prompt_template.format(input=\"Only France and Britain backed Fischler 's proposal .\"),  \n",
    "    max_new_tokens=128,\n",
    ")\n",
    "resp.generated_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "65923d86",
   "metadata": {
    "height": 182
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': 'Output',\n",
       " 'type': 'object',\n",
       " 'properties': {'person': {'title': 'Person',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}},\n",
       "  'organization': {'title': 'Organization',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}},\n",
       "  'location': {'title': 'Location',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}},\n",
       "  'miscellaneous': {'title': 'Miscellaneous',\n",
       "   'type': 'array',\n",
       "   'items': {'type': 'string'}}},\n",
       " 'required': ['person', 'organization', 'location', 'miscellaneous']}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from typing import List\n",
    "\n",
    "class Output(BaseModel):\n",
    "    person: List[str]\n",
    "    organization: List[str]\n",
    "    location: List[str]\n",
    "    miscellaneous: List[str]\n",
    "\n",
    "schema = Output.schema()\n",
    "schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a29a1eb2",
   "metadata": {
    "height": 177
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'person': ['Fischler'],\n",
       " 'organization': ['France', 'Britain'],\n",
       " 'location': [],\n",
       " 'miscellaneous': []}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = client.generate(\n",
    "    prompt_template.format(input=\"Only France and Britain backed Fischler 's proposal .\"),\n",
    "    response_format={\n",
    "        \"type\": \"json_object\",\n",
    "        \"schema\": schema,\n",
    "    },\n",
    "    max_new_tokens=128,\n",
    ")\n",
    "json.loads(resp.generated_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4c56fca8",
   "metadata": {
    "height": 143
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'person': ['Fischler'],\n",
       " 'organization': [],\n",
       " 'location': ['France', 'Britain'],\n",
       " 'miscellaneous': []}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = client.generate(\n",
    "    prompt_template.format(input=\"Only France and Britain backed Fischler 's proposal .\"),\n",
    "    adapter_id=\"predibase/conllpp\",\n",
    "    adapter_source=\"hub\",\n",
    "    max_new_tokens=128,\n",
    ")\n",
    "json.loads(resp.generated_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee30357c",
   "metadata": {},
   "source": [
    "## Additional Predibase Resources: \n",
    "\n",
    "- LoRA Land demo environment: https://predibase.com/lora-land \n",
    "- Predibase blog articles: https://predibase.com/blog \n",
    "- LoRAX Github repo: https://github.com/predibase/lorax \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed8382a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}