examples/openai_vs_vertex/vertex_rag_vs_openai_rag.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "from openai import OpenAI\n",
    "from tonic_validate import ValidateScorer, Benchmark, BenchmarkItem, LLMResponse, BenchmarkItem, Run\n",
    "from tonic_validate.metrics import AnswerSimilarityMetric\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "with open(\"qa_pairs.json\", \"r\") as f:\n",
    "    qa_pairs = json.load(f)\n",
    "\n",
    "# for testing\n",
    "qa_pairs = qa_pairs\n",
    "\n",
    "benchmark = Benchmark(\n",
    "    questions=[x[\"question\"] for x in qa_pairs],\n",
    "    answers=[x[\"answer\"] for x in qa_pairs]\n",
    ")\n",
    "\n",
    "def run_to_dataframe(run: Run) -> pd.DataFrame:\n",
    "    return pd.DataFrame(\n",
    "        {\n",
    "            \"reference_question\": [x.reference_question for x in run.run_data],\n",
    "            \"reference_answer\": [x.reference_answer for x in run.run_data],\n",
    "            \"llm_answer\": [x.llm_answer for x in run.run_data],\n",
    "            \"llm_context\": [json.dumps(x.llm_context) for x in run.run_data],\n",
    "            \"answer_similarity\": [x.scores[\"answer_similarity\"] for x in run.run_data]\n",
    "        }\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vertex Conversation and Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.api_core.client_options import ClientOptions\n",
    "from google.cloud import discoveryengine_v1 as discoveryengine\n",
    "\n",
    "\n",
    "# The 3 variables below need to be obtained from your GCP account.\n",
    "# The data_store_id is the data store created in Vertex AI Search and Conversation.\n",
    "# To load data simply upload the ~200 essays in examples/titan_vs_cohere/paul_graham_essays into a GCS bucket and reference the bucket from your data store. \n",
    "project_id=''\n",
    "data_store_id=''\n",
    "location='global'\n",
    "\n",
    "def get_vertex_response(benchMarkItem: BenchmarkItem) -> List[discoveryengine.SearchResponse]:\n",
    "    \n",
    "    client_options = (\n",
    "        ClientOptions(api_endpoint=f\"{location}-discoveryengine.googleapis.com\")\n",
    "        if location != \"global\"\n",
    "        else None\n",
    "    )\n",
    "\n",
    "    client = discoveryengine.SearchServiceClient(client_options=client_options)\n",
    "\n",
    "    serving_config = client.serving_config_path(\n",
    "        project=project_id,\n",
    "        location=location,\n",
    "        data_store=data_store_id,\n",
    "        serving_config=\"default_config\",\n",
    "    )\n",
    "\n",
    "    content_search_spec = discoveryengine.SearchRequest.ContentSearchSpec(\n",
    "        snippet_spec=discoveryengine.SearchRequest.ContentSearchSpec.SnippetSpec(\n",
    "            return_snippet=True\n",
    "        ),\n",
    "        summary_spec=discoveryengine.SearchRequest.ContentSearchSpec.SummarySpec(\n",
    "            summary_result_count=5,\n",
    "            include_citations=True,\n",
    "            ignore_adversarial_query=True,\n",
    "            ignore_non_summary_seeking_query=True,\n",
    "        ),\n",
    "    )\n",
    "    \n",
    "    request = discoveryengine.SearchRequest(\n",
    "        serving_config=serving_config,\n",
    "        query=benchMarkItem.question,\n",
    "        page_size=10,\n",
    "        content_search_spec=content_search_spec,\n",
    "        query_expansion_spec=discoveryengine.SearchRequest.QueryExpansionSpec(\n",
    "            condition=discoveryengine.SearchRequest.QueryExpansionSpec.Condition.AUTO,\n",
    "        ),\n",
    "        spell_correction_spec=discoveryengine.SearchRequest.SpellCorrectionSpec(\n",
    "            mode=discoveryengine.SearchRequest.SpellCorrectionSpec.Mode.AUTO\n",
    "        ),\n",
    "    )\n",
    "\n",
    "    response = client.search(request)\n",
    "    return response.summary.summary_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "According to Paul Graham, you only need two kinds of people to create a technology hub: rich people and nerds [1]. To create a startup hub, you need to establish a first-rate research university in a place so nice that rich people wanted to live there [4]. The town would be hospitable to both groups you need: both founders and investors [4].\n"
     ]
    }
   ],
   "source": [
    "benchmark_item = BenchmarkItem(\n",
    "    question=\"What key components are necessary to create a technology hub according to Paul Graham?\",\n",
    "    answer=\"\"\n",
    ")\n",
    "\n",
    "print(get_vertex_response(benchmark_item))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Run through all the questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 55/55 [00:24<00:00,  2.24it/s]\n"
     ]
    }
   ],
   "source": [
    "with ThreadPoolExecutor(max_workers=10) as executor:\n",
    "    raw_vertex_responses = list(tqdm(executor.map(get_vertex_response, benchmark.items), total=len(benchmark.items)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "vertex_responses = [\n",
    "    LLMResponse(\n",
    "        llm_answer=r, llm_context_list=[], benchmark_item=bi\n",
    "    ) for r, bi in zip(raw_vertex_responses, benchmark.items)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "scorer = ValidateScorer([AnswerSimilarityMetric()])\n",
    "vertex_run = scorer.score_run(vertex_responses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "vertex_run_df = run_to_dataframe(vertex_run)\n",
    "vertex_run_df.to_csv(\"vertex_run.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAHHCAYAAABKudlQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0pElEQVR4nO3deViVdf7/8dcR5KCAuIMkW7ikGJq5TLlSGJFbpZP6VUNLbSaXjHKSHPdSm6WoNLX5TtLkuJSm+dXUDLesLJdhXDJTUyMXsBpBcESD+/dHl+fXCVBA5D4fez6u676u7s+9vc99zrEXn/tz38dhWZYlAAAAQ1WxuwAAAIBrQZgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmMGvxpQpU+RwOCrlWF27dlXXrl1d85s3b5bD4dCyZcsq5fhDhgxRREREpRyrvHJzczVs2DAFBwfL4XBo7NixdpfkETIzM9W3b1/VqVNHDodDKSkpdpcEeDzCDIyUmpoqh8Phmnx9fRUSEqL4+Hi98sorOnfuXIUc5+TJk5oyZYrS09MrZH8VyZNrK40ZM2YoNTVVv//97/XWW29p8ODBdpfkEZ588kmtX79eycnJeuutt3Tvvfdel+O8++67cjgc+t///d8S19mwYYMcDodeeeWVCjnma6+9ptTU1ArZF/BzDn6bCSZKTU3V0KFDNW3aNEVGRurSpUs6ffq0Nm/erA0bNigsLEyrVq1STEyMa5sff/xRP/74o3x9fUt9nJ07d6pt27ZasGCBhgwZUurtLl68KEny8fGR9FPPTGxsrN555x317du31Pspb22XLl1SYWGhnE5nhRzrevjNb34jb29vbdu2ze5SPEpwcLDi4uK0cOHC63qc/Px8BQUFqXXr1tq4cWOx6wwdOlRvvfWWTp48qfr161/zMVu0aKG6detq8+bN17wv4Oe87S4AuBYJCQlq06aNaz45OVkbN25Ujx491KtXLx04cEDVqlWTJHl7e8vb+/p+5M+fP6/q1au7QoxdqlatauvxSyMrK0vNmze3u4xSKSws1MWLF8sUhMsrKytLNWvWrLD9XbhwQT4+PqpSxb0j3ul0qm/fvlqwYIFOnjypkJCQItutWLFC3bp1u+Ygc/l7AVwvXGbCDeeuu+7SxIkTdfz4cbe/bosbM7NhwwZ17NhRNWvWlL+/v5o2bapnn31W0k+9KW3btpX001+oly9pXe4m79q1q1q0aKFdu3apc+fOql69umvbX46ZuaygoEDPPvusgoOD5efnp169eikjI8NtnYiIiGJ7gX6+z6vVVtyYmby8PD311FMKDQ2V0+lU06ZN9Ze//EW/7Jx1OBwaNWqUVq5cqRYtWsjpdCo6Olrr1q0r/oT/QlZWlh599FEFBQXJ19dXLVu21Jtvvulafnn80NGjR7VmzRpX7ceOHStxn1d6ny67cOGCpkyZoiZNmsjX11cNGjTQgw8+qCNHjpT7HPzzn/9UdHS0nE6n6/WfOHFCjzzyiIKCglzn5o033ihS86uvvqro6GhVr15dtWrVUps2bbRo0aISX+PlS6eWZWnOnDmu83LZ119/rd/+9reqXbu2qlevrt/85jdas2aN2z4un9slS5boj3/8o2666SZVr15dOTk5xR5z0KBBKiws1JIlS4osW7NmjbKzszVw4EBX28KFC3X77berWrVqql27tvr371/k81vS9yIiIkL79+/Xli1bXK/t59+Rs2fPauzYsa73plGjRnrhhRdUWFgoSbIsS7GxsapXr56ysrJc2128eFG33nqroqKilJeXV+L5xY2NnhnckAYPHqxnn31WH3zwgYYPH17sOvv371ePHj0UExOjadOmyel06vDhw/r4448lSc2aNdO0adM0adIkjRgxQp06dZIk3Xnnna59fP/990pISFD//v01aNAgBQUFXbGu559/Xg6HQ88884yysrKUkpKiuLg4paenu3qQSqM0tf2cZVnq1auXNm3apEcffVStWrXS+vXrNW7cOJ04cUIvvfSS2/rbtm3Tu+++q8cff1wBAQF65ZVX1KdPH33zzTeqU6dOiXX997//VdeuXXX48GGNGjVKkZGReueddzRkyBCdPXtWTzzxhJo1a6a33npLTz75pBo2bKinnnpKklSvXr1i93m190n6KST26NFDaWlp6t+/v5544gmdO3dOGzZs0L59+xQVFVXmc7Bx40a9/fbbGjVqlOrWrauIiAhlZmbqN7/5jSvs1KtXT2vXrtWjjz6qnJwc1yDmv/3tbxozZoz69u2rJ554QhcuXNCePXv02Wef6X/+53+KfZ2dO3d2jR3q1q2bHn74YdeyzMxM3XnnnTp//rzGjBmjOnXq6M0331SvXr20bNkyPfDAA277mj59unx8fPT0008rPz+/xJ7Czp07q2HDhlq0aJGSkpLcli1atEjVq1fX/fffL+mnz+7EiRP10EMPadiwYTpz5oxeffVVde7cWf/617/cepOK+1507dpVo0ePlr+/vyZMmCBJru/L+fPn1aVLF504cUKPPfaYwsLC9Mknnyg5OVmnTp1SSkqKHA6H3njjDcXExOh3v/ud3n33XUnS5MmTtX//fm3evFl+fn7Fvk78CliAgRYsWGBJsnbs2FHiOoGBgdZtt93mmp88ebL184/8Sy+9ZEmyzpw5U+I+duzYYUmyFixYUGRZly5dLEnWvHnzil3WpUsX1/ymTZssSdZNN91k5eTkuNrffvttS5L18ssvu9rCw8OtxMTEq+7zSrUlJiZa4eHhrvmVK1dakqznnnvObb2+fftaDofDOnz4sKtNkuXj4+PW9u9//9uSZL366qtFjvVzKSkpliRr4cKFrraLFy9ad9xxh+Xv7+/22sPDw63u3btfcX+WVbr36Y033rAkWS+++GKRZYWFhZZllf0cVKlSxdq/f7/buo8++qjVoEED67vvvnNr79+/vxUYGGidP3/esizL6t27txUdHX3V11YcSdbIkSPd2saOHWtJsj766CNX27lz56zIyEgrIiLCKigosCzr/3/Obr75ZlctVzNu3DhLknXw4EFXW3Z2tuXr62sNGDDAsizLOnbsmOXl5WU9//zzbtvu3bvX8vb2dmu/0vciOjra7TN82fTp0y0/Pz/rq6++cmsfP3685eXlZX3zzTeutvnz57s+Y9u3b7e8vLyssWPHluq14sbFZSbcsPz9/a94V9PlvyTfe+89V1d2WTmdTg0dOrTU6z/88MMKCAhwzfft21cNGjTQ+++/X67jl9b7778vLy8vjRkzxq39qaeekmVZWrt2rVt7XFycoqKiXPMxMTGqUaOGvv7666seJzg4WAMGDHC1Va1aVWPGjFFubq62bNlS5tpL8z4tX75cdevW1ejRo4ssu3yppqznoEuXLm5jeizL0vLly9WzZ09ZlqXvvvvONcXHxys7O1u7d+921fztt99qx44dZX69xXn//ffVrl07dezY0dXm7++vESNG6NixY/riiy/c1k9MTCx1T9+gQYMkye0S2PLly3XhwgXXJaZ3331XhYWFeuihh9xed3BwsBo3bqxNmza57bOs34t33nlHnTp1Uq1atdz2HxcXp4KCAm3dutW17ogRIxQfH6/Ro0dr8ODBioqK0owZM0p9LNyYCDO4YeXm5roFh1/q16+fOnTooGHDhikoKEj9+/fX22+/XaZgc9NNN5VpsG/jxo3d5h0Ohxo1anTF8SIV4fjx4woJCSlyPpo1a+Za/nNhYWFF9lGrVi395z//uepxGjduXGSwaUnHKY3SvE9HjhxR06ZNrzjAu6znIDIy0m3+zJkzOnv2rF5//XXVq1fPbbr8P+7LYzmeeeYZ+fv7q127dmrcuLFGjhzpdlmsrI4fP66mTZsWaS9t7VcSExOjFi1aaPHixa62RYsWqW7duoqPj5ckHTp0SJZlqXHjxkVe+4EDB9zGsEhl/14cOnRI69atK7LvuLg4SSqy/7///e86f/68Dh06pNTU1DJdosWNiTEzuCF9++23ys7OVqNGjUpcp1q1atq6das2bdqkNWvWaN26dVq6dKnuuusuffDBB/Ly8rrqca7HP6IlPdivoKCgVDVVhJKOY9nwJIeKeJ/Ke9yfuxyeBg0apMTExGK3ufwogGbNmungwYNavXq11q1bp+XLl+u1117TpEmTNHXq1OtS78+V9XM5aNAgjR8/Xjt37lTDhg21adMmPfbYY65wWFhYKIfDobVr1xZ7vv39/a/p+IWFherWrZv+8Ic/FLu8SZMmbvObN29Wfn6+JGnv3r264447ynQ83HgIM7ghvfXWW5Lk+suyJFWqVNHdd9+tu+++Wy+++KJmzJihCRMmaNOmTYqLi6vwJwYfOnTIbd6yLB0+fNjteTi1atXS2bNni2x7/Phx3Xzzza75stQWHh6uDz/8UOfOnXPrmfjyyy9dyytCeHi49uzZo8LCQrfemWs9ztXep6ioKH322We6dOlSibelX+s5qFevngICAlRQUODqMbgSPz8/9evXT/369dPFixf14IMP6vnnn1dycnKZb/EODw/XwYMHi7RX1Ps3YMAAJScna9GiRQoPD1dBQYHbXUyXB1BHRkYWCRZlUdJnNioqSrm5uaU6r6dOndLo0aN1zz33uAY5x8fHV9hnGGbiMhNuOBs3btT06dMVGRnp9g/yL/3www9F2lq1aiVJrr/6Lt8dUVy4KI9//OMfbuN4li1bplOnTikhIcHVFhUVpe3bt7sevCdJq1evLnILbFlqu++++1RQUKDZs2e7tb/00ktyOBxux78W9913n06fPq2lS5e62n788Ue9+uqr8vf3V5cuXcq8z9K8T3369NF3331X5PVJ/7836VrPgZeXl/r06aPly5dr3759RZafOXPG9d/ff/+92zIfHx81b95clmXp0qVLVzxOce677z59/vnn+vTTT11teXl5ev311xUREXHNz+sJCwtTp06dtHTpUi1cuFCRkZFud8Y9+OCD8vLy0tSpU4v0zlmWVeT1lsTPz6/Yz+tDDz2kTz/9VOvXry+y7OzZs/rxxx9d88OHD1dhYaH+/ve/6/XXX5e3t7ceffRRW3oN4TnomYHR1q5dqy+//FI//vijMjMztXHjRm3YsEHh4eFatWrVFf8CnjZtmrZu3aru3bsrPDxcWVlZeu2119SwYUPXQMuoqCjVrFlT8+bNU0BAgPz8/NS+ffsyjUn4udq1a6tjx44aOnSoMjMzlZKSokaNGrndPj5s2DAtW7ZM9957rx566CEdOXJECxcudBuQW9baevbsqdjYWE2YMEHHjh1Ty5Yt9cEHH+i9997T2LFji+y7vEaMGKH58+dryJAh2rVrlyIiIrRs2TJ9/PHHSklJueIYppKU5n16+OGH9Y9//ENJSUn6/PPP1alTJ+Xl5enDDz/U448/rt69e1fIOZg1a5Y2bdqk9u3ba/jw4WrevLl++OEH7d69Wx9++KEreN1zzz0KDg5Whw4dFBQUpAMHDmj27Nnq3r17uc7B+PHjtXjxYiUkJGjMmDGqXbu23nzzTR09elTLly8vMkapPAYNGqQRI0bo5MmTrlunL4uKitJzzz2n5ORkHTt2TPfff78CAgJ09OhRrVixQiNGjNDTTz991WPcfvvtmjt3rp577jk1atRI9evX11133aVx48Zp1apV6tGjh4YMGaLbb79deXl52rt3r5YtW6Zjx46pbt26WrBggdasWaPU1FQ1bNhQ0k/P8xk0aJDmzp2rxx9//JrPAwxlyz1UwDW6fGv25cnHx8cKDg62unXrZr388stutwBf9stbs9PS0qzevXtbISEhlo+PjxUSEmINGDCgyO2h7733ntW8eXPL29vb7VboLl26lHj7bUm3Zi9evNhKTk626tevb1WrVs3q3r27dfz48SLb//Wvf7Vuuukmy+l0Wh06dLB27txZZJ9Xqu2Xt2Zb1k+38j755JNWSEiIVbVqVatx48bWn//8Z9ety5epmFuDLavkW8Z/KTMz0xo6dKhVt25dy8fHx7r11luLvX28tLdml/Z9On/+vDVhwgQrMjLSqlq1qhUcHGz17dvXOnLkSIWdg8uvb+TIkVZoaKjrOHfffbf1+uuvu9aZP3++1blzZ6tOnTqW0+m0oqKirHHjxlnZ2dlXfb0lHfvIkSNW3759rZo1a1q+vr5Wu3btrNWrV7utc/lz9s4771z1OL/0ww8/WE6n05JkffHFF8Wus3z5cqtjx46Wn5+f5efnZ91yyy3WyJEj3W7rvtL34vTp01b37t2tgIAAS5Lb5/ncuXNWcnKy1ahRI8vHx8eqW7eudeedd1p/+ctfrIsXL1oZGRlWYGCg1bNnzyL7feCBByw/Pz/r66+/LvPrxo2B32YCAABGY8wMAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRbviH5hUWFurkyZMKCAio8EfTAwCA68OyLJ07d04hISFXfTDkDR9mTp48qdDQULvLAAAA5ZCRkeF64nNJbvgwc/nR4RkZGapRo4bN1QAAgNLIyclRaGhoqX4C5IYPM5cvLdWoUYMwAwCAYUozRIQBwAAAwGiEGQAAYDTCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAoxFmAACA0QgzAADAaIQZAABgNMIMAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjedtdAAAAN4LFz35mdwm2GDCjvd0l0DMDAADMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMJqtYWbr1q3q2bOnQkJC5HA4tHLlyiLrHDhwQL169VJgYKD8/PzUtm1bffPNN5VfLAAA8Ei2hpm8vDy1bNlSc+bMKXb5kSNH1LFjR91yyy3avHmz9uzZo4kTJ8rX17eSKwUAAJ7K286DJyQkKCEhocTlEyZM0H333ac//elPrraoqKjKKA0AABjCY8fMFBYWas2aNWrSpIni4+NVv359tW/fvthLUT+Xn5+vnJwctwkAANy4PDbMZGVlKTc3V7NmzdK9996rDz74QA888IAefPBBbdmypcTtZs6cqcDAQNcUGhpaiVUDAIDK5rFhprCwUJLUu3dvPfnkk2rVqpXGjx+vHj16aN68eSVul5ycrOzsbNeUkZFRWSUDAAAb2Dpm5krq1q0rb29vNW/e3K29WbNm2rZtW4nbOZ1OOZ3O610eAADwEB7bM+Pj46O2bdvq4MGDbu1fffWVwsPDbaoKAAB4Glt7ZnJzc3X48GHX/NGjR5Wenq7atWsrLCxM48aNU79+/dS5c2fFxsZq3bp1+r//+z9t3rzZvqIBAIBHsTXM7Ny5U7Gxsa75pKQkSVJiYqJSU1P1wAMPaN68eZo5c6bGjBmjpk2bavny5erYsaNdJQMAAA9ja5jp2rWrLMu64jqPPPKIHnnkkUqqCAAAmMZjx8wAAACUBmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNG87S4AAOB5Fj/7md0l2GLAjPZ2l4ByoGcGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMJqtYWbr1q3q2bOnQkJC5HA4tHLlyhLX/d3vfieHw6GUlJRKqw8AAHg+W8NMXl6eWrZsqTlz5lxxvRUrVmj79u0KCQmppMoAAIApvO08eEJCghISEq64zokTJzR69GitX79e3bt3r6TKAACAKWwNM1dTWFiowYMHa9y4cYqOji7VNvn5+crPz3fN5+TkXK/yAACAB/DoAcAvvPCCvL29NWbMmFJvM3PmTAUGBrqm0NDQ61ghAACwm8eGmV27dunll19WamqqHA5HqbdLTk5Wdna2a8rIyLiOVQIAALt5bJj56KOPlJWVpbCwMHl7e8vb21vHjx/XU089pYiIiBK3czqdqlGjhtsEAABuXB47Zmbw4MGKi4tza4uPj9fgwYM1dOhQm6oCAACextYwk5ubq8OHD7vmjx49qvT0dNWuXVthYWGqU6eO2/pVq1ZVcHCwmjZtWtmlAgAAD2VrmNm5c6diY2Nd80lJSZKkxMREpaam2lQVAAAwia1hpmvXrrIsq9TrHzt27PoVAwAAjOSxA4ABAABKgzADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGjedhcAoHQWP/uZ3SXYYsCM9naXAMDD0TMDAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMJqtYWbr1q3q2bOnQkJC5HA4tHLlSteyS5cu6ZlnntGtt94qPz8/hYSE6OGHH9bJkyftKxgAAHgcW8NMXl6eWrZsqTlz5hRZdv78ee3evVsTJ07U7t279e677+rgwYPq1auXDZUCAABP5W3nwRMSEpSQkFDsssDAQG3YsMGtbfbs2WrXrp2++eYbhYWFVUaJAADAw9kaZsoqOztbDodDNWvWLHGd/Px85efnu+ZzcnIqoTIAAGAXYwYAX7hwQc8884wGDBigGjVqlLjezJkzFRgY6JpCQ0MrsUoAAFDZjAgzly5d0kMPPSTLsjR37twrrpucnKzs7GzXlJGRUUlVAgAAO3j8ZabLQeb48ePauHHjFXtlJMnpdMrpdFZSdQAAwG4eHWYuB5lDhw5p06ZNqlOnjt0lAQAAD2NrmMnNzdXhw4dd80ePHlV6erpq166tBg0aqG/fvtq9e7dWr16tgoICnT59WpJUu3Zt+fj42FU2AADwILaGmZ07dyo2NtY1n5SUJElKTEzUlClTtGrVKklSq1at3LbbtGmTunbtWlllAgAAD2ZrmOnatassyypx+ZWWAQAASIbczQQAAFASwgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAoxFmAACA0QgzAADAaIQZAABgNMIMAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKPZGma2bt2qnj17KiQkRA6HQytXrnRbblmWJk2apAYNGqhatWqKi4vToUOH7CkWAAB4JFvDTF5enlq2bKk5c+YUu/xPf/qTXnnlFc2bN0+fffaZ/Pz8FB8frwsXLlRypQAAwFN523nwhIQEJSQkFLvMsiylpKToj3/8o3r37i1J+sc//qGgoCCtXLlS/fv3r8xSAQCAh/LYMTNHjx7V6dOnFRcX52oLDAxU+/bt9emnn9pYGQAA8CS29sxcyenTpyVJQUFBbu1BQUGuZcXJz89Xfn6+az4nJ+f6FAgAADyCx/bMlNfMmTMVGBjomkJDQ+0uCQAAXEceG2aCg4MlSZmZmW7tmZmZrmXFSU5OVnZ2tmvKyMi4rnUCAAB7eWyYiYyMVHBwsNLS0lxtOTk5+uyzz3THHXeUuJ3T6VSNGjXcJgAAcOOydcxMbm6uDh8+7Jo/evSo0tPTVbt2bYWFhWns2LF67rnn1LhxY0VGRmrixIkKCQnR/fffb1/RAADAo5QrzNx8883asWOH6tSp49Z+9uxZtW7dWl9//XWp9rNz507Fxsa65pOSkiRJiYmJSk1N1R/+8Afl5eVpxIgROnv2rDp27Kh169bJ19e3PGUDAIAbULnCzLFjx1RQUFCkPT8/XydOnCj1frp27SrLskpc7nA4NG3aNE2bNq08ZQIAgF+BMoWZVatWuf57/fr1CgwMdM0XFBQoLS1NERERFVYcAADA1ZQpzFweq+JwOJSYmOi2rGrVqoqIiNBf//rXCisOAADgasoUZgoLCyX9dKfRjh07VLdu3etSFAAAQGmVa8zM0aNHK7oOAACAcin3rdlpaWlKS0tTVlaWq8fmsjfeeOOaCwMAACiNcoWZqVOnatq0aWrTpo0aNGggh8NR0XUBAACUSrnCzLx585SamqrBgwdXdD0AAABlUq6fM7h48aLuvPPOiq4FAACgzMoVZoYNG6ZFixZVdC0AAABlVq7LTBcuXNDrr7+uDz/8UDExMapatarb8hdffLFCigMAALiacoWZPXv2qFWrVpKkffv2uS1jMDAAAKhM5QozmzZtqug6AAAAyqVcY2YAAAA8Rbl6ZmJjY694OWnjxo3lLggAAKAsyhVmLo+XuezSpUtKT0/Xvn37ivwAJQAAwPVUrjDz0ksvFds+ZcoU5ebmXlNBAAAAZVGhY2YGDRrE7zIBAIBKVaFh5tNPP5Wvr29F7hIAAOCKynWZ6cEHH3SbtyxLp06d0s6dOzVx4sQKKQwAAKA0yhVmAgMD3earVKmipk2batq0abrnnnsqpDAAAIDSKFeYWbBgQUXXAQAAUC7lCjOX7dq1SwcOHJAkRUdH67bbbquQogAAAEqrXGEmKytL/fv31+bNm1WzZk1J0tmzZxUbG6slS5aoXr16FVkjAABAicp1N9Po0aN17tw57d+/Xz/88IN++OEH7du3Tzk5ORozZkxF1wgAAFCicvXMrFu3Th9++KGaNWvmamvevLnmzJnDAGAAAFCpytUzU1hYqKpVqxZpr1q1qgoLC6+5KAAAgNIqV5i566679MQTT+jkyZOuthMnTujJJ5/U3XffXWHFAQAAXE25wszs2bOVk5OjiIgIRUVFKSoqSpGRkcrJydGrr75a0TUCAACUqFxjZkJDQ7V79259+OGH+vLLLyVJzZo1U1xcXIUWBwAAcDVl6pnZuHGjmjdvrpycHDkcDnXr1k2jR4/W6NGj1bZtW0VHR+ujjz66XrUCAAAUUaYwk5KSouHDh6tGjRpFlgUGBuqxxx7Tiy++WGHFAQAAXE2Zwsy///1v3XvvvSUuv+eee7Rr165rLuqygoICTZw4UZGRkapWrZqioqI0ffp0WZZVYccAAABmK9OYmczMzGJvyXbtzNtbZ86cueaiLnvhhRc0d+5cvfnmm4qOjtbOnTs1dOhQBQYG8nA+AAAgqYxh5qabbtK+ffvUqFGjYpfv2bNHDRo0qJDCJOmTTz5R79691b17d0lSRESEFi9erM8//7zCjgEAAMxWpstM9913nyZOnKgLFy4UWfbf//5XkydPVo8ePSqsuDvvvFNpaWn66quvJP10mWvbtm1KSEgocZv8/Hzl5OS4TQAA4MZVpp6ZP/7xj3r33XfVpEkTjRo1Sk2bNpUkffnll5ozZ44KCgo0YcKECitu/PjxysnJ0S233CIvLy8VFBTo+eef18CBA0vcZubMmZo6dWqF1QAAADxbmcJMUFCQPvnkE/3+979XcnKyayCuw+FQfHy85syZo6CgoAor7u2339Y///lPLVq0SNHR0UpPT9fYsWMVEhKixMTEYrdJTk5WUlKSaz4nJ0ehoaEVVhMAAPAsZX5oXnh4uN5//3395z//0eHDh2VZlho3bqxatWpVeHHjxo3T+PHj1b9/f0nSrbfequPHj2vmzJklhhmn0ymn01nhtQAAAM9UricAS1KtWrXUtm3biqyliPPnz6tKFfdhPV5eXvyYJQAAcCl3mKkMPXv21PPPP6+wsDBFR0frX//6l1588UU98sgjdpcGAAA8hEeHmVdffVUTJ07U448/rqysLIWEhOixxx7TpEmT7C4NAAB4CI8OMwEBAUpJSVFKSordpQAAAA9VpufMAAAAeBrCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAoxFmAACA0QgzAADAaIQZAABgNMIMAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAo3l8mDlx4oQGDRqkOnXqqFq1arr11lu1c+dOu8sCAAAewtvuAq7kP//5jzp06KDY2FitXbtW9erV06FDh1SrVi27SwMAAB7Co8PMCy+8oNDQUC1YsMDVFhkZaWNFAADA03j0ZaZVq1apTZs2+u1vf6v69evrtttu09/+9rcrbpOfn6+cnBy3CQAA3Lg8umfm66+/1ty5c5WUlKRnn31WO3bs0JgxY+Tj46PExMRit5k5c6amTp1ayZWiLBY/+5ndJdhmwIz2dpcAADccj+6ZKSwsVOvWrTVjxgzddtttGjFihIYPH6558+aVuE1ycrKys7NdU0ZGRiVWDAAAKptHh5kGDRqoefPmbm3NmjXTN998U+I2TqdTNWrUcJsAAMCNy6PDTIcOHXTw4EG3tq+++krh4eE2VQQAADyNR4eZJ598Utu3b9eMGTN0+PBhLVq0SK+//rpGjhxpd2kAAMBDeHSYadu2rVasWKHFixerRYsWmj59ulJSUjRw4EC7SwMAAB7Co+9mkqQePXqoR48edpcBAAA8lEf3zAAAAFwNYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAoxFmAACA0QgzAADAaIQZAABgNMIMAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGhGhZlZs2bJ4XBo7NixdpcCAAA8hDFhZseOHZo/f75iYmLsLgUAAHgQI8JMbm6uBg4cqL/97W+qVauW3eUAAAAPYkSYGTlypLp37664uLirrpufn6+cnBy3CQAA3Li87S7gapYsWaLdu3drx44dpVp/5syZmjp16nWuCgAAeAqP7pnJyMjQE088oX/+85/y9fUt1TbJycnKzs52TRkZGde5SgAAYCeP7pnZtWuXsrKy1Lp1a1dbQUGBtm7dqtmzZys/P19eXl5u2zidTjmdzsouFQAA2MSjw8zdd9+tvXv3urUNHTpUt9xyi5555pkiQQYAAPz6eHSYCQgIUIsWLdza/Pz8VKdOnSLtAADg18mjx8wAAABcjUf3zBRn8+bNdpcAAAA8CD0zAADAaIQZAABgNMIMAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAoxFmAACA0QgzAADAaIQZAABgNMIMAAAwGmEGAAAYjTADAACMRpgBAABG8/gwM3PmTLVt21YBAQGqX7++7r//fh08eNDusgAAgIfw+DCzZcsWjRw5Utu3b9eGDRt06dIl3XPPPcrLy7O7NAAA4AG87S7gatatW+c2n5qaqvr162vXrl3q3LmzTVUBAABP4fFh5peys7MlSbVr1y52eX5+vvLz813zOTk5lVIXAACwh1FhprCwUGPHjlWHDh3UokWLYteZOXOmpk6dWmk1LX72s0o7licZMKO93SUApcJ3FLjxefyYmZ8bOXKk9u3bpyVLlpS4TnJysrKzs11TRkZGJVYIAAAqmzE9M6NGjdLq1au1detWNWzYsMT1nE6nnE5nJVYGAADs5PFhxrIsjR49WitWrNDmzZsVGRlpd0kAAMCDeHyYGTlypBYtWqT33ntPAQEBOn36tCQpMDBQ1apVs7k6AABgN48fMzN37lxlZ2era9euatCggWtaunSp3aUBAAAP4PE9M5Zl2V0CAADwYB7fMwMAAHAlhBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYjzAAAAKMRZgAAgNEIMwAAwGiEGQAAYDTCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAoxFmAACA0QgzAADAaIQZAABgNMIMAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEYzIszMmTNHERER8vX1Vfv27fX555/bXRIAAPAQHh9mli5dqqSkJE2ePFm7d+9Wy5YtFR8fr6ysLLtLAwAAHsDjw8yLL76o4cOHa+jQoWrevLnmzZun6tWr64033rC7NAAA4AE8OsxcvHhRu3btUlxcnKutSpUqiouL06effmpjZQAAwFN4213AlXz33XcqKChQUFCQW3tQUJC+/PLLYrfJz89Xfn6+az47O1uSlJOTc11qPJ+fd1326+mu5Xz+Ws+ZxHkrj2v97nLeyofzVnacs+uzX8uyrr6y5cFOnDhhSbI++eQTt/Zx48ZZ7dq1K3abyZMnW5KYmJiYmJiYboApIyPjqnnBo3tm6tatKy8vL2VmZrq1Z2ZmKjg4uNhtkpOTlZSU5JovLCzUDz/8oDp16sjhcFzXeitTTk6OQkNDlZGRoRo1athdjhE4Z+XDeSsfzlv5cN7K7kY9Z5Zl6dy5cwoJCbnquh4dZnx8fHT77bcrLS1N999/v6SfwklaWppGjRpV7DZOp1NOp9OtrWbNmte5UvvUqFHjhvrwVgbOWflw3sqH81Y+nLeyuxHPWWBgYKnW8+gwI0lJSUlKTExUmzZt1K5dO6WkpCgvL09Dhw61uzQAAOABPD7M9OvXT2fOnNGkSZN0+vRptWrVSuvWrSsyKBgAAPw6eXyYkaRRo0aVeFnp18rpdGry5MlFLqmhZJyz8uG8lQ/nrXw4b2XHOZMcllWae54AAAA8k0c/NA8AAOBqCDMAAMBohBkAAGA0wgwAADAaYcZAc+bMUUREhHx9fdW+fXt9/vnndpfk8bZu3aqePXsqJCREDodDK1eutLskjzdz5ky1bdtWAQEBql+/vu6//34dPHjQ7rI83ty5cxUTE+N6gNkdd9yhtWvX2l2WUWbNmiWHw6GxY8faXYpHmzJlihwOh9t0yy232F2WLQgzhlm6dKmSkpI0efJk7d69Wy1btlR8fLyysrLsLs2j5eXlqWXLlpozZ47dpRhjy5YtGjlypLZv364NGzbo0qVLuueee5SX9+v8Mb3SatiwoWbNmqVdu3Zp586duuuuu9S7d2/t37/f7tKMsGPHDs2fP18xMTF2l2KE6OhonTp1yjVt27bN7pJswa3Zhmnfvr3atm2r2bNnS/rp5x1CQ0M1evRojR8/3ubqzOBwOLRixQrXT2SgdM6cOaP69etry5Yt6ty5s93lGKV27dr685//rEcffdTuUjxabm6uWrdurddee03PPfecWrVqpZSUFLvL8lhTpkzRypUrlZ6ebncptqNnxiAXL17Url27FBcX52qrUqWK4uLi9Omnn9pYGX4NsrOzJf30P2aUTkFBgZYsWaK8vDzdcccddpfj8UaOHKnu3bu7/RuHKzt06JBCQkJ08803a+DAgfrmm2/sLskWRjwBGD/57rvvVFBQUOSnHIKCgvTll1/aVBV+DQoLCzV27Fh16NBBLVq0sLscj7d3717dcccdunDhgvz9/bVixQo1b97c7rI82pIlS7R7927t2LHD7lKM0b59e6Wmpqpp06Y6deqUpk6dqk6dOmnfvn0KCAiwu7xKRZgBcFUjR47Uvn37frXX48uqadOmSk9PV3Z2tpYtW6bExERt2bKFQFOCjIwMPfHEE9qwYYN8fX3tLscYCQkJrv+OiYlR+/btFR4errfffvtXd0mTMGOQunXrysvLS5mZmW7tmZmZCg4Otqkq3OhGjRql1atXa+vWrWrYsKHd5RjBx8dHjRo1kiTdfvvt2rFjh15++WXNnz/f5so8065du5SVlaXWrVu72goKCrR161bNnj1b+fn58vLysrFCM9SsWVNNmjTR4cOH7S6l0jFmxiA+Pj66/fbblZaW5morLCxUWloa1+NR4SzL0qhRo7RixQpt3LhRkZGRdpdkrMLCQuXn59tdhse6++67tXfvXqWnp7umNm3aaODAgUpPTyfIlFJubq6OHDmiBg0a2F1KpaNnxjBJSUlKTExUmzZt1K5dO6WkpCgvL09Dhw61uzSPlpub6/bXytGjR5Wenq7atWsrLCzMxso818iRI7Vo0SK99957CggI0OnTpyVJgYGBqlatms3Vea7k5GQlJCQoLCxM586d06JFi7R582atX7/e7tI8VkBAQJGxWH5+fqpTpw5jtK7g6aefVs+ePRUeHq6TJ09q8uTJ8vLy0oABA+wurdIRZgzTr18/nTlzRpMmTdLp06fVqlUrrVu3rsigYLjbuXOnYmNjXfNJSUmSpMTERKWmptpUlWebO3euJKlr165u7QsWLNCQIUMqvyBDZGVl6eGHH9apU6cUGBiomJgYrV+/Xt26dbO7NNxgvv32Ww0YMEDff/+96tWrp44dO2r79u2qV6+e3aVVOp4zAwAAjMaYGQAAYDTCDAAAMBphBgAAGI0wAwAAjEaYAQAARiPMAAAAoxFmAACA0QgzAADAaIQZALY4c+aMfv/73yssLExOp1PBwcGKj4/Xxx9/bHdpAAzDzxkAsEWfPn108eJFvfnmm7r55puVmZmptLQ0ff/999fleBcvXpSPj8912TcAe9EzA6DSnT17Vh999JFeeOEFxcbGKjw8XO3atVNycrJ69erlWuexxx5TUFCQfH191aJFC61evdq1j+XLlys6OlpOp1MRERH661//6naMiIgITZ8+XQ8//LBq1KihESNGSJK2bdumTp06qVq1agoNDdWYMWOUl5dXeS8eQIUjzACodP7+/vL399fKlSuVn59fZHlhYaESEhL08ccfa+HChfriiy80a9YseXl5SZJ27dqlhx56SP3799fevXs1ZcoUTZw4sciPhv7lL39Ry5Yt9a9//UsTJ07UkSNHdO+996pPnz7as2ePli5dqm3btmnUqFGV8bIBXCf80CQAWyxfvlzDhw/Xf//7X7Vu3VpdunRR//79FRMTow8++EAJCQk6cOCAmjRpUmTbgQMH6syZM/rggw9cbX/4wx+0Zs0a7d+/X9JPPTO33XabVqxY4Vpn2LBh8vLy0vz5811t27ZtU5cuXZSXlydfX9/r+IoBXC/0zACwRZ8+fXTy5EmtWrVK9957rzZv3qzWrVsrNTVV6enpatiwYbFBRpIOHDigDh06uLV16NBBhw4dUkFBgautTZs2buv8+9//VmpqqqtnyN/fX/Hx8SosLNTRo0cr/kUCqBQMAAZgG19fX3Xr1k3dunXTxIkTNWzYME2ePFlPP/10hezfz8/PbT43N1ePPfaYxowZU2TdsLCwCjkmgMpHmAHgMZo3b66VK1cqJiZG3377rb766qtie2eaNWtW5Bbujz/+WE2aNHGNqylO69at9cUXX6hRo0YVXjsA+3CZCUCl+/7773XXXXdp4cKF2rNnj44ePap33nlHf/rTn9S7d2916dJFnTt3Vp8+fbRhwwYdPXpUa9eu1bp16yRJTz31lNLS0jR9+nR99dVXevPNNzV79uyr9ug888wz+uSTTzRq1Cilp6fr0KFDeu+99xgADBiOnhkAlc7f31/t27fXSy+9pCNHjujSpUsKDQ3V8OHD9eyzz0r6aYDw008/rQEDBigvL0+NGjXSrFmzJP3Uw/L2229r0qRJmj59uho0aKBp06ZpyJAhVzxuTEyMtmzZogkTJqhTp06yLEtRUVHq16/f9X7JAK4j7mYCAABG4zITAAAwGmEGAAAYjTADAACMRpgBAABGI8wAAACjEWYAAIDRCDMAAMBohBkAAGA0wgwAADAaYQYAABiNMAMAAIxGmAEAAEb7f7P61jdH3Vs+AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vertex_answer_similarity_scores = pd.Series([x.scores[\"answer_similarity\"] for x in vertex_run.run_data])\n",
    "category_counts = vertex_answer_similarity_scores.value_counts()\n",
    "plt.bar(category_counts.index, category_counts.values)\n",
    "\n",
    "plt.title('Distribution of scores for Vertex')\n",
    "plt.xlabel('Score')\n",
    "plt.ylabel('Count')\n",
    "\n",
    "plt.bar(category_counts.index, category_counts.values, color='#A679C8')\n",
    "\n",
    "# Remove all scores except whole numbers\n",
    "plt.xticks(range(0, 6, 1))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OpenAI Rag Assistant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_openai_rag_response(benchmarkItem: BenchmarkItem, assistant):\n",
    "    prompt = benchmarkItem.question    \n",
    "    thread = client.beta.threads.create(\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": prompt\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "    run = client.beta.threads.runs.create(\n",
    "        thread_id=thread.id,\n",
    "        assistant_id=assistant.id,\n",
    "    )\n",
    "    max_retries = 10\n",
    "    base = 0.1\n",
    "    num_retries = 0\n",
    "    try:\n",
    "        while max_retries > 0:\n",
    "            if max_retries == 0:\n",
    "                client.beta.threads.delete(thread.id)\n",
    "                raise Exception(\"Max tries exceeded\")\n",
    "            messages = client.beta.threads.messages.list(\n",
    "                thread_id=thread.id,\n",
    "            )\n",
    "            if len(messages.data)>0 and len(messages.data[0].content)>0:\n",
    "                response_message = messages.data[0].content[0].text.value\n",
    "                if response_message != prompt and response_message.strip():\n",
    "                    annotations = messages.data[0].content[0].text.annotations\n",
    "                    quotes = [x.file_citation.quote for x in annotations if x.file_citation]\n",
    "                    client.beta.threads.delete(thread.id)\n",
    "                    return response_message\n",
    "            time.sleep(base * (2 ** num_retries))\n",
    "            num_retries += 1\n",
    "            max_retries -= 1\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        client.beta.threads.delete(thread.id)\n",
    "        raise e\n",
    "\n",
    "#openai rag only supports a max of 20 files. So we just combine all essays into single file\n",
    "def upload_essay():\n",
    "    file_ids=[]\n",
    "    with open(\"all_essays_in_single_file.txt\", 'rb') as essay_file:\n",
    "        file = client.files.create(\n",
    "            file=essay_file,\n",
    "            purpose='assistants'\n",
    "        )   \n",
    "        file_ids.append(file.id)\n",
    "    return file_ids\n",
    "\n",
    "def create_assistant(file_ids):\n",
    "    return client.beta.assistants.create(\n",
    "        name=f\"OpenAI Rag Test {len(file_ids)} Files\",\n",
    "        instructions=(\n",
    "            \"You are a chatbot that answers questions about Paul Graham's essays. \"\n",
    "            \"Use your knowledge base to best respond to questions. \"\n",
    "            \"NO MATTER WHAT, DO NOT PULL INFORMATION FROM EXTERNAL KNOWLEDGE. ONLY USE YOUR OWN KNOWLEDGE BASE.\"\n",
    "        ),\n",
    "        model=\"gpt-4-1106-preview\",\n",
    "        tools=[{\"type\": \"retrieval\"}],\n",
    "        file_ids=file_ids\n",
    "    )\n",
    "\n",
    "def setup_assistant():\n",
    "    file_ids = upload_essay()\n",
    "    return create_assistant(file_ids)\n",
    "\n",
    "# WARNING\n",
    "# this deletes all files associated with your openai api key.\n",
    "#def cleanup_files():\n",
    "#    for f in client.files.list():\n",
    "#        client.files.delete(f.id)\n",
    "#        time.sleep(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI()\n",
    "assistant = setup_assistant()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "According to Paul Graham, key components necessary to create a technology hub or attract the \"creative class\" include the following:\n",
      "\n",
      "1. **Personality and Character**: A town must have personality, which usually means avoiding mass-produced environments and having a variety of unique and densely packed buildings. Graham suggests that building codes should ensure density and ban large-scale developments【9†source】.\n",
      "\n",
      "2. **Right Personality for Nerds**: Since many technology hubs are formed by nerds, a city must appeal to their tastes, which differ from the rest of the creative class. Nerds tend to like places where people are friendlier and prefer quieter pleasures like cafes, bookshops, and areas with natural beauty. A nerd's idea of paradise would be places like Berkeley or Boulder【10†source】.\n",
      "\n",
      "3. **Youth Appeal**: Technology hubs tend to be youthful in feel and having an intact city center is important, as young people do not want to live in the suburbs. Boulder and Portland are cited as examples of cities with the kind of effervescent feel that attracts the young【10†source】.\n",
      "\n",
      "4. **Organic Growth**: You can't forcefully create a Silicon Valley; instead, a hub must grow organically. This usually starts with a few key people or companies that lay the groundwork, inspiring a self-sustaining chain reaction as people who work for startups launch or fund their own【10†source】.\n",
      "\n",
      "5. **Time**: The growth of a technology hub takes time, and cannot be rushed. It takes an average of about five years for a company to succeed, and this organic growth cycle is necessary for a community to develop the necessary expertise and culture of innovation【10†source】.\n",
      "\n",
      "In summary, a strong appeal to nerds and young people, an environment that fosters organic and unique development, combined with ample time for growth, are critical ingredients for creating a successful technology hub.\n"
     ]
    }
   ],
   "source": [
    "benchmark_item = BenchmarkItem(\n",
    "    question=\"What key components are necessary to create a technology hub according to Paul Graham?\",\n",
    "    answer=\"\"\n",
    ")\n",
    "\n",
    "print(get_openai_rag_response(benchmark_item, assistant))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Run through all the questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 55/55 [33:28<00:00, 36.51s/it]\n"
     ]
    }
   ],
   "source": [
    "raw_openai_responses = []\n",
    "for x in tqdm(benchmark.items):\n",
    "    raw_openai_responses.append(get_openai_rag_response(x, assistant))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_responses = [\n",
    "    LLMResponse(\n",
    "        llm_answer=r, llm_context_list=[], benchmark_item=bi\n",
    "    ) for r, bi in zip(raw_openai_responses, benchmark.items)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "scorer = ValidateScorer([AnswerSimilarityMetric()])\n",
    "openai_run = scorer.score_run(openai_responses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_run_df = run_to_dataframe(openai_run)\n",
    "openai_run_df.to_csv(\"openai_run.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyAElEQVR4nO3deVhV5f7//9d2YFARR0AUAcfUQpvkGM6SSDmVflKb1FPap0BTs4HMnCr71KloMLNznaRJyzS1YznilKamdsgsM/GrpalolqCYaHD//ujHPm0ZBELWvu35uK51Xa57rb3We997Cy/ufa+1XcYYIwAAAAtVcroAAACAsiLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsjgkjV58mS5XK4KOVfXrl3VtWtX9/ratWvlcrk0f/78Cjn/sGHDFBERUSHnKqtTp07p7rvvVkhIiFwul8aMGeN0SV4hIyNDAwcOVN26deVyuZScnOx0SYBVCDKwQkpKilwul3vx8/NTaGio4uLi9NJLL+nkyZPlcp5Dhw5p8uTJSktLK5fjlSdvrq0knnrqKaWkpOjee+/V22+/rTvuuMPpkrzC2LFjtXz5ciUlJentt99Wr169Lvo5s7OzNW3aNEVFRalatWoKDAxUp06d9NZbb8mbv7Wmffv2crlcmjlzZqHb839ObNu2rYIrg6MMYIHZs2cbSWbq1Knm7bffNm+88YZ56qmnTM+ePY3L5TLh4eHmyy+/9HjMuXPnzK+//lqq82zdutVIMrNnzy7V43JyckxOTo57fc2aNUaS+eCDD0p1nLLWdvbsWXPmzJlyO9fFEB0dbWJiYpwuw+sEBweb2267rcLOd+TIEdOmTRtTqVIlc+utt5pZs2aZF1980XTu3NlIMoMGDTK//fZbhdVTUt99952RZCIiIop8H+X/nNi6dWsFVwcnVXEwQwGlFh8fr2uuuca9npSUpNWrV6t3797q27evdu3aJX9/f0lSlSpVVKXKxX2Lnz59WtWqVZOPj89FPc+FVK1a1dHzl8TRo0fVunVrp8sokby8PJ09e1Z+fn4X/VxHjx5VrVq1yu14Z86ckY+PjypVKnzAfejQodq1a5cWLlyovn37uttHjx6tBx98UP/4xz905ZVX6uGHHy63msrDO++8o6CgID333HMaOHCg9u/f7/Ufp6KCOJ2kgJK40F9aTz31lJFkXn/9dXfbpEmTzPlv8RUrVpiYmBgTGBhoqlevblq0aGGSkpKMMf8dRTl/yR8B6dKli2nTpo3Ztm2b6dSpk/H39zf333+/e1uXLl3c58k/1nvvvWeSkpJMcHCwqVatmunTp4/54YcfPGoKDw83Q4cOLfCc/njMC9U2dOhQEx4e7vH4U6dOmXHjxplGjRoZHx8f06JFC/Pss8+avLw8j/0kmYSEBLNw4ULTpk0b4+PjY1q3bm2WLl1aaF+fLyMjw/z97383QUFBxtfX10RFRZmUlJQCfXH+sm/fviKPWdzrlO/XX381kyZNMs2bNze+vr4mJCTE3HTTTSY9Pb3MffDOO++Y1q1bmypVqpiFCxcaY4w5ePCgGT58uAkKCnL3zb/+9a8CNb/00kumdevWxt/f39SqVctcffXV5t133y3yOea/p89f8u3du9cMHDjQ1K5d2/j7+5vo6GizZMkSj2Pk9+3cuXPNhAkTTGhoqHG5XOaXX34p9JybNm0ykszf//73QrefO3fONG/e3NSuXducPn3aGGPMvn37jCTz7LPPmueff940btzY+Pn5mc6dO5uvvvqqwDF27dplBgwYYGrXrm18fX3N1VdfbRYvXlzoc9+wYYMZO3asqVevnqlWrZrp37+/OXr0aKG1NWvWzNx3330mJyfH1KpVyzz55JNF9ikjMn8tjMjgknDHHXfo0Ucf1YoVKzRixIhC9/n666/Vu3dvRUVFaerUqfL19VV6ero2btwoSWrVqpWmTp2qxx9/XCNHjlSnTp0kSdddd537GMePH1d8fLwGDx6s22+/XcHBwcXW9eSTT8rlcunhhx/W0aNHlZycrNjYWKWlpblHjkqiJLX9kTFGffv21Zo1a3TXXXepXbt2Wr58uR588EH9+OOPeuGFFzz237Bhgz788EPdd999CggI0EsvvaQBAwbohx9+UN26dYus69dff1XXrl2Vnp6uxMRERUZG6oMPPtCwYcN04sQJ3X///WrVqpXefvttjR07Vo0aNdIDDzwgSapfv36hx7zQ6yRJubm56t27t1JTUzV48GDdf//9OnnypFauXKmdO3eqadOmpe6D1atXa968eUpMTFS9evUUERGhjIwM/e1vf5PL5VJiYqLq16+vpUuX6q677lJWVpZ7wvI///lPjR49WgMHDtT999+vM2fOaMeOHdqyZYtuvfXWQp9n586d3XOFrr/+et15553ubRkZGbruuut0+vRpjR49WnXr1tWbb76pvn37av78+brppps8jjVt2jT5+Pho/PjxysnJKXKE8N///rckeZzrj6pUqaJbb71VU6ZM0caNGxUbG+ve9tZbb+nkyZNKSEjQmTNn9OKLL6p79+766quv3P8Pvv76a8XExKhhw4Z65JFHVL16dc2bN0/9+/fXggULCtQ9atQo1a5dW5MmTdL+/fuVnJysxMREvf/++x77bdmyRenp6Zo9e7Z8fHx08803691339Wjjz5a6PPAX4zTSQooiZL8pRUYGGiuvPJK9/r5IzIvvPCCkWSOHTtW5DGKm4fSpUsXI8m89tprhW4rbESmYcOGJisry90+b948I8m8+OKL7raSjMhcqLbzR2QWLVpkJJknnnjCY7+BAwcal8vlMWohyfj4+Hi0ffnll0aSefnllwuc64+Sk5ONJPPOO++4286ePWs6dOhgatSo4fHcw8PDzY033ljs8Ywp2ev0xhtvGEnm+eefL7Atf7SltH1QqVIl8/XXX3vse9ddd5kGDRqYn376yaN98ODBJjAw0D1q0a9fP9OmTZsLPrfC6P8fDfqjMWPGGEnm008/dbedPHnSREZGmoiICJObm2uM+e/7rEmTJu5aitO/f38jqcgRG2OM+fDDD40k89JLLxlj/jsi4+/vbw4ePOjeb8uWLUaSGTt2rLutR48e5oorrvCYr5WXl2euu+4607x5c3db/v/n2NhYj9GxsWPHmsqVK5sTJ0541JSYmGjCwsLc+65YscJIMv/5z3889mNE5q+Jq5ZwyahRo0axVy/lz0NYvHix8vLyynQOX19fDR8+vMT733nnnQoICHCvDxw4UA0aNNAnn3xSpvOX1CeffKLKlStr9OjRHu0PPPCAjDFaunSpR3tsbKyaNm3qXo+KilLNmjX1//7f/7vgeUJCQjRkyBB3W9WqVTV69GidOnVK69atK3XtJXmdFixYoHr16mnUqFEFtuVfcl/aPujSpYvHHB5jjBYsWKA+ffrIGKOffvrJvcTFxSkzM1NffPGFu+aDBw9q69atpX6+hfnkk0/Uvn17dezY0d1Wo0YNjRw5Uvv379c333zjsf/QoUNLNMKX///jj+/J8+Vvy8rK8mjv37+/GjZs6F5v3769oqOj3e/ln3/+WatXr9Ytt9yikydPuvvq+PHjiouL0549e/Tjjz96HHPkyJEet0jo1KmTcnNz9f3337vbfvvtN73//vsaNGiQe9/u3bsrKChI77777gWfMy59BBlcMk6dOlXsD+hBgwYpJiZGd999t4KDgzV48GDNmzevVKGmYcOGpZrY27x5c491l8ulZs2aaf/+/SU+Rll8//33Cg0NLdAfrVq1cm//o8aNGxc4Ru3atfXLL79c8DzNmzcvMLG0qPOURElep71796ply5bFTuYubR9ERkZ6rB87dkwnTpzQ66+/rvr163ss+WH26NGjkqSHH35YNWrUUPv27dW8eXMlJCR4fBRWWt9//71atmxZoL2ktRclvy+KC/xFhZ3z38uS1KJFC/d7OT09XcYYTZw4sUB/TZo0SdJ/+yvf+e+72rVrS5LH+27FihU6duyY2rdvr/T0dKWnp2vfvn3q1q2b5s6dW+Y/SnDpYI4MLgkHDx5UZmammjVrVuQ+/v7+Wr9+vdasWaOPP/5Yy5Yt0/vvv6/u3btrxYoVqly58gXPU5p5LSVV1E37cnNzS1RTeSjqPMaBe4qUx+tU1vP+Uf4vyNtvv11Dhw4t9DFRUVGSfg8Yu3fv1pIlS7Rs2TItWLBAr776qh5//HFNmTLlotT7RyV9X7Zq1UqLFi3Sjh071Llz50L32bFjhySV+gqz/P4aP3684uLiCt3n/P+fJXnf5Y+63HLLLYXuu27dOnXr1q1UteLSQpDBJeHtt9+WpCJ/gOarVKmSevTooR49euj555/XU089pQkTJmjNmjWKjY0t9zsB79mzx2PdGKP09HT3L0Dp979CT5w4UeCx33//vZo0aeJeL01t4eHhWrVqlU6ePOnxl/W3337r3l4ewsPDtWPHDuXl5XmMyvzZ81zodWratKm2bNmic+fOFXnp+Z/tg/r16ysgIEC5ubkek16LUr16dQ0aNEiDBg3S2bNndfPNN+vJJ59UUlJSqS/jDg8P1+7duwu0/9l+7d27t6ZPn6633nqr0CCTm5urOXPmqHbt2oqJifHYdv57WZK+++479yXQ+e/VqlWrlqi/SiI7O1uLFy/WoEGDNHDgwALbR48erXfffZcg8xfHR0uw3urVqzVt2jRFRkbqtttuK3K/n3/+uUBbu3btJEk5OTmSfv9lJKnQYFEW+Vd65Js/f74OHz6s+Ph4d1vTpk21efNmnT171t22ZMkSHThwwONYpanthhtuUG5url555RWP9hdeeEEul8vj/H/GDTfcoCNHjnhcZfLbb7/p5ZdfVo0aNdSlS5dSH7Mkr9OAAQP0008/FXh+0n//mv+zfVC5cmUNGDBACxYs0M6dOwtsP3bsmPvfx48f99jm4+Oj1q1byxijc+fOFXuewtxwww36/PPPtWnTJndbdna2Xn/9dUVERJT5fjzXXXedYmNjNXv2bC1ZsqTA9gkTJui7777TQw89VGCUZ9GiRR5zXD7//HNt2bLF3Y9BQUHq2rWrZs2apcOHDxc49h/7q6QWLlyo7OxsJSQkaODAgQWW3r17a8GCBe73Bf6aGJGBVZYuXapvv/1Wv/32mzIyMrR69WqtXLlS4eHh+uijj4r9y3fq1Klav369brzxRoWHh+vo0aN69dVX1ahRI/ekyqZNm6pWrVp67bXXFBAQoOrVqys6OrrEcxDOV6dOHXXs2FHDhw9XRkaGkpOT1axZM49LxO+++27Nnz9fvXr10i233KK9e/fqnXfe8Zh8W9ra+vTpo27dumnChAnav3+/2rZtqxUrVmjx4sUaM2ZMgWOX1ciRIzVr1iwNGzZM27dvV0REhObPn6+NGzcqOTm52DlLRSnJ63TnnXfqrbfe0rhx4/T555+rU6dOys7O1qpVq3TfffepX79+5dIHTz/9tNasWaPo6GiNGDFCrVu31s8//6wvvvhCq1atcoeunj17KiQkRDExMQoODtauXbv0yiuv6MYbbyxTHzzyyCOaO3eu4uPjNXr0aNWpU0dvvvmm9u3bpwULFhR5s7uSeOutt9SjRw/169dPt956qzp16qScnBx9+OGHWrt2rQYNGqQHH3ywwOOaNWumjh076t5771VOTo6Sk5NVt25dPfTQQ+59ZsyYoY4dO+qKK67QiBEj1KRJE2VkZGjTpk06ePCgvvzyy1LV+u6776pu3bpF3magb9+++uc//6mPP/5YN998c+k6ApcOpy6XAkrj/JuH+fj4mJCQEHP99debF1980eMy33znX36dmppq+vXrZ0JDQ42Pj48JDQ01Q4YMMd99953H4xYvXuy+KZoKuSFeYYq6/Hru3LkmKSnJBAUFGX9/f3PjjTea77//vsDjn3vuOdOwYUPj6+trYmJizLZt2wocs7jaCrsh3smTJ83YsWNNaGioqVq1qmnevHmxN4M7X1GXhZ8vIyPDDB8+3NSrV8/4+PiYK664otBLxEt6+XVJX6fTp0+bCRMmmMjISFO1alUTEhJiBg4caPbu3VtufZD//BISEkxYWJj7PD169PC4+eKsWbNM586dTd26dY2vr69p2rSpefDBB01mZuYFn29R586/IV6tWrWMn5+fad++fZE3xCvtV2GcPHnSTJ482bRp08b4+/ubgIAAExMTY1JSUgr0zR9viPfcc8+ZsLAw4+vrazp16lTga0Hy677zzjtNSEiIqVq1qmnYsKHp3bu3mT9/vnufoi6Tzn8+a9asMRkZGaZKlSrmjjvuKPJ5nD592lSrVs3cdNNNxR4XlzaXMV78DWEAAEft379fkZGRevbZZzV+/HinywEKYI4MAACwFkEGAABYiyADAACsxRwZAABgLUZkAACAtQgyAADAWpf8DfHy8vJ06NAhBQQElPvt5wEAwMVhjNHJkycVGhpa7E0gL/kgc+jQIYWFhTldBgAAKIMDBw6oUaNGRW6/5INM/u3BDxw4oJo1azpcDQAAKImsrCyFhYVd8Gs+Lvkgk/9xUs2aNQkyAABY5kLTQpjsCwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWcjTITJ8+Xddee60CAgIUFBSk/v37a/fu3R77dO3aVS6Xy2P53//9X4cqBgAA3sTRILNu3TolJCRo8+bNWrlypc6dO6eePXsqOzvbY78RI0bo8OHD7uWZZ55xqGIAAOBNHP3SyGXLlnmsp6SkKCgoSNu3b1fnzp3d7dWqVVNISEhFlwcAALycV82RyczMlCTVqVPHo/3dd99VvXr1dPnllyspKUmnT592ojwAAOBlHB2R+aO8vDyNGTNGMTExuvzyy93tt956q8LDwxUaGqodO3bo4Ycf1u7du/Xhhx8WepycnBzl5OS417Oysi567QAAwBleE2QSEhK0c+dObdiwwaN95MiR7n9fccUVatCggXr06KG9e/eqadOmBY4zffp0TZky5aLXCwDAH819dIvTJThiyFPRjp7fKz5aSkxM1JIlS7RmzRo1atSo2H2jo3/vsPT09EK3JyUlKTMz070cOHCg3OsFAADewdERGWOMRo0apYULF2rt2rWKjIy84GPS0tIkSQ0aNCh0u6+vr3x9fcuzTAAA4KUcDTIJCQmaM2eOFi9erICAAB05ckSSFBgYKH9/f+3du1dz5szRDTfcoLp162rHjh0aO3asOnfurKioKCdLBwAAXsDRIDNz5kxJv9/07o9mz56tYcOGycfHR6tWrVJycrKys7MVFhamAQMG6LHHHnOgWgAA4G0c/2ipOGFhYVq3bl0FVQMAAGzjFZN9AQAAyoIgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrORpkpk+frmuvvVYBAQEKCgpS//79tXv3bo99zpw5o4SEBNWtW1c1atTQgAEDlJGR4VDFAADAmzgaZNatW6eEhARt3rxZK1eu1Llz59SzZ09lZ2e79xk7dqz+/e9/64MPPtC6det06NAh3XzzzQ5WDQAAvEUVJ0++bNkyj/WUlBQFBQVp+/bt6ty5szIzM/Wvf/1Lc+bMUffu3SVJs2fPVqtWrbR582b97W9/c6JsAADgJbxqjkxmZqYkqU6dOpKk7du369y5c4qNjXXvc9lll6lx48batGlTocfIyclRVlaWxwIAAC5NXhNk8vLyNGbMGMXExOjyyy+XJB05ckQ+Pj6qVauWx77BwcE6cuRIoceZPn26AgMD3UtYWNjFLh0AADjEa4JMQkKCdu7cqffee+9PHScpKUmZmZnu5cCBA+VUIQAA8DaOzpHJl5iYqCVLlmj9+vVq1KiRuz0kJERnz57ViRMnPEZlMjIyFBISUuixfH195evre7FLBgAAXsDRERljjBITE7Vw4UKtXr1akZGRHtuvvvpqVa1aVampqe623bt364cfflCHDh0qulwAAOBlHB2RSUhI0Jw5c7R48WIFBAS4570EBgbK399fgYGBuuuuuzRu3DjVqVNHNWvW1KhRo9ShQweuWAIAAM4GmZkzZ0qSunbt6tE+e/ZsDRs2TJL0wgsvqFKlShowYIBycnIUFxenV199tYIrBQAA3sjRIGOMueA+fn5+mjFjhmbMmFEBFQEAAJt4zVVLAAAApUWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrORpk1q9frz59+ig0NFQul0uLFi3y2D5s2DC5XC6PpVevXs4UCwAAvI6jQSY7O1tt27bVjBkzitynV69eOnz4sHuZO3duBVYIAAC8WRUnTx4fH6/4+Phi9/H19VVISEgFVQQAAGzi9XNk1q5dq6CgILVs2VL33nuvjh8/7nRJAADASzg6InMhvXr10s0336zIyEjt3btXjz76qOLj47Vp0yZVrly50Mfk5OQoJyfHvZ6VlVVR5QIAgArm1UFm8ODB7n9fccUVioqKUtOmTbV27Vr16NGj0MdMnz5dU6ZMqagSAQCAg7z+o6U/atKkierVq6f09PQi90lKSlJmZqZ7OXDgQAVWCAAAKpJXj8ic7+DBgzp+/LgaNGhQ5D6+vr7y9fWtwKoAAIBTHA0yp06d8hhd2bdvn9LS0lSnTh3VqVNHU6ZM0YABAxQSEqK9e/fqoYceUrNmzRQXF+dg1QAAwFs4GmS2bdumbt26udfHjRsnSRo6dKhmzpypHTt26M0339SJEycUGhqqnj17atq0aYy4AAAASQ4Hma5du8oYU+T25cuXV2A1AADANlZN9gUAAPgjggwAALBWmYJMkyZNCr3D7okTJ9SkSZM/XRQAAEBJlCnI7N+/X7m5uQXac3Jy9OOPP/7pogAAAEqiVJN9P/roI/e/ly9frsDAQPd6bm6uUlNTFRERUW7FAQAAFKdUQaZ///6SJJfLpaFDh3psq1q1qiIiIvTcc8+VW3EAAADFKVWQycvLkyRFRkZq69atqlev3kUpCgAAoCTKdB+Zffv2lXcdAAAApVbmG+KlpqYqNTVVR48edY/U5HvjjTf+dGEAAAAXUqYgM2XKFE2dOlXXXHONGjRoIJfLVd51AQAAXFCZgsxrr72mlJQU3XHHHeVdDwAAQImV6T4yZ8+e1XXXXVfetQAAAJRKmYLM3XffrTlz5pR3LQAAAKVSpo+Wzpw5o9dff12rVq1SVFSUqlat6rH9+eefL5fiAAAAilOmILNjxw61a9dOkrRz506PbUz8BQAAFaVMQWbNmjXlXQcAAECplWmODAAAgDco04hMt27div0IafXq1WUuCAAAoKTKFGTy58fkO3funNLS0rRz584CXyYJAABwsZQpyLzwwguFtk+ePFmnTp36UwUBAACUVLnOkbn99tv5niUAAFBhyjXIbNq0SX5+fuV5SAAAgCKV6aOlm2++2WPdGKPDhw9r27ZtmjhxYrkUBgAAcCFlCjKBgYEe65UqVVLLli01depU9ezZs1wKAwAAuJAyBZnZs2eXdx0AAAClVqYgk2/79u3atWuXJKlNmza68sory6UoAACAkihTkDl69KgGDx6stWvXqlatWpKkEydOqFu3bnrvvfdUv3798qwRAACgUGW6amnUqFE6efKkvv76a/3888/6+eeftXPnTmVlZWn06NHlXSMAAEChyjQis2zZMq1atUqtWrVyt7Vu3VozZsxgsi8AAKgwZRqRycvLU9WqVQu0V61aVXl5eX+6KAAAgJIoU5Dp3r277r//fh06dMjd9uOPP2rs2LHq0aNHuRUHAABQnDIFmVdeeUVZWVmKiIhQ06ZN1bRpU0VGRiorK0svv/xyedcIAABQqDLNkQkLC9MXX3yhVatW6dtvv5UktWrVSrGxseVaHAAAQHFKNSKzevVqtW7dWllZWXK5XLr++us1atQojRo1Stdee63atGmjTz/99GLVCgAA4KFUQSY5OVkjRoxQzZo1C2wLDAzUPffco+eff77cigMAAChOqYLMl19+qV69ehW5vWfPntq+ffufLgoAAKAkShVkMjIyCr3sOl+VKlV07NixP10UAABASZQqyDRs2FA7d+4scvuOHTvUoEGDP10UAABASZQqyNxwww2aOHGizpw5U2Dbr7/+qkmTJql3797lVhwAAEBxSnX59WOPPaYPP/xQLVq0UGJiolq2bClJ+vbbbzVjxgzl5uZqwoQJF6VQAACA85UqyAQHB+uzzz7Tvffeq6SkJBljJEkul0txcXGaMWOGgoODL0qhAAAA5yv1DfHCw8P1ySef6JdfflF6erqMMWrevLlq1659MeoDAAAoUpnu7CtJtWvX1rXXXluetQAAAJRKmb5rCQAAwBsQZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArFXm+8gAqFhzH93idAmOGPJUtNMlAPBijMgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGs5GmTWr1+vPn36KDQ0VC6XS4sWLfLYbozR448/rgYNGsjf31+xsbHas2ePM8UCAACv42iQyc7OVtu2bTVjxoxCtz/zzDN66aWX9Nprr2nLli2qXr264uLidObMmQquFAAAeCNHv2spPj5e8fHxhW4zxig5OVmPPfaY+vXrJ0l66623FBwcrEWLFmnw4MEVWSoAAPBCXjtHZt++fTpy5IhiY2PdbYGBgYqOjtamTZuKfFxOTo6ysrI8FgAAcGny2iBz5MgRSVJwcLBHe3BwsHtbYaZPn67AwED3EhYWdlHrBAAAzvHaIFNWSUlJyszMdC8HDhxwuiQAAHCReG2QCQkJkSRlZGR4tGdkZLi3FcbX11c1a9b0WAAAwKXJa4NMZGSkQkJClJqa6m7LysrSli1b1KFDBwcrAwAA3sLRq5ZOnTql9PR09/q+ffuUlpamOnXqqHHjxhozZoyeeOIJNW/eXJGRkZo4caJCQ0PVv39/54oGAABew9Egs23bNnXr1s29Pm7cOEnS0KFDlZKSooceekjZ2dkaOXKkTpw4oY4dO2rZsmXy8/NzqmQAAOBFHA0yXbt2lTGmyO0ul0tTp07V1KlTK7AqAABgC6+dIwMAAHAhBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGs5eh8ZAID3mfvoFqdLcMyQp6KdLgGlxIgMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACs5dVBZvLkyXK5XB7LZZdd5nRZAADAS1RxuoALadOmjVatWuVer1LF60sGAAAVxOtTQZUqVRQSEuJ0GQAAwAt59UdLkrRnzx6FhoaqSZMmuu222/TDDz8Uu39OTo6ysrI8FgAAcGny6hGZ6OhopaSkqGXLljp8+LCmTJmiTp06aefOnQoICCj0MdOnT9eUKVMquFKUxtxHtzhdgiOGPBXtdAkAcMnx6hGZ+Ph4/c///I+ioqIUFxenTz75RCdOnNC8efOKfExSUpIyMzPdy4EDByqwYgAAUJG8ekTmfLVq1VKLFi2Unp5e5D6+vr7y9fWtwKoAAIBTvHpE5nynTp3S3r171aBBA6dLAQAAXsCrg8z48eO1bt067d+/X5999pluuukmVa5cWUOGDHG6NAAA4AW8+qOlgwcPasiQITp+/Ljq16+vjh07avPmzapfv77TpQEAAC/g1UHmvffec7oEAADgxbz6oyUAAIDiEGQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKzl1feR8XZ/1W9xlvgmZwCAd2BEBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWIsgAAABrEWQAAIC1CDIAAMBaBBkAAGAtggwAALAWQQYAAFiLIAMAAKxVxekCAOBimvvoFqdLcMSQp6KdLgGoEIzIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACwFkEGAABYiyADAACsRZABAADWsiLIzJgxQxEREfLz81N0dLQ+//xzp0sCAABewOuDzPvvv69x48Zp0qRJ+uKLL9S2bVvFxcXp6NGjTpcGAAAc5vVB5vnnn9eIESM0fPhwtW7dWq+99pqqVaumN954w+nSAACAw7w6yJw9e1bbt29XbGysu61SpUqKjY3Vpk2bHKwMAAB4A6/+0siffvpJubm5Cg4O9mgPDg7Wt99+W+hjcnJylJOT417PzMyUJGVlZZV7fadzssv9mLb4M/35V+23P/sepN/Khn4rvb9qn0n0W1lcjN+vfzyuMab4HY0X+/HHH40k89lnn3m0P/jgg6Z9+/aFPmbSpElGEgsLCwsLC8slsBw4cKDYrODVIzL16tVT5cqVlZGR4dGekZGhkJCQQh+TlJSkcePGudfz8vL0888/q27dunK5XBe13oqUlZWlsLAwHThwQDVr1nS6HCvQZ2VDv5UN/VY29FvpXap9ZozRyZMnFRoaWux+Xh1kfHx8dPXVVys1NVX9+/eX9HswSU1NVWJiYqGP8fX1la+vr0dbrVq1LnKlzqlZs+Yl9catCPRZ2dBvZUO/lQ39VnqXYp8FBgZecB+vDjKSNG7cOA0dOlTXXHON2rdvr+TkZGVnZ2v48OFOlwYAABzm9UFm0KBBOnbsmB5//HEdOXJE7dq107JlywpMAAYAAH89Xh9kJCkxMbHIj5L+qnx9fTVp0qQCH6OhaPRZ2dBvZUO/lQ39Vnp/9T5zGXOh65oAAAC8k1ffEA8AAKA4BBkAAGAtggwAALAWQQYAAFiLIGOhGTNmKCIiQn5+foqOjtbnn3/udElebf369erTp49CQ0Plcrm0aNEip0uywvTp03XttdcqICBAQUFB6t+/v3bv3u10WV5t5syZioqKct+YrEOHDlq6dKnTZVnn6aeflsvl0pgxY5wuxatNnjxZLpfLY7nsssucLqvCEWQs8/7772vcuHGaNGmSvvjiC7Vt21ZxcXE6evSo06V5rezsbLVt21YzZsxwuhSrrFu3TgkJCdq8ebNWrlypc+fOqWfPnsrO/mt+MV5JNGrUSE8//bS2b9+ubdu2qXv37urXr5++/vprp0uzxtatWzVr1ixFRUU5XYoV2rRpo8OHD7uXDRs2OF1ShePya8tER0fr2muv1SuvvCLp969sCAsL06hRo/TII484XJ33c7lcWrhwofsrL1Byx44dU1BQkNatW6fOnTs7XY416tSpo2effVZ33XWX06V4vVOnTumqq67Sq6++qieeeELt2rVTcnKy02V5rcmTJ2vRokVKS0tzuhRHMSJjkbNnz2r79u2KjY11t1WqVEmxsbHatGmTg5XhryAzM1PS77+YcWG5ubl67733lJ2drQ4dOjhdjhUSEhJ04403evyMQ/H27Nmj0NBQNWnSRLfddpt++OEHp0uqcFbc2Re/++mnn5Sbm1vg6xmCg4P17bffOlQV/gry8vI0ZswYxcTE6PLLL3e6HK/21VdfqUOHDjpz5oxq1KihhQsXqnXr1k6X5fXee+89ffHFF9q6davTpVgjOjpaKSkpatmypQ4fPqwpU6aoU6dO2rlzpwICApwur8IQZABcUEJCgnbu3PmX/Py9tFq2bKm0tDRlZmZq/vz5Gjp0qNatW0eYKcaBAwd0//33a+XKlfLz83O6HGvEx8e7/x0VFaXo6GiFh4dr3rx5f6mPMgkyFqlXr54qV66sjIwMj/aMjAyFhIQ4VBUudYmJiVqyZInWr1+vRo0aOV2O1/Px8VGzZs0kSVdffbW2bt2qF198UbNmzXK4Mu+1fft2HT16VFdddZW7LTc3V+vXr9crr7yinJwcVa5c2cEK7VCrVi21aNFC6enpTpdSoZgjYxEfHx9dffXVSk1Ndbfl5eUpNTWVz+BR7owxSkxM1MKFC7V69WpFRkY6XZKV8vLylJOT43QZXq1Hjx766quvlJaW5l6uueYa3XbbbUpLSyPElNCpU6e0d+9eNWjQwOlSKhQjMpYZN26chg4dqmuuuUbt27dXcnKysrOzNXz4cKdL81qnTp3y+Atl3759SktLU506ddS4cWMHK/NuCQkJmjNnjhYvXqyAgAAdOXJEkhQYGCh/f3+Hq/NOSUlJio+PV+PGjXXy5EnNmTNHa9eu1fLly50uzasFBAQUmHtVvXp11a1blzlZxRg/frz69Omj8PBwHTp0SJMmTVLlypU1ZMgQp0urUAQZywwaNEjHjh3T448/riNHjqhdu3ZatmxZgQnA+K9t27apW7du7vVx48ZJkoYOHaqUlBSHqvJ+M2fOlCR17drVo3327NkaNmxYxRdkgaNHj+rOO+/U4cOHFRgYqKioKC1fvlzXX3+906XhEnTw4EENGTJEx48fV/369dWxY0dt3rxZ9evXd7q0CsV9ZAAAgLWYIwMAAKxFkAEAANYiyAAAAGsRZAAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgA8ARx44d07333qvGjRvL19dXISEhiouL08aNG50uDYBF+IoCAI4YMGCAzp49qzfffFNNmjRRRkaGUlNTdfz48YtyvrNnz8rHx+eiHBuAcxiRAVDhTpw4oU8//VT/93//p27duik8PFzt27dXUlKS+vbt697nnnvuUXBwsPz8/HT55ZdryZIl7mMsWLBAbdq0ka+vryIiIvTcc895nCMiIkLTpk3TnXfeqZo1a2rkyJGSpA0bNqhTp07y9/dXWFiYRo8erezs7Ip78gDKFUEGQIWrUaOGatSooUWLFiknJ6fA9ry8PMXHx2vjxo1655139M033+jpp59W5cqVJUnbt2/XLbfcosGDB+urr77S5MmTNXHixAJfAvqPf/xDbdu21X/+8x9NnDhRe/fuVa9evTRgwADt2LFD77//vjZs2KDExMSKeNoALgK+NBKAIxYsWKARI0bo119/1VVXXaUuXbpo8ODBioqK0ooVKxQfH69du3apRYsWBR5722236dixY1qxYoW77aGHHtLHH3+sr7/+WtLvIzJXXnmlFi5c6N7n7rvvVuXKlTVr1ix324YNG9SlSxdlZ2fLz8/vIj5jABcDIzIAHDFgwAAdOnRIH330kXr16qW1a9fqqquuUkpKitLS0tSoUaNCQ4wk7dq1SzExMR5tMTEx2rNnj3Jzc91t11xzjcc+X375pVJSUtwjQjVq1FBcXJzy8vK0b9++8n+SAC46JvsCcIyfn5+uv/56XX/99Zo4caLuvvtuTZo0SePHjy+X41evXt1j/dSpU7rnnns0evToAvs2bty4XM4JoGIRZAB4jdatW2vRokWKiorSwYMH9d133xU6KtOqVasCl2lv3LhRLVq0cM+jKcxVV12lb775Rs2aNSv32gE4g4+WAFS448ePq3v37nrnnXe0Y8cO7du3Tx988IGeeeYZ9evXT126dFHnzp01YMAArVy5Uvv27dPSpUu1bNkySdIDDzyg1NRUTZs2Td99953efPNNvfLKKxccyXn44Yf12WefKTExUWlpadqzZ48WL17MZF/AYozIAKhwNWrUUHR0tF544QXt3btX586dU1hYmEaMGKFHH31U0u+TgcePH68hQ4YoOztbzZo109NPPy3p95GVefPm6fHHH9e0adPUoEEDTZ06VcOGDSv2vFFRUVq3bp0mTJigTp06yRijpk2batCgQRf7KQO4SLhqCQAAWIuPlgAAgLUIMgAAwFoEGQAAYC2CDAAAsBZBBgAAWIsgAwAArEWQAQAA1iLIAAAAaxFkAACAtQgyAADAWgQZAABgLYIMAACw1v8H3washx8qeswAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "openai_answer_similarity_scores = pd.Series([x.scores[\"answer_similarity\"] for x in openai_run.run_data])\n",
    "category_counts = openai_answer_similarity_scores.value_counts()\n",
    "plt.bar(category_counts.index, category_counts.values)\n",
    "\n",
    "plt.title('Distribution of scores for OpenAI')\n",
    "plt.xlabel('Score')\n",
    "plt.ylabel('Count')\n",
    "\n",
    "plt.bar(category_counts.index, category_counts.values, color='#A679C8')\n",
    "\n",
    "# Remove all scores except whole numbers\n",
    "plt.xticks(range(0, 6, 1))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwwAAAHdCAYAAACwgv7YAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB750lEQVR4nO3dd1QUZ9sG8GvpIFVAiiJgxwIWlBcbqCgaYySxRxEVNTHYYkk0xp6osdeo8VWxt9jy2hUVoxEVFCsSIWCJFCsKKCI83x8e5nOFocmylOt3zp7DzjzzzP3cO7vMvVNWIYQQICIiIiIiyoGGugMgIiIiIqKSiwUDERERERHJYsFARERERESyWDAQEREREZEsFgxERERERCSLBQMREREREcliwUBERERERLJYMBARERERkSwWDEREREREJIsFAxEpmTZtGhQKRbGsy9PTE56entLz06dPQ6FQ4Pfffy+W9Q8YMAAODg7Fsq7CSk5OxuDBg2FtbQ2FQoHRo0erO6QSISEhAd27d4e5uTkUCgUWL16s7pCIiMosFgxEZVhgYCAUCoX00NPTg62tLby9vbF06VK8fPmySNbz8OFDTJs2DeHh4UXSX1EqybHlx6xZsxAYGIhhw4Zh06ZN8PX1VXdIJcK3336Lo0ePYuLEidi0aRM6duyo8nWmpKRg5syZcHZ2hoGBAUxMTNCqVSts3LgRQgiVr7+wmjVrBoVCgZUrV+Y4P+tzIjQ0NF/9nT9/Hh4eHjA2NkalSpXQqVMnnDt3Ll/LLly4EAqFAidOnJBts2bNGigUCvzxxx/56jMvs2bNwr59+4qkL6JySxBRmbV+/XoBQMyYMUNs2rRJrFu3TsyaNUt06NBBKBQKYW9vL65evaq0THp6unj16lWB1nPp0iUBQKxfv75Ay6WlpYm0tDTp+alTpwQAsWvXrgL1U9jY3rx5I16/fl1k61IFNzc30aJFC3WHUeJYWVmJvn37Ftv64uPjRb169YSGhob48ssvxerVq8WSJUtE69atBQDRq1cv8fbt22KLJ7/+/vtvAUA4ODjIbkdZnxOXLl3Ks7+7d+8KIyMj4eDgIBYuXCjmz58vvL29xfTp0/MVz7///is0NDTEwIEDZdt4enoKc3Nz8ebNm3z1mZcKFSoIPz+/IumLqLzSUmOtQkTFpFOnTnB1dZWeT5w4ESdPnsSnn36Kzz77DBEREdDX1wcAaGlpQUtLtR8NqampMDAwgI6OjkrXkxdtbW21rj8/EhMTUbduXXWHkS+ZmZl48+YN9PT0VL6uxMREmJqaFll/r1+/ho6ODjQ0cj7w7ufnh4iICOzduxefffaZNH3kyJEYP3485s+fj0aNGuH7778vspiKwubNm1GpUiUsWLAA3bt3R2xs7Eedhnfw4EG8fPkSQUFBaNq0KQBg7NixSEtLy9fytra2aNOmDfbs2YOVK1dCV1dXaf6///6LM2fOYOjQoR/1/hRC4PXr19LnGhF9HJ6SRFROtW3bFpMnT8bdu3exefNmaXpO1zAcP34cLVu2hKmpKQwNDVG7dm388MMPAN5dd5C14zBw4EDp9KfAwEAA765TqF+/PsLCwtC6dWsYGBhIy354DUOWjIwM/PDDD7C2tkaFChXw2Wef4f79+0ptHBwcMGDAgGzLvt9nXrHldA1DSkoKxo4dCzs7O+jq6qJ27dqYP39+tlNOFAoFhg8fjn379qF+/frQ1dVFvXr1cOTIkZwT/oHExET4+/vDysoKenp6cHFxwYYNG6T5WddzxMTE4ODBg1LssbGxsn3m9jplef36NaZNm4ZatWpBT08PNjY2+OKLLxAdHV3oHGzZsgX16tWDrq6uNP5///0XgwYNgpWVlZSbdevWZYt52bJlqFevHgwMDGBmZgZXV1ds3bpVdoxZp88IIbBixQopL1n++ecf9OjRAxUrVoSBgQH+85//4ODBg0p9ZOV2+/bt+PHHH1G5cmUYGBjgxYsXOa4zJCQER48exYABA5SKhSyzZ89GzZo18csvv+DVq1cAgNjYWCgUCsyfPx+LFi2Cvb099PX14eHhgRs3bmTr4/bt2+jevTsqVqwIPT09uLq6ZjslJ2vs586dw5gxY2BpaYkKFSrg888/x6NHj3KMfevWrejevTs+/fRTmJiY5Jrb/MgqqD7cFj7c8c9Nv379kJSUlO11AYDt27cjMzMTffv2BfCuCF28eDHq1asHPT09WFlZ4auvvsKzZ8+UlnNwcMCnn36Ko0ePwtXVFfr6+li9ejUUCgVSUlKwYcMGaVt5/3Mjr+301atXqFOnDurUqSO9tgDw9OlT2NjYoHnz5sjIyMj32IlKLbUe3yAilcrrVIP79+8LAKJ79+7StKlTp4r3Pxpu3LghdHR0hKurq1iyZIlYtWqVGDdunGjdurUQ4t2pGjNmzBAAxNChQ8WmTZvEpk2bRHR0tBBCCA8PD2FtbS0sLS3FiBEjxOrVq8W+ffukeR4eHtK6sk5JatCggXB2dhYLFy4UEyZMEHp6eqJWrVoiNTVVamtvb5/jaQbv95lXbH5+fsLe3l5aNjMzU7Rt21YoFAoxePBgsXz5ctGlSxcBQIwePVppPQCEi4uLsLGxETNnzhSLFy8W1apVEwYGBuLx48e5vi6pqanCyclJaGtri2+//VYsXbpUtGrVSgAQixcvlmLftGmTsLCwEA0bNpRiT05OzrHPvF4nIYR4+/ataNeunQAgevfuLZYvXy5mz54t2rZtK70mBc2Bk5OTsLS0FNOnTxcrVqwQV65cEfHx8aJKlSrCzs5OzJgxQ6xcuVJ89tlnAoBYtGiRtPxvv/0mbX9Zp/j4+/uLkSNHyuYuOjpabNq0SQAQ7du3l/KSlTMrKythZGQkJk2aJBYuXChcXFyEhoaG2LNnj9RH1nZWt25d0bBhQ7Fw4UIxe/ZskZKSkuM6f/jhBwFAnD59WjaurPfN8ePHhRBCxMTESNuyg4OD+OWXX8T06dNFxYoVhaWlpYiPj1d67UxMTETdunXFL7/8IpYvXy5at24tFAqFUtxZ7+dGjRqJtm3bimXLlomxY8cKTU1N0bNnz2wxhYSECADizz//FEIIMWjQIFG3bt1s7QpySlJiYqIwMTERrVu3VjqdsCCSkpKEnp6e6NatW7Z5jRs3Fvb29iIzM1MIIcTgwYOFlpaWGDJkiFi1apX4/vvvRYUKFUTTpk2VTlmyt7cXNWrUEGZmZmLChAli1apV4tSpU2LTpk1CV1dXtGrVStpW/vrrLyGEyPd2GhISIjQ1NcW3334rTevdu7fQ19cXkZGRhcoBUWnDgoGoDMvPjoCJiYlo1KiR9PzDgmHRokUCgHj06JFsH7ldJ+Dh4SEAiFWrVuU4L6eCoXLlyuLFixfS9J07dwoAYsmSJdK0/BQMecX2YcGwb98+AUD89NNPSu26d+8uFAqFiIqKkqYBEDo6OkrTrl69KgCIZcuWZVvX+xYvXiwAiM2bN0vT3rx5I9zd3YWhoaHS2O3t7UXnzp1z7U+I/L1O69atEwDEwoULs83L2kEraA40NDTEzZs3ldr6+/sLGxubbIVT7969hYmJiVT4de3aVdSrVy/PseUEgAgICFCaNnr0aKUdZCGEePnypXB0dBQODg4iIyNDCPH/21m1atWUilA5Pj4+AoB49uyZbJs9e/YIAGLp0qVCiP8vGPT19cWDBw+kdhcuXBAAlHY+27VrJxo0aKB0PU1mZqZo3ry5qFmzpjQt6/3s5eUlvV5CCPHtt98KTU1N8fz5c6WYhg8fLuzs7KS2x44dEwDElStXlNoVpGD466+/hJmZmdDR0RE9evQo9HUbPXr0EHp6eiIpKUmadvv2bQFATJw4UQghxJ9//ikAiC1btigte+TIkWzT7e3tBQBx5MiRbOuSu4Yhv9upEEJMnDhRaGhoiDNnzohdu3YpFfdE5QFPSSIq5wwNDXO9W1LWeeL79+9HZmZmodahq6uLgQMH5rt9//79YWRkJD3v3r07bGxscOjQoUKtP78OHToETU1NjBw5Umn62LFjIYTA4cOHlaZ7eXmhevXq0nNnZ2cYGxvjn3/+yXM91tbW6NOnjzRNW1sbI0eORHJyMoKDgwsce35ep927d8PCwgIjRozINi/rtJ6C5sDDw0PpGgshBHbv3o0uXbpACIHHjx9LD29vbyQlJeHy5ctSzA8ePMClS5cKPN6cHDp0CM2aNUPLli2laYaGhhg6dChiY2Nx69YtpfZ+fn75Osc96/3x/jb5oax5H57W5OPjg8qVK0vPmzVrBjc3N2lbfvr0KU6ePImePXvi5cuXUq6ePHkCb29v3LlzB//++69Sn0OHDlU6DatVq1bIyMjA3bt3pWlv377Fjh070KtXL6lt27ZtUalSJWzZsiXPMefk7t27+OSTT+Dv7499+/Zh7969GDJkiNLpSV999RXs7Ozy7Ktfv354/fo19uzZI03LOl0q63SkXbt2wcTEBO3bt1fajpo0aQJDQ0OcOnVKqU9HR0d4e3vnaywF2U6Bd6dq1qtXD35+fvjmm2/g4eGR7T1CVJaxYCAq55KTk3PdEerVqxdatGiBwYMHw8rKCr1798bOnTsLVDxUrly5QBc416xZU+m5QqFAjRo1cj1/vyjcvXsXtra22fLh5OQkzX9f1apVs/VhZmaW7fzqnNZTs2bNbBfYyq0nP/LzOkVHR6N27dq5XtRe0Bw4OjoqPX/06BGeP3+O3377DZaWlkqPrKIxMTERAPD999/D0NAQzZo1Q82aNREQEJDv23PKxV67du1s0/Mbu5ysXORWWMsVFR9uywBQq1YtaVuOioqCEAKTJ0/Olq+pU6cC+P98ZflwuzMzMwMApe3u2LFjePToEZo1a4aoqChERUUhJiYGbdq0wbZt2wpV/M+ePRsaGhr46aef0KlTJ6xbtw6BgYFKvw1y48YNuLm55dlXp06dULFiRaVrKrZt2wYXFxfUq1cPAHDnzh0kJSWhUqVK2XKTnJycLS/5fT2Bgm2nAKCjo4N169YhJiYGL1++xPr164vt92qISgLeJYmoHHvw4AGSkpJQo0YN2Tb6+vo4c+YMTp06hYMHD+LIkSPYsWMH2rZti2PHjkFTUzPP9ajiTiVy/6wzMjLyFVNRkFvP+9+4FpeieJ0Ku973Ze2I9uvXD35+fjku4+zsDODdjnxkZCQOHDiAI0eOYPfu3fj1118xZcoUTJ8+XSXxvi+/26WTkxP27duHa9euoXXr1jm2uXbtGgAU+I5WWfkaN26c7LfjH74/87PdZR1F6NmzZ45tg4OD0aZNmwLF+tdff6Fhw4bSBc6+vr5ISEjA+PHjYWRkhN69e+P8+fPYvXt3nn1pa2ujZ8+eWLNmDRISEnDv3j3cuXMHc+fOldpkZmbmekTE0tJS6XlBPmcKsp1mOXr0KIB3Nw64c+dOgQoUotKOBQNRObZp0yYAyPMwvoaGBtq1a4d27dph4cKFmDVrFiZNmoRTp07By8uryL9pu3PnjtJzIQSioqKU/oGbmZnh+fPn2Za9e/cuqlWrJj0vSGz29vY4ceIEXr58qfRN8e3bt6X5RcHe3h7Xrl1DZmam0lGGj11PXq9T9erVceHCBaSnp8vesvJjc2BpaQkjIyNkZGTAy8srz5grVKiAXr16oVevXnjz5g2++OIL/Pzzz5g4cWKBb89qb2+PyMjIbNM/Nq+ffvopZs+ejY0bN+ZYMGRkZGDr1q0wMzNDixYtlOZ9uC0DwN9//y3dnStrW9XW1s5XvvIjJSUF+/fvR69evdC9e/ds80eOHIktW7YUuGBQKBTZ7lY2btw4JCQk4Oeff8aWLVvQqFEjdO3aNV/99e3bF6tWrcKOHTsQExMDhUKhdJpe9erVceLECbRo0eKjvnTI6TOgoNvptWvXMGPGDAwcOBDh4eEYPHgwrl+/DhMTk0LHRVSa8JQkonLq5MmTmDlzJhwdHaVzhnPy9OnTbNMaNmwIANK91ytUqAAAOe7AF8bGjRuVTv/4/fffERcXh06dOknTqlevjpCQELx580aaduDAgWw7NAWJ7ZNPPkFGRgaWL1+uNH3RokVQKBRK6/8Yn3zyCeLj47Fjxw5p2tu3b7Fs2TIYGhrCw8OjwH3m53Xq1q0bHj9+nG18wP9/O/2xOdDU1ES3bt2we/fuHG8f+v7tP588eaI0T0dHB3Xr1oUQAunp6bmuJyeffPIJLl68iPPnz0vTUlJS8Ntvv8HBwaHQv2fRvHlzeHl5Yf369Thw4EC2+ZMmTcLff/+N7777LtuO7b59+5SuQbh48SIuXLgg5bFSpUrw9PTE6tWrERcXl61vudul5mbv3r1ISUlBQEAAunfvnu3x6aefYvfu3fn+7YQsXl5euHPnjvRFQ5Y5c+agbt26iI2NxWeffSb7WxYfatGiBRwcHLB582bs2LEDHh4eqFKlijS/Z8+eyMjIwMyZM7Mt+/bt23x/3lSoUCFb24Jsp+np6RgwYABsbW2xZMkSBAYGIiEhAd9++22+1k9UFvAIA1E5cPjwYdy+fRtv375FQkICTp48iePHj8Pe3h5//PFHrt/kzpgxA2fOnEHnzp1hb2+PxMRE/Prrr6hSpYp0cWn16tVhamqKVatWwcjICBUqVICbm1uhD9lXrFgRLVu2xMCBA5GQkIDFixejRo0aGDJkiNRm8ODB+P3339GxY0f07NkT0dHR2Lx5s9JFyAWNrUuXLmjTpg0mTZqE2NhYuLi44NixY9i/fz9Gjx6dre/CGjp0KFavXo0BAwYgLCwMDg4O+P3333Hu3DksXrw412tK5OTnderfvz82btyIMWPG4OLFi2jVqhVSUlJw4sQJfPPNN+jatWuR5GDOnDk4deoU3NzcMGTIENStWxdPnz7F5cuXceLECam46dChA6ytrdGiRQtYWVkhIiICy5cvR+fOnQuVgwkTJmDbtm3o1KkTRo4ciYoVK2LDhg2IiYnB7t27870jm5ONGzeiXbt26Nq1K7788ku0atUKaWlp2LNnD06fPo1evXph/Pjx2ZarUaMGWrZsiWHDhiEtLQ2LFy+Gubk5vvvuO6nNihUr0LJlSzRo0ABDhgxBtWrVkJCQgPPnz+PBgwe4evVqgWLdsmULzM3N0bx58xznf/bZZ1izZg0OHjyIL774It/9Tpw4Efv27YOfnx+OHz+O5s2bIzk5Gdu2bUNMTAyaNm2Kn376Ce7u7ujQoUOe/SkUCnz55ZeYNWsWgHfb8Ps8PDzw1VdfYfbs2QgPD0eHDh2gra2NO3fuYNeuXViyZEmOR1A+1KRJE5w4cQILFy6Era0tHB0d4ebmlu/t9KeffkJ4eDiCgoJgZGQEZ2dnTJkyBT/++CO6d++OTz75JN85JCq11HJvJiIqFlm3S8x66OjoCGtra9G+fXuxZMkSpdt3ZvnwtqpBQUGia9euwtbWVujo6AhbW1vRp08f8ffffystt3//flG3bl2hpaWldBtTDw8P2Vtnyt1Wddu2bWLixImiUqVKQl9fX3Tu3FncvXs32/ILFiwQlStXFrq6uqJFixYiNDQ0W5+5xfbhbVWFeHcbzm+//VbY2toKbW1tUbNmTTFv3jyl21gKkfNtPYWQv93rhxISEsTAgQOFhYWF0NHREQ0aNMjx1q/5va1qfl+n1NRUMWnSJOHo6Ci0tbWFtbW16N69u/TbFEWRg6zxBQQECDs7O2k97dq1E7/99pvUZvXq1aJ169bC3Nxc6OrqiurVq4vx48cr3WpTjty6o6OjRffu3YWpqanQ09MTzZo1EwcOHFBqk7Wd7dq1K8/1vO/ly5di2rRpol69ekJfX18YGRmJFi1aiMDAwGy5ybqt6rx588SCBQuEnZ2d9HsAV69ezTHu/v37C2tra6GtrS0qV64sPv30U/H7779LbeRuf5o1nlOnTomEhAShpaUlfH19ZceRmpoqDAwMxOeff55rvzl5/PixdLtWLS0tYW1tLfr37y9u374tXrx4IerUqSOMjY3F9evX8+xLCCFu3rwpAAhdXV3Z29b+9ttvokmTJlLOGzRoIL777jvx8OFDqU1u75Pbt2+L1q1bC319fQFA6f2Z13YaFhYmtLS0xIgRI5T6fPv2rWjatKmwtbXN9Xa7RGWFQgg1XJ1HRERUhsXGxsLR0RHz5s3DuHHj1B0OEdFH4TUMREREREQkiwUDERERERHJYsFARERERESyeA0DERERERHJ4hEGIiIiIiKSxYKBiIiIiIhk8YfbZGRmZuLhw4cwMjLK8WfliYiIiIhKMyEEXr58CVtb21x/3JIFg4yHDx/Czs5O3WEQEREREanU/fv3UaVKFdn5LBhkGBkZAXiXQGNjYzVHQ0RERERUtF68eAE7Oztpv1cOCwYZWachGRsbs2AgIiIiojIrr9PvedEzERERERHJYsFARERERESyWDAQEREREZEsXsPwETIzM/HmzRt1h1GuaWtrQ1NTU91hEBEREZVZLBgK6c2bN4iJiUFmZqa6Qyn3TE1NYW1tzd/LICIiIlIBFgyFIIRAXFwcNDU1YWdnl+sPXZDqCCGQmpqKxMREAICNjY2aIyIiIiIqe1gwFMLbt2+RmpoKW1tbGBgYqDucck1fXx8AkJiYiEqVKvH0JCIiIqIiVuq+Gp89ezaaNm0KIyMjVKpUCT4+PoiMjFRq4+npCYVCofT4+uuviyyGjIwMAICOjk6R9UmFl1W0paenqzkSIiIiorKn1BUMwcHBCAgIQEhICI4fP4709HR06NABKSkpSu2GDBmCuLg46TF37twij4XnzJcMfB2IiIiIVKfUnZJ05MgRpeeBgYGoVKkSwsLC0Lp1a2m6gYEBrK2tizs8IiIiIqIypdQdYfhQUlISAKBixYpK07ds2QILCwvUr18fEydORGpqaq79pKWl4cWLF0oPIiIiIqLyrtQdYXhfZmYmRo8ejRYtWqB+/frS9C+//BL29vawtbXFtWvX8P333yMyMhJ79uyR7Wv27NmYPn36R8Xjt/3JRy1fUBt6mxdqufv372Pq1Kk4cuQIHj9+DBsbG/j4+GDKlCkwNy9cnx/r/PnzaNmyJTp27IiDBw8qzYuNjYWjoyOuXLmChg0bqiU+IiIiKhrFvb8kp7D7UeVRqS4YAgICcOPGDZw9e1Zp+tChQ6W/GzRoABsbG7Rr1w7R0dGoXr16jn1NnDgRY8aMkZ6/ePECdnZ2qglcjf755x+4u7ujVq1a2LZtGxwdHXHz5k2MHz8ehw8fRkhISLajNcVh7dq1GDFiBNauXYuHDx/C1ta22GMgIiIiouxK7SlJw4cPx4EDB3Dq1ClUqVIl17Zubm4AgKioKNk2urq6MDY2VnqURQEBAdDR0cGxY8fg4eGBqlWrolOnTjhx4gT+/fdfTJo0CQDg4OCAmTNnok+fPqhQoQIqV66MFStWKPX1/PlzDB48GJaWljA2Nkbbtm1x9epVaf60adPQsGFDbNq0CQ4ODjAxMUHv3r3x8uVLpX6Sk5OxY8cODBs2DJ07d0ZgYKDK80BERERE+VPqCgYhBIYPH469e/fi5MmTcHR0zHOZ8PBwAPxhr6dPn+Lo0aP45ptvpN8vyGJtbY2+fftix44dEEIAAObNmwcXFxdcuXIFEyZMwKhRo3D8+HFpmR49eiAxMRGHDx9GWFgYGjdujHbt2uHp06dSm+joaOzbtw8HDhzAgQMHEBwcjDlz5iite+fOnahTpw5q166Nfv36Yd26dVIMRERERKRepe6UpICAAGzduhX79++HkZER4uPjAQAmJibQ19dHdHQ0tm7dik8++QTm5ua4du0avv32W7Ru3RrOzs5qjl697ty5AyEEnJyccpzv5OSEZ8+e4dGjRwCAFi1aYMKECQCAWrVq4dy5c1i0aBHat2+Ps2fP4uLFi0hMTISuri4AYP78+di3bx9+//136bSwzMxMBAYGwsjICADg6+uLoKAg/Pzzz9J6165di379+gEAOnbsiKSkJAQHB8PT01MleSAiIiKi/Ct1RxhWrlyJpKQkeHp6wsbGRnrs2LEDwLsfUztx4gQ6dOiAOnXqYOzYsejWrRv+97//qTnykiO/3967u7tnex4REQEAuHr1KpKTk2Fubg5DQ0PpERMTg+joaGkZBwcHqVgA3h3lSUxMlJ5HRkbi4sWL6NOnDwBAS0sLvXr1wtq1aws9PiIiIiIqOqXuCENeO7t2dnYIDg4upmhKlxo1akChUCAiIgKff/55tvkREREwMzODpaVlnn0lJyfDxsYGp0+fzjbP1NRU+ltbW1tpnkKhQGZmpvR87dq1ePv2rdJFzkII6OrqYvny5TAxMcnHyIiIiIhIVUrdEQYqPHNzc7Rv3x6//vorXr16pTQvPj4eW7ZsQa9evaRfTg4JCVFqExISIp3O1LhxY8THx0NLSws1atRQelhYWOQrnrdv32Ljxo1YsGABwsPDpcfVq1dha2uLbdu2FcGoiYiIiOhjsGAoZ5YvX460tDR4e3vjzJkzuH//Po4cOYL27dujcuXKStcWnDt3DnPnzsXff/+NFStWYNeuXRg1ahQAwMvLC+7u7vDx8cGxY8cQGxuLv/76C5MmTUJoaGi+Yjlw4ACePXsGf39/1K9fX+nRrVs3npZEREREVAKwYChnatasidDQUFSrVg09e/ZE9erVMXToULRp0wbnz59X+g2GsWPHIjQ0FI0aNcJPP/2EhQsXwtvbG8C7U4sOHTqE1q1bY+DAgahVqxZ69+6Nu3fvwsrKKl+xrF27Fl5eXjmedtStWzeEhobi2rVrRTNwIiIiIioUheD9K3P04sULmJiYICkpKdtvMrx+/RoxMTFwdHSEnp6emiJULQcHB4wePRqjR49Wdyh5Kg+vBxERUVnBX3ouOXLb330fjzAQEREREZEsFgxERERERCSr1N1WlYpHbGysukMgIiIiohKARxiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZPF3GIrQth8uFOv6+sxyy3fbLl26ID09HUeOHMk2788//0Tr1q1x9epVODs7FyiG2NhYODo64sqVK2jYsGGBliUiIiKiko9HGMoJf39/HD9+HA8ePMg2b/369XB1dS1wsfDmzZuiCo+IiIiISigWDOXEp59+CktLSwQGBipNT05Oxq5du+Dv74+zZ8+iVatW0NfXh52dHUaOHImUlBSprYODA2bOnIn+/fvD2NgYQ4cOhaOjIwCgUaNGUCgU8PT0lNr/97//hZOTE/T09FCnTh38+uuv0rxBgwbB2dkZaWlpAN4VH40aNUL//v1VlwQiIiIiKjAWDOWElpYW+vfvj8DAQAghpOm7du1CRkYG3N3d0bFjR3Tr1g3Xrl3Djh07cPbsWQwfPlypn/nz58PFxQVXrlzB5MmTcfHiRQDAiRMnEBcXhz179gAAtmzZgilTpuDnn39GREQEZs2ahcmTJ2PDhg0AgKVLlyIlJQUTJkwAAEyaNAnPnz/H8uXLiyMdRERERJRPvIahHBk0aBDmzZuH4OBg6UjA+vXr0a1bNyxbtgx9+/bF6NGjAQA1a9bE0qVL4eHhgZUrV0JPTw8A0LZtW4wdO1bqU1NTEwBgbm4Oa2trafrUqVOxYMECfPHFFwAAR0dH3Lp1C6tXr4afnx8MDQ2xefNmeHh4wMjICIsXL8apU6dgbGxcDJkgIiIiovziEYZypE6dOmjevDnWrVsHAIiKisKff/4Jf39/XL16FYGBgTA0NJQe3t7eyMzMRExMjNSHq6trnutJSUlBdHQ0/P39lfr76aefEB0dLbVzd3fHuHHjMHPmTIwdOxYtW7Ys+kETERER0UfhEYZyxt/fHyNGjMCKFSuwfv16VK9eHR4eHkhOTsZXX32FkSNHZlumatWq0t8VKlTIcx3JyckAgDVr1sDNTflOTllHJAAgMzMT586dg6amJqKiogo7JCIiIiJSIRYM5UzPnj0xatQobN26FRs3bsSwYcOgUCjQuHFj3Lp1CzVq1ChQfzo6OgCAjIwMaZqVlRVsbW3xzz//oG/fvrLLzps3D7dv30ZwcDC8vb2xfv16DBw4sHADIyIiIiKVYMFQzhgaGqJXr16YOHEiXrx4gQEDBgAAvv/+e/znP//B8OHDMXjwYFSoUAG3bt3C8ePHc70QuVKlStDX18eRI0dQpUoV6OnpwcTEBNOnT8fIkSNhYmKCjh07Ii0tDaGhoXj27BnGjBmDK1euYMqUKfj999/RokULLFy4EKNGjYKHhweqVatWTNkgIiIiorywYChCBfkhNXXy9/fH2rVr8cknn8DW1hYA4OzsjODgYEyaNAmtWrWCEALVq1dHr169cu1LS0sLS5cuxYwZMzBlyhS0atUKp0+fxuDBg2FgYIB58+Zh/PjxqFChAho0aIDRo0fj9evX6NevHwYMGIAuXboAAIYOHYqDBw/C19cXZ86cUTp1iYiIiIjURyHev8cmSV68eAETExMkJSVlu3PP69evERMTA0dHR+nuQaQ+fD2IiIhKD7/tT9QdAgBgQ29zdYegdrnt776Pd0kiIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYPgIvF68ZODrQERERKQ6LBgKIeuWn2/evFFzJAQAqampAABtbW01R0JERERU9vB3GApBS0sLBgYGePToEbS1taGhwbpLHYQQSE1NRWJiIkxNTfnbDUREREQqwIKhEBQKBWxsbBATE4O7d++qO5xyz9TUFNbW1uoOg4iIiKhMYsFQSDo6OqhZsyZPS1IzbW1tHlkgIiIiUiEWDB9BQ0ODvyxMRERERGUaT74nIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWaWuYJg9ezaaNm0KIyMjVKpUCT4+PoiMjFRq8/r1awQEBMDc3ByGhobo1q0bEhIS1BQxEREREVHpVeoKhuDgYAQEBCAkJATHjx9Heno6OnTogJSUFKnNt99+i//973/YtWsXgoOD8fDhQ3zxxRdqjJqIiIiIqHTSUncABXXkyBGl54GBgahUqRLCwsLQunVrJCUlYe3atdi6dSvatm0LAFi/fj2cnJwQEhKC//znP+oIm4iIiIioVCp1Rxg+lJSUBACoWLEiACAsLAzp6enw8vKS2tSpUwdVq1bF+fPnZftJS0vDixcvlB5EREREROVdqS4YMjMzMXr0aLRo0QL169cHAMTHx0NHRwempqZKba2srBAfHy/b1+zZs2FiYiI97OzsVBk6EREREVGpUKoLhoCAANy4cQPbt2//6L4mTpyIpKQk6XH//v0iiJCIiIiIqHQrddcwZBk+fDgOHDiAM2fOoEqVKtJ0a2trvHnzBs+fP1c6ypCQkABra2vZ/nR1daGrq6vKkImIiIiISp1Sd4RBCIHhw4dj7969OHnyJBwdHZXmN2nSBNra2ggKCpKmRUZG4t69e3B3dy/ucImIiIiISrVSd4QhICAAW7duxf79+2FkZCRdl2BiYgJ9fX2YmJjA398fY8aMQcWKFWFsbIwRI0bA3d2dd0giIiIiIiqgUlcwrFy5EgDg6empNH39+vUYMGAAAGDRokXQ0NBAt27dkJaWBm9vb/z666/FHCkRERERUelX6goGIUSebfT09LBixQqsWLGiGCIiIiIiIiq7St01DEREREREVHxYMBARERERkSwWDEREREREJIsFAxERERERyWLBQEREREREslgwEBERERGRLBYMREREREQkiwUDERERERHJYsFARERERESyWDAQEREREZEsFgxERERERCSLBQMREREREcliwUBERERERLJYMBARERERkSwWDEREREREJIsFAxERERERyWLBQEREREREslgwEBERERGRLBYMREREREQkiwUDERERERHJYsFARERERESyWDAQEREREZEsFgxERERERCSLBQMREREREcliwUBERERERLJYMBARERERkSwtdQdARERERFTctv1wQd0hAAD6zHJTdwh54hEGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpJVKguGM2fOoEuXLrC1tYVCocC+ffuU5g8YMAAKhULp0bFjR/UES0RERERUipXKgiElJQUuLi5YsWKFbJuOHTsiLi5Oemzbtq0YIyQiIiIiKhu01B1AYXTq1AmdOnXKtY2uri6sra2LKSIiIiIiorKpVB5hyI/Tp0+jUqVKqF27NoYNG4YnT57k2j4tLQ0vXrxQehARERERlXel8ghDXjp27IgvvvgCjo6OiI6Oxg8//IBOnTrh/Pnz0NTUzHGZ2bNnY/r06cUcKREREZU22364oO4QAAB9ZrmpOwQqJ8pkwdC7d2/p7wYNGsDZ2RnVq1fH6dOn0a5duxyXmThxIsaMGSM9f/HiBezs7FQeKxERERFRSVZmT0l6X7Vq1WBhYYGoqCjZNrq6ujA2NlZ6EBERERGVd8VWMFSrVi3H6wieP3+OatWqqXTdDx48wJMnT2BjY6PS9RARERERlTXFdkpSbGwsMjIysk1PS0vDv//+W6C+kpOTlY4WxMTEIDw8HBUrVkTFihUxffp0dOvWDdbW1oiOjsZ3332HGjVqwNvb+6PHQURERERUnqi8YPjjjz+kv48ePQoTExPpeUZGBoKCguDg4FCgPkNDQ9GmTRvpeda1B35+fli5ciWuXbuGDRs24Pnz57C1tUWHDh0wc+ZM6OrqftxgiIiIiIjKGZUXDD4+PgAAhUIBPz8/pXna2tpwcHDAggULCtSnp6cnhBCy848ePVrgOImIiIiIKDuVFwyZmZkAAEdHR1y6dAkWFhaqXiURERERERWRYruGISYmprhWRURERERERaRYf4chKCgIQUFBSExMlI48ZFm3bl1xhkJERERERPlQbAXD9OnTMWPGDLi6usLGxgYKhaK4Vk1ERERERIVUbAXDqlWrEBgYCF9f3+JaJRERERERfaRi++G2N2/eoHnz5sW1OiIiIiIiKgLFVjAMHjwYW7duLa7VERERERFRESi2U5Jev36N3377DSdOnICzszO0tbWV5i9cuLC4QiEiIiIionwqtoLh2rVraNiwIQDgxo0bSvN4ATQRERERUclUbAXDqVOnimtVRERERERURIrtGgYiIiIiIip9iu0IQ5s2bXI99ejkyZPFFQoREREREeVTsRUMWdcvZElPT0d4eDhu3LgBPz+/4gqDiIiIiIgKoNgKhkWLFuU4fdq0aUhOTi6uMIiIiIiIqADUfg1Dv379sG7dOnWHQUREREREOSi2Iwxyzp8/Dz09PXWHQURU4m374YK6QwAA9Jnlpu4QiIioGBVbwfDFF18oPRdCIC4uDqGhoZg8eXJxhUFERERERAVQbAWDiYmJ0nMNDQ3Url0bM2bMQIcOHYorDCIiIiIiKoBiKxjWr19fXKsiIiIiIqIiUuzXMISFhSEiIgIAUK9ePTRq1Ki4QyAiIiIionwqtoIhMTERvXv3xunTp2FqagoAeP78Odq0aYPt27fD0tKyuEIhIiIiIqJ8Krbbqo4YMQIvX77EzZs38fTpUzx9+hQ3btzAixcvMHLkyOIKg4iIiIiICqDYjjAcOXIEJ06cgJOTkzStbt26WLFiBS96JiIiIiIqoYrtCENmZia0tbWzTdfW1kZmZmZxhUFERERERAVQbAVD27ZtMWrUKDx8+FCa9u+//+Lbb79Fu3btiisMIiIiIiIqgGIrGJYvX44XL17AwcEB1atXR/Xq1eHo6IgXL15g2bJlxRUGEREREREVQLFdw2BnZ4fLly/jxIkTuH37NgDAyckJXl5exRUCEREREREVkMqPMJw8eRJ169bFixcvoFAo0L59e4wYMQIjRoxA06ZNUa9ePfz555+qDoOIiIiIiApB5QXD4sWLMWTIEBgbG2ebZ2Jigq+++goLFy5UdRhERERERFQIKi8Yrl69io4dO8rO79ChA8LCwlQdBhERERERFYLKC4aEhIQcb6eaRUtLC48ePVJ1GEREREREVAgqLxgqV66MGzduyM6/du0abGxsVB0GEREREREVgsoLhk8++QSTJ0/G69evs8179eoVpk6dik8//VTVYRARERERUSGo/LaqP/74I/bs2YNatWph+PDhqF27NgDg9u3bWLFiBTIyMjBp0iRVh0FERERERIWg8oLBysoKf/31F4YNG4aJEydCCAEAUCgU8Pb2xooVK2BlZaXqMIiIiIiIqBCK5Yfb7O3tcejQITx79gxRUVEQQqBmzZowMzMrjtUTEREREVEhFdsvPQOAmZkZmjZtWpyrJCIiIiKij1CsBQMRUWnkt/2JukMAAMj/og0REZHqqPwuSUREREREVHqxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWaWyYDhz5gy6dOkCW1tbKBQK7Nu3T2m+EAJTpkyBjY0N9PX14eXlhTt37qgnWCIiIiKiUqxUFgwpKSlwcXHBihUrcpw/d+5cLF26FKtWrcKFCxdQoUIFeHt74/Xr18UcKRERERFR6VYqf7itU6dO6NSpU47zhBBYvHgxfvzxR3Tt2hUAsHHjRlhZWWHfvn3o3bt3cYZKRERERFSqlcojDLmJiYlBfHw8vLy8pGkmJiZwc3PD+fPn1RgZEREREVHpUyqPMOQmPj4eAGBlZaU03crKSpqXk7S0NKSlpUnPX7x4oZoAiYiIiIhKkTJ3hKGwZs+eDRMTE+lhZ2en7pCIiIiIiNSuzBUM1tbWAICEhASl6QkJCdK8nEycOBFJSUnS4/79+yqNk4iIiIioNChzBYOjoyOsra0RFBQkTXvx4gUuXLgAd3d32eV0dXVhbGys9CAiIiIiKu9K5TUMycnJiIqKkp7HxMQgPDwcFStWRNWqVTF69Gj89NNPqFmzJhwdHTF58mTY2trCx8dHfUETEREREZVCpbJgCA0NRZs2baTnY8aMAQD4+fkhMDAQ3333HVJSUjB06FA8f/4cLVu2xJEjR6Cnp6eukImIiIiISqVSWTB4enpCCCE7X6FQYMaMGZgxY0YxRkVEREREVPaUuWsYiIiIiIio6LBgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhklcq7JBEREVH547f9ibpDAAB0VHcARMWMRxiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWVrqDoCIVG/bDxfUHQIAoM8sN3WHQERERAXEIwxERERERCSLBQMREREREcliwUBERERERLJYMBARERERkSwWDEREREREJIsFAxERERERySqTBcO0adOgUCiUHnXq1FF3WEREREREpU6Z/R2GevXq4cSJE9JzLa0yO1QiIiIiIpUps3vRWlpasLa2VncYRERERESlWpk8JQkA7ty5A1tbW1SrVg19+/bFvXv3cm2flpaGFy9eKD2IiIiIiMq7MnmEwc3NDYGBgahduzbi4uIwffp0tGrVCjdu3ICRkVGOy8yePRvTp08v5kiprPPb/kTdIQAAOqo7ACIiIiq1yuQRhk6dOqFHjx5wdnaGt7c3Dh06hOfPn2Pnzp2yy0ycOBFJSUnS4/79+8UYMRERERFRyVQmjzB8yNTUFLVq1UJUVJRsG11dXejq6hZjVEREREREJV+ZPMLwoeTkZERHR8PGxkbdoRARERERlSplsmAYN24cgoODERsbi7/++guff/45NDU10adPH3WHRkRERERUqpTJU5IePHiAPn364MmTJ7C0tETLli0REhICS0tLdYdGRERERFSqlMmCYfv27eoOgYiIiIioTCiTpyQREREREVHRYMFARERERESyWDAQEREREZEsFgxERERERCSrTF70XBb4bX+i7hAkG3qbqzsEIiIiIlITHmEgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWVrqDoBKvm0/XFB3CACAPrPc1B0CEdFHKSmfpwA/U4ko/3iEgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlla6g6AiIhI1fy2P1F3CACAjuoOgIioEHiEgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZZbpgWLFiBRwcHKCnpwc3NzdcvHhR3SEREREREZUqZbZg2LFjB8aMGYOpU6fi8uXLcHFxgbe3NxITE9UdGhERERFRqVFmC4aFCxdiyJAhGDhwIOrWrYtVq1bBwMAA69atU3doRERERESlRpn8pec3b94gLCwMEydOlKZpaGjAy8sL58+fz3GZtLQ0pKWlSc+TkpIAAC9evFBtsDLepL5Uy3pzkpqWou4QAKjvtfgYJeV15Gv4cfg6KiuNryNfw+z4OhZeSXkdS+NrCPB1/JA6X8esdQshcm2nEHm1KIUePnyIypUr46+//oK7u7s0/bvvvkNwcDAuXLiQbZlp06Zh+vTpxRkmEREREZHa3b9/H1WqVJGdXyaPMBTGxIkTMWbMGOl5ZmYmnj59CnNzcygUCjVGpl4vXryAnZ0d7t+/D2NjY3WHQ4XA17Bs4OtY+vE1LBv4OpYNfB3fEULg5cuXsLW1zbVdmSwYLCwsoKmpiYSEBKXpCQkJsLa2znEZXV1d6OrqKk0zNTVVVYiljrGxcbl+Q5UFfA3LBr6OpR9fw7KBr2PZwNcRMDExybNNmbzoWUdHB02aNEFQUJA0LTMzE0FBQUqnKBERERERUe7K5BEGABgzZgz8/Pzg6uqKZs2aYfHixUhJScHAgQPVHRoRERERUalRZguGXr164dGjR5gyZQri4+PRsGFDHDlyBFZWVuoOrVTR1dXF1KlTs52uRaUHX8Oyga9j6cfXsGzg61g28HUsmDJ5lyQiIiIiIioaZfIaBiIiIiIiKhosGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgIFkrVqyAg4MD9PT04ObmhosXL6o7JCqAM2fOoEuXLrC1tYVCocC+ffvUHRIV0OzZs9G0aVMYGRmhUqVK8PHxQWRkpLrDogJauXIlnJ2dpR+Icnd3x+HDh9UdFn2EOXPmQKFQYPTo0eoOhQpg2rRpUCgUSo86deqoO6xSgQUD5WjHjh0YM2YMpk6disuXL8PFxQXe3t5ITExUd2iUTykpKXBxccGKFSvUHQoVUnBwMAICAhASEoLjx48jPT0dHTp0QEpKirpDowKoUqUK5syZg7CwMISGhqJt27bo2rUrbt68qe7QqBAuXbqE1atXw9nZWd2hUCHUq1cPcXFx0uPs2bPqDqlU4G1VKUdubm5o2rQpli9fDuDdL2Xb2dlhxIgRmDBhgpqjo4JSKBTYu3cvfHx81B0KfYRHjx6hUqVKCA4ORuvWrdUdDn2EihUrYt68efD391d3KFQAycnJaNy4MX799Vf89NNPaNiwIRYvXqzusCifpk2bhn379iE8PFzdoZQ6PMJA2bx58wZhYWHw8vKSpmloaMDLywvnz59XY2RE5VtSUhKAdzubVDplZGRg+/btSElJgbu7u7rDoQIKCAhA586dlf4/Uuly584d2Nraolq1aujbty/u3bun7pBKhTL7S89UeI8fP0ZGRka2X8W2srLC7du31RQVUfmWmZmJ0aNHo0WLFqhfv766w6ECun79Otzd3fH69WsYGhpi7969qFu3rrrDogLYvn07Ll++jEuXLqk7FCokNzc3BAYGonbt2oiLi8P06dPRqlUr3LhxA0ZGRuoOr0RjwUBEVAoEBATgxo0bPN+2lKpduzbCw8ORlJSE33//HX5+fggODmbRUErcv38fo0aNwvHjx6Gnp6fucKiQOnXqJP3t7OwMNzc32NvbY+fOnTw9MA8sGCgbCwsLaGpqIiEhQWl6QkICrK2t1RQVUfk1fPhwHDhwAGfOnEGVKlXUHQ4Vgo6ODmrUqAEAaNKkCS5duoQlS5Zg9erVao6M8iMsLAyJiYlo3LixNC0jIwNnzpzB8uXLkZaWBk1NTTVGSIVhamqKWrVqISoqSt2hlHi8hoGy0dHRQZMmTRAUFCRNy8zMRFBQEM+5JSpGQggMHz4ce/fuxcmTJ+Ho6KjukKiIZGZmIi0tTd1hUD61a9cO169fR3h4uPRwdXVF3759ER4ezmKhlEpOTkZ0dDRsbGzUHUqJxyMMlKMxY8bAz88Prq6uaNasGRYvXoyUlBQMHDhQ3aFRPiUnJyt9axITE4Pw8HBUrFgRVatWVWNklF8BAQHYunUr9u/fDyMjI8THxwMATExMoK+vr+boKL8mTpyITp06oWrVqnj58iW2bt2K06dP4+jRo+oOjfLJyMgo27VDFSpUgLm5Oa8pKkXGjRuHLl26wN7eHg8fPsTUqVOhqamJPn36qDu0Eo8FA+WoV69eePToEaZMmYL4+Hg0bNgQR44cyXYhNJVcoaGhaNOmjfR8zJgxAAA/Pz8EBgaqKSoqiJUrVwIAPD09laavX78eAwYMKP6AqFASExPRv39/xMXFwcTEBM7Ozjh69Cjat2+v7tCIypUHDx6gT58+ePLkCSwtLdGyZUuEhITA0tJS3aGVePwdBiIiIiIiksVrGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiOijDRgwAD4+Ph/Vx+nTp6FQKPD8+XMAQGBgIExNTT86ttjYWCgUCoSHh390X0RE5RELBiKiInT+/Hloamqic+fO6g6lSK1ZswYuLi4wNDSEqakpGjVqhNmzZ0vzlyxZgsDAwI9aR/PmzREXFwcTE5OPjFaZnZ0d4uLiUL9+fQDZC5OPERwcjLZt26JixYowMDBAzZo14efnhzdv3nx030REJQULBiKiIrR27VqMGDECZ86cwcOHD9UdToHI7eSuW7cOo0ePxsiRIxEeHo5z587hu+++Q3JystTGxMTko48G6OjowNraGgqF4qP6ed+bN2+gqakJa2traGlpFVm/AHDr1i107NgRrq6uOHPmDK5fv45ly5ZBR0cHGRkZRbquLEIIvH37ViV9ExHJYcFARFREkpOTsWPHDgwbNgydO3fO9o171jfbQUFBcHV1hYGBAZo3b47IyEipzdWrV9GmTRsYGRnB2NgYTZo0QWhoKIQQsLS0xO+//y61bdiwIWxsbKTnZ8+eha6uLlJTUwEAz58/x+DBg2FpaQljY2O0bdsWV69eldpPmzYNDRs2xH//+184OjpCT08vx3H98ccf6NmzJ/z9/VGjRg3Uq1cPffr0wc8//yy1+fCUJE9PT4wYMQKjR4+GmZkZrKyssGbNGqSkpGDgwIEwMjJCjRo1cPjw4Wz5kfvmPzo6Gl27doWVlRUMDQ3RtGlTnDhxQqmNg4MDZs6cif79+8PY2BhDhw5VOiUpNjYWbdq0AQCYmZlBoVBgwIAB2LhxI8zNzZGWlqbUn4+PD3x9fXOM59ixY7C2tsbcuXNRv359VK9eHR07dsSaNWugr68vtTt37hw8PT1hYGAAMzMzeHt749mzZwCAtLQ0jBw5EpUqVYKenh5atmyJS5cuZcvJ4cOH0aRJE+jq6uLs2bPIzMzE7Nmz4ejoCH19fbi4uChtG0RERYkFAxFREdm5cyfq1KmD2rVro1+/fli3bh2EENnaTZo0CQsWLEBoaCi0tLQwaNAgaV7fvn1RpUoVXLp0CWFhYZgwYQK0tbWhUCjQunVrnD59GgDw7NkzRERE4NWrV7h9+zaAd6fHNG3aFAYGBgCAHj16IDExEYcPH0ZYWBgaN26Mdu3a4enTp9L6oqKisHv3buzZs0f2HH9ra2uEhITg7t27BcrHhg0bYGFhgYsXL2LEiBEYNmwYevTogebNm+Py5cvo0KEDfH19pQInL8nJyfjkk08QFBSEK1euoGPHjujSpQvu3bun1G7+/PlwcXHBlStXMHnyZKV5dnZ22L17NwAgMjIScXFxWLJkCXr06IGMjAz88ccfUtvExEQcPHhQ6fV5n7W1NeLi4nDmzBnZmMPDw9GuXTvUrVsX58+fx9mzZ9GlSxfpCMR3332H3bt3Y8OGDbh8+TJq1KgBb29vpdcIACZMmIA5c+YgIiICzs7OmD17NjZu3IhVq1bh5s2b+Pbbb9GvXz8EBwfnK5dERAUiiIioSDRv3lwsXrxYCCFEenq6sLCwEKdOnZLmnzp1SgAQJ06ckKYdPHhQABCvXr0SQghhZGQkAgMDc+x/6dKlol69ekIIIfbt2yfc3NxE165dxcqVK4UQQnh5eYkffvhBCCHEn3/+KYyNjcXr16+V+qhevbpYvXq1EEKIqVOnCm1tbZGYmJjruB4+fCj+85//CACiVq1aws/PT+zYsUNkZGRIbfz8/ETXrl2l5x4eHqJly5bS87dv34oKFSoIX19faVpcXJwAIM6fP6+Un2fPngkhhFi/fr0wMTHJNbZ69eqJZcuWSc/t7e2Fj4+PUpuYmBgBQFy5ciXH9WQZNmyY6NSpk/R8wYIFolq1aiIzMzPHdb99+1YMGDBAABDW1tbCx8dHLFu2TCQlJUlt+vTpI1q0aJHj8snJyUJbW1ts2bJFmvbmzRtha2sr5s6dqxTrvn37pDavX78WBgYG4q+//lLqz9/fX/Tp0yfHdRERfQweYSAiKgKRkZG4ePEi+vTpAwDQ0tJCr169sHbt2mxtnZ2dpb+zTilKTEwEAIwZMwaDBw+Gl5cX5syZg+joaKmth4cHbt26hUePHiE4OBienp7w9PTE6dOnkZ6ejr/++guenp4A3p3alJycDHNzcxgaGkqPmJgYpT7t7e1haWmZ69hsbGxw/vx5XL9+HaNGjcLbt2/h5+eHjh07IjMzU3a598epqakJc3NzNGjQQJpmZWWlNPa8JCcnY9y4cXBycoKpqSkMDQ0RERGR7QiDq6trvvr70JAhQ3Ds2DH8+++/AN7dpWnAgAGy11Roampi/fr1ePDgAebOnYvKlStj1qxZqFevHuLi4gD8/xGGnERHRyM9PR0tWrSQpmlra6NZs2aIiIiQHVNUVBRSU1PRvn17pdd248aNSq8tEVFRKdorwIiIyqm1a9fi7du3sLW1laYJIaCrq4vly5cr3flHW1tb+jtrZzRrx3vatGn48ssvcfDgQRw+fBhTp07F9u3b8fnnn6NBgwaoWLEigoODERwcjJ9//hnW1tb45ZdfcOnSJaSnp6N58+YA3u1c29jYSKcwve/9i5MrVKiQ7zHWr18f9evXxzfffIOvv/4arVq1QnBwsHRNwIfeH2fWWHMbe17GjRuH48ePY/78+ahRowb09fXRvXv3bBdrF2RM72vUqBFcXFywceNGdOjQATdv3sTBgwfzXK5y5crw9fWFr68vZs6ciVq1amHVqlWYPn260rUMH+P9MWVdbH7w4EFUrlxZqZ2urm6RrI+I6H0sGIiIPtLbt2+xceNGLFiwAB06dFCa5+Pjg23btuHrr7/Od3+1atVCrVq18O2336JPnz5Yv349Pv/8cygUCrRq1Qr79+/HzZs30bJlSxgYGCAtLQ2rV6+Gq6urtGPZuHFjxMfHQ0tLCw4ODkU5XABA3bp1AQApKSlF3recc+fOYcCAAfj8888BvNtxjo2NLXA/Ojo6AJDjnYwGDx6MxYsX499//4WXlxfs7OwK1LeZmRlsbGykvDg7OyMoKAjTp0/P1rZ69erQ0dHBuXPnYG9vDwBIT0/HpUuXMHr0aNl11K1bF7q6urh37x48PDwKFB8RUWGwYCAi+kgHDhzAs2fP4O/vn+03BLp164a1a9fmq2B49eoVxo8fj+7du8PR0REPHjzApUuX0K1bN6mNp6cnxo4dC1dXVxgaGgIAWrdujS1btmD8+PFSOy8vL7i7u8PHxwdz585FrVq18PDhQxw8eBCff/55gU7bGTZsGGxtbdG2bVtUqVIFcXFx+Omnn2BpaQl3d/d89/OxatasiT179qBLly5QKBSYPHlyvo9OvM/e3h4KhQIHDhzAJ598An19fSmXX375JcaNG4c1a9Zg48aNufazevVqhIeH4/PPP0f16tXx+vVrbNy4ETdv3sSyZcsAABMnTkSDBg2kozI6Ojo4deoUevToAQsLCwwbNgzjx49HxYoVUbVqVcydOxepqanw9/eXXa+RkRHGjRuHb7/9FpmZmWjZsiWSkpJw7tw5GBsbw8/Pr8A5ISLKDa9hICL6SGvXroWXl1eOPzjWrVs3hIaG4tq1a3n2o6mpiSdPnqB///6oVasWevbsiU6dOil9O+3h4YGMjAzpWgXgXRHx4TSFQoFDhw6hdevWGDhwIGrVqoXevXvj7t270rUD+eXl5YWQkBD06NEDtWrVQrdu3aCnp4egoCCYm5sXqK+PsXDhQpiZmaF58+bo0qULvL290bhx4wL3U7lyZUyfPh0TJkyAlZUVhg8fLs0zMTFBt27dYGhomOcvVzdr1gzJycn4+uuvUa9ePXh4eCAkJAT79u2TvvmvVasWjh07hqtXr6JZs2Zwd3fH/v37pd+EmDNnDrp16wZfX180btwYUVFROHr0KMzMzHJd98yZMzF58mTMnj0bTk5O6NixIw4ePAhHR8cC54OIKC8KIXK45x8REVE51a5dO9SrVw9Lly5VdyhERCUCCwYiIiK8+22L06dPo3v37rh16xZq166t7pCIiEoEXsNARESEd3dJevbsGX755RcWC0RE7+ERBiIiIiIiksWLnomIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIlla6g6gJLt37x4eP36s7jDKpLS0NOjq6qo7jDKL+VUd5la1mF/VYW5Vi/lVHeZWtSwsLFC1atVc27BgkHHv3j04OTkhNTVV3aGUSZqamsjIyFB3GGUW86s6zK1qMb+qw9yqFvOrOsytahkYGCAiIiLXooEFg4zHjx8jNTUVmzdvhpOTk7rDKVMOHTqEyZMnM7cqwvyqDnOrWsyv6jC3qsX8qg5zq1oRERHo168fHj9+zILhYzg5OaFx48bqDqNMiYiIAMDcqgrzqzrMrWoxv6rD3KoW86s6zG3JwIueiYiIiIhIFgsGIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiKrFiY2OhUCgQHh6u7lBKrdOnT0OhUOD58+fqDoWowDw9PTF69GjpuYODAxYvXqy2eIjKKxYMxJ0yKnYDBgyAj4+PusMoF5o3b464uDiYmJioO5QSa/bs2WjatCmMjIxQqVIl+Pj4IDIyUqmNp6cnFAqF0uPrr7+W5j99+hRdunSBoaEhGjVqhCtXrigtHxAQgAULFhTLeMqyS5cuYejQoeoOo8RwcHDItl0qFAoEBAQA4HZLRYcFAxFRGaajowNra2soFAp1h1JiBQcHIyAgACEhITh+/DjS09PRoUMHpKSkKLUbMmQI4uLipMfcuXOleT///DNevnyJy5cvw9PTE0OGDJHmhYSE4MKFC0rflFPhWFpawsDAQN1hlBiXLl1S2iaPHz8OAOjRo4fUhtstFQUWDCVAZmYm5s6dixo1akBXVxdVq1bFzz//DAC4fv062rZtC319fZibm2Po0KFITk6Wlv3wcC0A+Pj4YMCAAdJzBwcHzJo1C4MGDYKRkRGqVq2K3377TZrv6OgIAGjUqBEUCgU8PT1VNtbi5OnpiREjRmD06NEwMzODlZUV1qxZg5SUFAwcOBBGRkaoUaMGDh8+LC1z48YNdOrUCYaGhrCysoKvry8eP34szT9y5AhatmwJU1NTmJub49NPP0V0dLQ0P+tozZ49e9CmTRsYGBjAxcUF58+fL9axlxS///47GjRoIG2/Xl5eGD9+PDZs2ID9+/dL33idPn0aAHDx4kU0atQIenp6cHV1zfZtFxV8u/7wlKTAwECYmpri6NGjcHJygqGhITp27Ii4uDg1jkq9jhw5ggEDBqBevXpwcXFBYGAg7t27h7CwMKV2BgYGsLa2lh7GxsbSvIiICPTu3Ru1atXC0KFDpR+bSk9Px9dff41Vq1ZBU1OzWMelSqr4fE1JSUH//v1haGgIGxubHL/Z/vCUpIULF6JBgwaoUKEC7Ozs8M033yj9jyzr27ulpaXSNnngwAFUr14dHh4eUhtut4VX0O08IyMD/v7+cHR0hL6+PmrXro0lS5ZI/b1+/Rr16tVTOkoWHR0NIyMjrFu3rtjHVxAsGEqAiRMnYs6cOZg8eTJu3bqFrVu3wsrKCikpKfD29oaZmRkuXbqEXbt24cSJExg+fHiB17FgwQJpB+ybb77BsGHDpEPuFy9eBACcOHECcXFx2LNnT5GOT502bNgACwsLXLx4ESNGjMCwYcPQo0cPNG/eHJcvX0aHDh3g6+uL1NRUPH/+HG3btkWjRo0QGhqKI0eOICEhAT179pT6S0lJwZgxYxAaGoqgoCBoaGjg888/R2ZmptJ6J02ahHHjxiE8PBy1atVCnz598Pbt2+IevlrFxcWhT58+GDRoECIiInD69Gl88cUXmDp1Knr27Cn9046Li0Pz5s2RnJyMTz/9FHXr1kVYWBimTZuGcePGqXsYJVJBtuucpKamYv78+di0aRPOnDmDe/fuMdfvSUpKAgBUrFhRafqWLVtgYWGB+vXrY+LEiUr5dXFxwcmTJ/H27VscPXoUzs7OAIC5c+fC09MTrq6uxTeAYlLUn6/jx49HcHAw9u/fj2PHjuH06dO4fPlyrjFoaGhg6dKluHnzJjZs2ICTJ0/iu+++U2pTXrb3N2/eYPPmzRg0aJDSEUVutx+nINt5ZmYmqlSpgl27duHWrVuYMmUKfvjhB+zcuRMAoKenhy1btkhfmmVkZKBfv35o3749Bg0apOaR5kFQjsLCwgQAERYWptL1vHjxQujq6oo1a9Zkm/fbb78JMzMzkZycLE07ePCg0NDQEPHx8UIIITw8PMSoUaOUluvatavw8/OTntvb24t+/fpJzzMzM0WlSpXEypUrhRBCxMTECADiypUrRTewXGzevLlYcuvh4SFatmwpPX/79q2oUKGC8PX1labFxcUJAOL8+fNi5syZokOHDkp93L9/XwAQkZGROa7j0aNHAoC4fv26EOL/c/nf//5XanPz5k0BQERERBTl8GQVV37zkvUeio2NzTbPz89PdO3aVWna6tWrhbm5uXj16pU0beXKlcW6bealJOS2oNv1qVOnBADx7NkzIYQQ69evFwBEVFSU1H7FihXCysqq2MYgpyTkNyMjQ3Tu3Fm0aNFCafrq1avFkSNHxLVr18TmzZtF5cqVxeeffy7Nf/78uejTp4+oWrWqaN26tbh586b4+++/Rc2aNcXjx4/FV199JRwdHUWPHj3E8+fPi3tYRZ7bov58ffnypdDR0RE7d+6U5j958kTo6+sr/Y+zt7cXixYtko1r165dwtzcXHpeXNt7Sdh2d+zYITQ1NcW///4rTSvt260Q6s1tQbfznAQEBIhu3bopTZs7d66wsLAQw4cPFzY2NuLx48eqGUA+5Hd/V6v4ShPKSUREBNLS0tCuXbsc57m4uKBChQrStBYtWiAzMxORkZGwsrLK93qyvjUAAIVCAWtrayQmJn5c8KXA++PW1NSEubk5GjRoIE3LymFiYiKuXr2KU6dOwdDQMFs/0dHRqFWrFu7cuYMpU6bgwoULePz4sXRk4d69e6hfv36O67WxsZHWUadOnaIdYAnm4uKCdu3aoUGDBvD29kaHDh3QvXt3mJmZ5dg+IiICzs7O0NPTk6a5u7sXV7ilSkG26/dPP8hiYGCA6tWrS89tbGzKxedBfgQEBODGjRs4e/as0vT3TyFo0KABbGxs0K5dO0RHR6N69eowMTHB1q1blZZp27Yt5s2bhy1btuCff/5BZGQkhgwZghkzZpSJC0mL8vP11atXePPmDdzc3KTpFStWRO3atXON4cSJE5g9ezZu376NFy9e4O3bt3j9+jVSU1Olax3Ky/a+du1adOrUCba2ttI0brcfryDbOQCsWLEC69atw71796TtumHDhkp9jh07Fvv27cPy5ctx+PBhmJubq34gH4mnJKmZvr7+Ry2voaEBIYTStPT09GzttLW1lZ4rFIpsp9GURTmN+/1pWYdtMzMzkZycjC5duiA8PFzpcefOHbRu3RoA0KVLFzx9+hRr1qzBhQsXcOHCBQDvDgXLrff9dZQnmpqaOH78OA4fPoy6deti2bJlqF27NmJiYtQdWqlXkO06v8t/+DlSHg0fPhwHDhzAqVOnUKVKlVzbZu3YRkVF5Th//fr1MDU1RdeuXXH69Gn4+PhAW1sbPXr0kK7ZKe2K+vO1oGJjY/Hpp5/C2dkZu3fvRlhYGFasWAFA+TO5PGzvd+/exYkTJzB48OBc23G7LbiCbOfbt2/HuHHj4O/vj2PHjiE8PBwDBw7Mto+QmJiIv//+G5qamrhz547qB1EEeIRBzWrWrAl9fX0EBQVle6M7OTkhMDAQKSkp0lGGc+fOQUNDQ/rWxdLSUunirYyMDNy4cQNt2rTJdww6OjrSsuVZ48aNsXv3bjg4OEBLK/tb48mTJ4iMjMSaNWvQqlUrAMj2LSQpUygUaNGiBVq0aIEpU6bA3t4ee/fuhY6OTrbtzcnJCZs2bcLr16+lowwhISHqCJvKGSEERowYgb179+L06dPSjSByk3Ub6qwjiO979OgRZsyYIX0+ZGRkSF/kpKenl8vP2rw+X6tXrw5tbW1cuHABVatWBQA8e/YMf//9t9IFvO8LCwtDZmYmFixYAA2Nd99/Zp0rXt6sX78elSpVQufOnXNtx+1Wtc6dO4fmzZvjm2++kaa9f2OULIMGDUKDBg3g7++PIUOGwMvLC05OTsUZaoHxCIOa6enp4fvvv8d3332HjRs3Ijo6GiEhIVi7di369u0LPT09+Pn54caNGzh16hRGjBgBX19f6RBY27ZtcfDgQRw8eBC3b9/GsGHDCvwDTZUqVYK+vr50EVrWBX/lTUBAAJ4+fYo+ffrg0qVLiI6OxtGjRzFw4EBkZGTAzMwM5ubm+O233xAVFYWTJ09izJgx6g67xLpw4QJmzZqF0NBQ3Lt3D3v27MGjR4/g5OQEBwcHXLt2DZGRkXj8+DHS09Px5ZdfQqFQYMiQIbh16xYOHTqE+fPnq3sYVA4EBARg8+bN2Lp1K4yMjBAfH4/4+Hi8evUKwLt/+DNnzkRYWBhiY2Pxxx9/oH///mjdurXS6QpZRo8ejbFjx6Jy5coA3p1KumnTJkREROC3335DixYtinV8JUFen6+Ghobw9/fH+PHjcfLkSdy4cQMDBgyQCoGc1KhRA+np6Vi2bBn++ecfbNq0CatWrSrGUZUMmZmZWL9+Pfz8/JSKMW63xa9mzZoIDQ3F0aNH8ffff2Py5Mm4dOmSUpsVK1bg/Pnz2LBhA/r27QsfHx/07ds321GIkoYFQwkwefJkjB07FlOmTIGTkxN69eqFxMREGBgY4OjRo3j69CmaNm2K7t27o127dli+fLm07KBBg+Dn54f+/fvDw8MD1apVK9DRBQDQ0tLC0qVLsXr1atja2qJr165FPcRSwdbWFufOnUNGRgY6dOiABg0aYPTo0TA1NYWGhgY0NDSwfft2hIWFoX79+vj2228xb948dYddYhkbG+PMmTP45JNPUKtWLfz4449YsGABOnXqhCFDhqB27dpwdXWFpaUlzp07B0NDQ/zvf//D9evX0ahRI0yaNAm//PKLuodB5cDKlSuRlJQET09P2NjYSI8dO3YAeHcU9sSJE+jQoQPq1KmDsWPHolu3bvjf//6Xra+jR48iKipK6RvG4cOHo1q1anBzc8ObN28wderUYhtbSZHX5ysAzJs3D61atUKXLl3g5eWFli1bokmTJrJ9uri4YOHChfjll19Qv359bNmyBbNnzy6uIZUYJ06cwL1797LdZYfbbfH76quv8MUXX6BXr15wc3PDkydPlHJ6+/ZtjB8/Hr/++ivs7OwAAL/++iseP36MyZMnqyvs/CmGC7BLpeK6S1J5VBLuJlGWMb+qw9yqFvOrOsytajG/qsPcqlZ+93d5hIGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSpZV3k3fu3buHx48fqzKWEiUiIgIAcOjQIelvKhrnzp0DwNyqCvOrOsytajG/qsPcqhbzqzrMrWrFxMTkq51CCCHyanTv3j04OTkhNTX1owMrTTQ0NJCZmanuMMok5la1mF/VYW5Vi/lVHeZWtZhf1WFuVUtTUxN//vkn3N3dZdvk6wjD48ePkZqais2bN8PJyanIAizJDh06hMmTJ5erMRcX5la1mF/VYW5Vi/lVHeZWtZhf1WFuVSsiIgL9+vWDrq5uru3yfUoSADg5OaFx48YfFVhpkXXYqzyNubgwt6rF/KoOc6tazK/qMLeqxfyqDnNbMvCiZyIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIFgsGIiIiIiKSxYKBiIiIiIhkFVnBcP/+fQwaNAi2trbQ0dGBvb09Ro0ahSdPnhTVKgrs/Pnz0NTUROfOnbPNi42NhUKhQHh4ePEHlotp06ZBoVAoPerUqSPN9/T0zDb/66+/zrVPIQSmTJkCGxsb6Ovrw8vLC3fu3FH1UNRu5cqVcHZ2hrGxMYyNjeHu7o7Dhw/na9nt27dDoVDAx8dHafqHuc96zJs3T2rz2WefoWrVqtDT04ONjQ18fX3x8OHDohxaiXDmzBl06dIFtra2UCgU2LdvX67tBwwYkGPu6tWrl2P7OXPmQKFQYPTo0UrTf/vtN3h6esLY2BgKhQLPnz8vmgGVUitWrICDgwP09PTg5uaGixcv5tp+165dqFOnDvT09NCgQQMcOnSomCItPWbPno2mTZvCyMgIlSpVgo+PDyIjI/NcjrktHLn3+oeY37zltQ+RE+Y1/8pzfoukYPjnn3/g6uqKO3fuYNu2bYiKisKqVasQFBQEd3d3PH36tChWU2Br167FiBEjcObMmVK1w1avXj3ExcVJj7NnzyrNHzJkiNL8uXPn5trf3LlzsXTpUqxatQoXLlxAhQoV4O3tjdevX6tyGGpXpUoVzJkzB2FhYQgNDUXbtm3RtWtX3Lx5M9flYmNjMW7cOLRq1SrbvPfzHhcXh3Xr1kGhUKBbt25SmzZt2mDnzp2IjIzE7t27ER0dje7duxf5+NQtJSUFLi4uWLFiRb7aL1myRCl39+/fR8WKFdGjR49sbS9duoTVq1fD2dk527zU1FR07NgRP/zww0ePobTbsWMHxowZg6lTp+Ly5ctwcXGBt7c3EhMTc2z/119/oU+fPvD398eVK1fg4+MDHx8f3Lhxo5gjL9mCg4MREBCAkJAQHD9+HOnp6ejQoQNSUlJkl2FuCye39/r7mN/8y2sf4n3Ma8GV2/yKfAgLCxMARFhYWI7zO3bsKKpUqSJSU1OVpsfFxQkDAwPx9ddfCyGEsLe3FzNmzBC9e/cWBgYGwtbWVixfvlxpmWfPngl/f39hYWEhjIyMRJs2bUR4eLg0f+rUqcLFxUVs3LhR2NvbC2NjY9GrVy/x4sULpX5evnwpDA0Nxe3bt0WvXr3Ezz//rDQ/JiZGABBXrlzJcUybN2/OdcyqkjU+OR4eHmLUqFH57i8zM1NYW1uLefPmSdOeP38udHV1xbZt2z4i0sJTV26FEMLMzEz897//lZ3/9u1b0bx5c/Hf//5X+Pn5ia5du+baX9euXUXbtm1zbbN//36hUCjEmzdvChNygakjvwDE3r17C7TM3r17hUKhELGxsUrTX758KWrWrCmOHz+e6/Z+6tQpAUA8e/ascEEXgjq33Zw0a9ZMBAQESM8zMjKEra2tmD17do7te/bsKTp37qw0zc3NTXz11VcqjTO/Slp+syQmJgoAIjg4WLYNc1tw+X2vC8H85lde+xAfKul5FaLk5FaIspnfvPbxs3z0EYanT5/i6NGj+Oabb6Cvr680z9raGn379sWOHTsghAAAzJs3Dy4uLrhy5QomTJiAUaNG4fjx49IyPXr0QGJiIg4fPoywsDA0btwY7dq1UzpKER0djX379uHAgQM4cOAAgoODMWfOHKV179y5E3Xq1EHt2rXRr18/rFu3ToqhpLtz5w5sbW1RrVo19O3bF/fu3VOav2XLFlhYWKB+/fqYOHEiUlNTZfuKiYlBfHw8vLy8pGkmJiZwc3PD+fPnVTaGkiYjIwPbt29HSkoK3N3dZdvNmDEDlSpVgr+/f559JiQk4ODBg7m2ffr0KbZs2YLmzZtDW1u7ULGXVWvXroWXlxfs7e2VpgcEBKBz585K2yxl9+bNG4SFhSnlSUNDA15eXrLv7fPnz2fLq7e3d7n6LCiMpKQkAEDFihVl2zC3BVeQ9zrzm3957UO8j3ktuPKaX62P7eDOnTsQQsDJySnH+U5OTnj27BkePXoEAGjRogUmTJgAAKhVqxbOnTuHRYsWoX379jh79iwuXryIxMRE6OrqAgDmz5+Pffv24ffff8fQoUMBAJmZmQgMDISRkREAwNfXF0FBQfj555+l9a5duxb9+vUDAHTs2BFJSUkIDg6Gp6fnxw5Zpdzc3BAYGIjatWsjLi4O06dPR6tWrXDjxg0YGRnhyy+/hL29PWxtbXHt2jV8//33iIyMxJ49e3LsLz4+HgBgZWWlNN3KykqaV5Zdv34d7u7ueP36NQwNDbF3717UrVs3x7Znz57F2rVr831dy4YNG2BkZIQvvvgi27zvv/8ey5cvR2pqKv7zn//gwIEDHzOMMufhw4c4fPgwtm7dqjR9+/btuHz5Mi5duqSmyEqPx48fIyMjI8f39u3bt3NcJj4+vtx+FhRWZmYmRo8ejRYtWqB+/fqy7Zjbginoe535zZ+89iE+xLwWTHnO70cXDFny++39h9/uuru7Y/HixQCAq1evIjk5Gebm5kptXr16hejoaOm5g4OD0gtjY2OjdM5uZGQkLl68iL179wIAtLS00KtXL6xdu7bEFwydOnWS/nZ2doabmxvs7e2xc+dO+Pv7S0UTADRo0AA2NjZo164doqOjUb16dXWEXKLVrl0b4eHhSEpKwu+//w4/Pz8EBwdnKxpevnwJX19frFmzBhYWFvnqe926dejbty/09PSyzRs/fjz8/f1x9+5dTJ8+Hf3798eBAwegUCiKZFyl3YYNG2Bqaqp0Ufn9+/elI4455ZRIHQICAnDjxo1cz1OmguF7XXXy2oegj1Oe8/vRBUONGjWgUCgQERGBzz//PNv8iIgImJmZwdLSMs++kpOTYWNjg9OnT2ebZ2pqKv394akdCoUCmZmZ0vO1a9fi7du3sLW1laYJIaCrq4vly5fDxMQkHyMrGUxNTVGrVi1ERUXlON/NzQ0AEBUVlWPBYG1tDeDd6TM2NjbS9ISEBDRs2LDoAy5hdHR0UKNGDQBAkyZNcOnSJSxZsgSrV69WahcdHY3Y2Fh06dJFmpa1TWlpaSEyMlIpv3/++SciIyOxY8eOHNdrYWEBCwsL1KpVC05OTrCzs0NISEiup0OVF0IIrFu3Dr6+vtDR0ZGmh4WFITExEY0bN5amZWRk4MyZM1i+fDnS0tKgqampjpBLJAsLC2hqaiIhIUFpekJCgvS+/5C1tXWB2pd3w4cPx4EDB3DmzBlUqVIl17bMbf4V5r3O/BZOXvsQzOvHKU/5/ehrGMzNzdG+fXv8+uuvePXqldK8+Ph4bNmyBb169ZK+WQ0JCVFqExISIp3O1LhxY8THx0NLSws1atRQeuT3W9+3b99i48aNWLBgAcLDw6XH1atXYWtri23btn3skItVcnIyoqOjlXb235d1+ozcfEdHR1hbWyMoKEia9uLFC1y4cKFc7rxmZmYiLS0t2/Q6derg+vXrStvMZ599hjZt2iA8PBx2dnZK7deuXYsmTZrAxcUlX+sEkON6y6Pg4GBERUVl+zamXbt22V4DV1dX9O3bF+Hh4SwWPqCjo4MmTZoovbczMzOlu9PlxN3dXak9ABw/frxcfhbkRgiB4cOHY+/evTh58iQcHR3zXIa5zb/CvNeZ38LJax+Cef045Sq/RXEF9d9//y0sLCxEq1atRHBwsLh37544fPiwqF+/vqhZs6Z48uSJEEJIdzX65ZdfRGRkpFi+fLnQ1NQUR44cEUK8u6NPy5YthYuLizh69KiIiYkR586dEz/88IO4dOmSECLnK9QXLVok7O3thRDv7ryio6Mjnj9/ni3O7777Tri6ugohSu5dksaOHStOnz4tjd3Ly0tYWFiIxMREERUVJWbMmCFCQ0NFTEyM2L9/v6hWrZpo3bq1Uh+1a9cWe/bskZ7PmTNHmJqaiv3794tr166Jrl27CkdHR/Hq1atiHVuW4srthAkTRHBwsIiJiRHXrl0TEyZMEAqFQhw7dkwIIYSvr6+YMGGC7PJyd0lKSkoSBgYGYuXKldnmhYSEiGXLlokrV66I2NhYERQUJJo3by6qV68uXr9+XWRjy01x5ffly5fiypUr4sqVKwKAWLhwobhy5Yq4e/euEOJd/n19fbMt169fP+Hm5pavdeR055S4uDhx5coVsWbNGgFAnDlzRly5ckX6nFGlknS3DiGE2L59u9DV1RWBgYHi1q1bYujQocLU1FTEx8cLIbJv4+fOnRNaWlpi/vz5IiIiQkydOlVoa2uL69evq2sISkpKfocNGyZMTEzE6dOnRVxcnPR4/06AzG3R+vC9zvwWTm77EEKUvrwKUXJyK0TZzG9+75JUJNcw1KxZE6GhoZg6dSp69uyJp0+fwtraGj4+Ppg6darSnSXGjh2L0NBQTJ8+HcbGxli4cCG8vb0BvDu16NChQ5g0aRIGDhyIR48ewdraGq1bt8520YicrDuv5HTaUbdu3TB37lxcu3YNxsbGRTH0IvfgwQP06dMHT548gaWlJVq2bImQkBBYWlri9evXOHHiBBYvXoyUlBTY2dmhW7du+PHHH5X6iIyMlO7qAQDfffcdUlJSMHToUDx//hwtW7bEkSNHyvy5o4mJiejfvz/i4uJgYmICZ2dnHD16FO3btwcA3Lt3DxoaBT/Itn37dggh0KdPn2zzDAwMsGfPHkydOhUpKSmwsbFBx44d8eOPP0oX8pcVoaGhaNOmjfR8zJgxAAA/Pz8EBgYiLi4u290jkpKSsHv3bixZsqTQ6121ahWmT58uPW/dujUAYP369RgwYECh+y2NevXqhUePHmHKlCmIj49Hw4YNceTIEenz8sNtvHnz5ti6dSt+/PFH/PDDD6hZsyb27duX68W85dHKlSsBINs1b+9vY8ytajG/hZPbPgTAvH6scp3foqw+8mJvby8WLVr0UX0Ul5JU0ZY1zK1qMb+qw9yqFvOrOsytajG/qsPcqlax/Q4DERERERGVXSwYiIiIiIhIVpH9DkN+xMbGFufqiIiIiIjoI/EIAxERERERyWLBQEREREREslgwEBERERGRLBYMREREREQkiwUDERERERHJKtBdkg4dOoSIiAhVxVKinDt3DkD5GnNxYW5Vi/lVHeZWtZhf1WFuVYv5VR3mVrViYmLy1U4hhBB5NTp//jxatWqFjIyMjw6sNNHQ0EBmZqa6wyiTmFvVYn5Vh7lVLeZXdZhb1WJ+VYe5VS1NTU38+eefcHd3l22TryMMurq6yMjIwObNm+Hk5FRkAZZkhw4dwuTJk8vVmIsLc6tazK/qMLeqxfyqDnOrWsyv6jC3qhUREYF+/fpBV1c313YFOiXJyckJjRs3/qjASousw17laczFhblVLeZXdZhb1WJ+VYe5VS3mV3WY25KBFz0TEREREZEsFgxERERERCSLBQMREREREcliwUBERERERLJYMBARERERkSwWDEREREREJKvICoYuXbqgY8eOOc77888/oVAocO3atQL3GxsbC4VCgfDw8I+MsHSYNm0aFAqF0qNOnTrSfE9Pz2zzv/7661z7FEJgypQpsLGxgb6+Pry8vHDnzh1VD0XtVq5cCWdnZxgbG8PY2Bju7u44fPiwbPs9e/bA1dUVpqamqFChAho2bIhNmzZJ89PT0/H999+jQYMGqFChAmxtbdG/f388fPhQanP69Olsr0/W49KlSyodb3E7c+YMunTpAltbWygUCuzbty/X9nK5iY+PV2r377//ol+/fjA3N4e+vj4aNGiA0NDQHPv8+uuvoVAosHjx4iIaVemzYsUKODg4QE9PD25ubrh48WKu7Xft2oU6depAT08PDRo0wKFDh4op0tJj9uzZaNq0KYyMjFCpUiX4+PggMjIyz+WY28KZM2cOFAoFRo8enWs75jdvee1D5IR5zb/ynN8iKxj8/f1x/PhxPHjwINu89evXw9XVFc7OzgXq882bN0UVXqlSr149xMXFSY+zZ88qzR8yZIjS/Llz5+ba39y5c7F06VKsWrUKFy5cQIUKFeDt7Y3Xr1+rchhqV6VKFcyZMwdhYWEIDQ1F27Zt0bVrV9y8eTPH9hUrVsSkSZNw/vx5XLt2DQMHDsTAgQNx9OhRAEBqaiouX76MyZMn4/Lly9izZw8iIyPx2WefSX00b95c6bWJi4vD4MGD4ejoCFdX12IZd3FJSUmBi4sLVqxYUaDlIiMjlfJTqVIlad6zZ8/QokULaGtr4/Dhw7h16xYWLFgAMzOzbP3s3bsXISEhsLW1/eixlFY7duzAmDFjMHXqVFy+fBkuLi7w9vZGYmJiju3/+usv9OnTB/7+/rhy5Qp8fHzg4+ODGzduFHPkJVtwcDACAgIQEhKC48ePIz09HR06dEBKSorsMsxt4Vy6dAmrV6/Oc/+A+c2/vPYh3se8Fly5za/Ih7CwMAFAhIWFybZJT08XVlZWYubMmUrTX758KQwNDcXKlSvFn3/+KVq2bCn09PRElSpVxIgRI0RycrLU1t7eXsyYMUP4+voKIyMj4efnJwAoPTw8PKT2a9asEXXq1BG6urqidu3aYsWKFdK8gQMHigYNGojXr18LIYRIS0sTDRs2FL6+vvkZsti8eXOeY1aFqVOnChcXF9n5Hh4eYtSoUfnuLzMzU1hbW4t58+ZJ054/fy50dXXFtm3bPiLSwlNXboUQwszMTPz3v//Nd/tGjRqJH3/8UXb+xYsXBQBx9+7dHOe/efNGWFpaihkzZhQ41sJSR34BiL179+ba5tSpUwKAePbsmWyb77//XrRs2TLP9T148EBUrlxZ3LhxQ9jb24tFixYVLOBCUue2m5NmzZqJgIAA6XlGRoawtbUVs2fPzrF9z549RefOnZWmubm5ia+++kqlceZXSctvlsTERAFABAcHy7Zhbgvu5cuXombNmuL48eN5/m9jfvMnr32ID5X0vApRcnIrRNnMb3728YUQosiOMGhpaaF///4IDAyEEEKavmvXLmRkZMDd3R0dO3ZEt27dcO3aNezYsQNnz57F8OHDlfqZP38+XFxccOXKFUyePFk6vH7ixAnExcVhz549AIAtW7ZgypQp+PnnnxEREYFZs2Zh8uTJ2LBhAwBg6dKlSElJwYQJEwAAkyZNwvPnz7F8+fKiGrLK3LlzB7a2tqhWrRr69u2Le/fuKc3fsmULLCwsUL9+fUycOBGpqamyfcXExCA+Ph5eXl7SNBMTE7i5ueH8+fMqG0NJk5GRge3btyMlJQXu7u55thdCICgoCJGRkWjdurVsu6SkJCgUCpiamuY4/48//sCTJ08wcODAwoZe5jRs2BA2NjZo3749zp07pzTvjz/+gKurK3r06IFKlSqhUaNGWLNmjVKbzMxM+Pr6Yvz48ahXr15xhl6ivHnzBmFhYUrvbQ0NDXh5ecm+t8+fP6/UHgC8vb3L1WdBYSQlJQF4dxRSDnNbcAEBAejcuXO2vOWE+c2/vPYh3se8Flx5za9WUXY2aNAgzJs3D8HBwfD09ATw7nSkbt26YdmyZejbt690jmLNmjWxdOlSeHh4YOXKldDT0wMAtG3bFmPHjpX61NTUBACYm5vD2tpamj516lQsWLAAX3zxBQDA0dERt27dwurVq+Hn5wdDQ0Ns3rwZHh4eMDIywuLFi3Hq1CkYGxsX5ZCLnJubGwIDA1G7dm3ExcVh+vTpaNWqFW7cuAEjIyN8+eWXsLe3h62tLa5du4bvv/8ekZGRUiH1oazzw62srJSmW1lZZTt3vCy6fv063N3d8fr1axgaGmLv3r2oW7eubPukpCRUrlwZaWlp0NTUxK+//or27dvn2Pb169f4/vvv0adPH9ntau3atfD29kaVKlWKZDylmY2NDVatWgVXV1ekpaXhv//9Lzw9PXHhwgU0btwYAPDPP/9g5cqVGDNmDH744QdcunQJI0eOhI6ODvz8/AAAv/zyC7S0tDBy5Eh1DkftHj9+jIyMjBzf27dv385xmfj4+HL7WVBYmZmZGD16NFq0aIH69evLtmNuC2b79u24fPlyvq/tYn7zJ699iA8xrwVTnvNbpAVDnTp10Lx5c6xbtw6enp6IiorCn3/+iRkzZmD8+PG4du0atmzZIrUXQiAzMxMxMTFwcnICgHyd552SkoLo6Gj4+/tjyJAh0vS3b9/CxMREeu7u7o5x48Zh5syZ+P7779GyZcsiHK1qdOrUSfrb2dkZbm5usLe3x86dO+Hv74+hQ4dK8xs0aAAbGxu0a9cO0dHRqF69ujpCLtFq166N8PBwJCUl4ffff4efnx+Cg4NliwYjIyOEh4cjOTkZQUFBGDNmDKpVqyYVwFnS09PRs2dPCCGwcuXKHPt68OABjh49ip07dxb1sEql2rVro3bt2tLz5s2bIzo6GosWLZIuLs/MzISrqytmzZoFAGjUqBFu3LiBVatWwc/PD2FhYViyZAkuX74MhUKhlnFQ+RIQEIAbN27kep4yFcz9+/cxatQoHD9+XPqykIpGXvsQ9HHKc36LtGAA3l38PGLECKxYsQLr169H9erV4eHhgeTkZHz11Vc5fitYtWpV6e8KFSrkuY7k5GQAwJo1a+Dm5qY0L+uIBPBu5+PcuXPQ1NREVFRUYYekVqampqhVq5Zs/Fnjj4qKyrFgyDoqk5CQABsbG2l6QkICGjZsWPQBlzA6OjqoUaMGAKBJkya4dOkSlixZgtWrV+fYXkNDQ2rfsGFDREREYPbs2UoFQ1axcPfuXZw8eVL26ML69ethbm6udFE0KWvWrJnSjpiNjU22Ys7JyQm7d+8G8O6Oa4mJiUqfGRkZGRg7diwWL16M2NjYYom7JLCwsICmpiYSEhKUpickJCgdjX2ftbV1gdqXd8OHD8eBAwdw5syZPI8SMrf5FxYWhsTEROnIIvDufXzmzBksX75cOsL7Pua3cPLah2BeP055ym+R/w5Dz549oaGhga1bt2Ljxo0YNGgQFAoFGjdujFu3bqFGjRrZHjo6OrL9Zc3LyMiQpllZWcHW1hb//PNPtr4cHR2ldvPmzcPt27cRHByMI0eOYP369UU9XJVLTk5GdHS00s7++7JuNys339HREdbW1ggKCpKmvXjxAhcuXMjXufxlTWZmJtLS0grdPqtYuHPnDk6cOAFzc/MclxNCYP369ejfvz+0tbU/Ou6yKjw8XGnbbdGiRbbbV/7999+wt7cHAPj6+uLatWsIDw+XHra2thg/frx0N6vyQkdHB02aNFF6b2dmZiIoKEj2ve3u7q7UHgCOHz9eLj8LciOEwPDhw7F3716cPHlS6f+KHOY2/9q1a4fr168rvY9dXV3Rt29fhIeHZysWAOa3sPLah2BeP065ym9RXkGdxd/fX5iZmQlNTU3x77//CiGEuHr1qtDX1xcBAQHiypUr4u+//xb79u1TusNHTnc7SU9PF/r6+uKnn34S8fHx4vnz50KId3dI0tfXF0uWLBGRkZHi2rVrYt26dWLBggVCCCEuX74sdHR0xB9//CGEEGL16tXCyMhIREdH52sM6roqf+zYseL06dMiJiZGnDt3Tnh5eQkLCwuRmJgooqKixIwZM0RoaKiIiYkR+/fvF9WqVROtW7dW6qN27dpiz5490vM5c+YIU1NTsX//fnHt2jXRtWtX4ejoKF69elWsY8tSXLmdMGGCCA4OFjExMeLatWtiwoQJQqFQiGPHjgkhhPD19RUTJkyQ2s+aNUscO3ZMREdHi1u3bon58+cLLS0tsWbNGiHEuzseffbZZ6JKlSoiPDxcxMXFSY+0tDSldZ84cUIAEBERESodY06KK78vX74UV65cEVeuXBEAxMKFC8WVK1ekO0ZNmDBB6a5kixYtEvv27RN37twR169fF6NGjRIaGhrixIkTUpuLFy8KLS0t8fPPP4s7d+6ILVu2CAMDA7F582bZOMrzXZK2b98udHV1RWBgoLh165YYOnSoMDU1FfHx8UKI7Nv4uXPnhJaWlpg/f76IiIgQU6dOFdra2uL69evqGoKSkpLfYcOGCRMTE3H69Gml93lqaqrUhrktWh/eJYn5LZzc9iGEKH15FaLk5FaIspnf/O7jq6Rg+OuvvwQA8cknnyhNv3jxomjfvr0wNDQUFSpUEM7OzuLnn3+W5sv941+zZo2ws7MTGhoaSrdV3bJli2jYsKHQ0dERZmZmonXr1mLPnj3i1atXom7dumLo0KFK/Xz22WeiefPm4u3bt3mOQV0baK9evYSNjY3Q0dERlStXFr169RJRUVFCCCHu3bsnWrduLSpWrCh0dXVFjRo1xPjx40VSUpJSHwDE+vXrpeeZmZli8uTJwsrKSujq6op27dqJyMjI4hyWkuLK7aBBg4S9vb3Q0dERlpaWol27dlKxIMS7f1B+fn7S80mTJokaNWoIPT09YWZmJtzd3cX27dul+TExMdlu85v1OHXqlNK6+/TpI5o3b67S8ckprvxm3Sb1w0dWTv38/JTer7/88ouoXr260NPTExUrVhSenp7i5MmT2fr93//+J+rXry90dXVFnTp1xG+//ZZrHOW5YBBCiGXLlomqVasKHR0d0axZMxESEiLN+3AbF0KInTt3ilq1agkdHR1Rr149cfDgwWKOWF5Jya/c+/z9z1Xmtmh9WDAwv4WT2z6EEKUvr0KUnNwKUTbzq9aCoSwoSRtoWcPcqhbzqzrMrWoxv6rD3KoW86s6zK1qFfvvMBARERERUdnDgoGIiIiIiGSxYCAiIiIiIlksGIiIiIiISBYLBiIiIiIiksWCgYiIiIiIZLFgICIiIiIiWSwYiIiIiIhIllZBGkdERKgqjhInJiYGQPkac3FhblWL+VUd5la1mF/VYW5Vi/lVHeZWtfKbV4UQQuTV6N69e3ByckJqaupHB1aaaGpqIiMjQ91hlEnMrWoxv6rD3KoW86s6zK1qMb+qw9yqloGBASIiIlC1alXZNvkqGIB3RcPjx4+LLLjSIC0tDbq6uuoOo0xiblWL+VUd5la1mF/VYW5Vi/lVHeZWtSwsLHItFoACFAxERERERFT+8KJnIiIiIiKSxYKBiIiIiIhksWAgIiIiIiJZLBiIiIiIiEgWCwYiIiIiIpLFgoGIiIiIiGSxYCAiIiIiIln/B/gaE92Z3vWYAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "openai_run_df = pd.read_csv(\"openai_run.csv\")\n",
    "vertex_run_df = pd.read_csv(\"vertex_run.csv\")\n",
    "\n",
    "combined_scores = pd.DataFrame({\n",
    "    \"OpenAI\": list(openai_run_df[\"answer_similarity\"]),\n",
    "    \"Vertex\": list(vertex_run_df[\"answer_similarity\"])\n",
    "})\n",
    "\n",
    "# Position of bars on x-axis\n",
    "ind = np.arange(6)\n",
    "\n",
    "# Figure size\n",
    "plt.figure(figsize=(10,5))\n",
    "\n",
    "# Width of a bar \n",
    "width = 0.3\n",
    "\n",
    "openai_counts = combined_scores[\"OpenAI\"].value_counts().reindex([0,1,2,3,4,5], fill_value=0)\n",
    "plt.bar(openai_counts.index, openai_counts.values, width, label=\"OpenAI\", color=\"#5EA7EC\")\n",
    "\n",
    "vertex_counts = combined_scores[\"Vertex\"].value_counts().reindex([0,1,2,3,4,5], fill_value=0)\n",
    "plt.bar(vertex_counts.index + width, vertex_counts.values, width, label=\"Vertex\", color=\"#A679C8\")\n",
    "\n",
    "plt.title('Distribution of scores for OpenAI & Vertex')\n",
    "plt.xlabel('Answer Similarity Score')\n",
    "plt.ylabel('Count')\n",
    "\n",
    "# Remove all scores except whole numbers\n",
    "plt.xticks(ind + width / 2, range(0, 6, 1))\n",
    "\n",
    "plt.legend(loc='best')\n",
    "\n",
    "# Rounding the descriptive statistics to 3 significant figures\n",
    "statistics_df = combined_scores.describe()\n",
    "# Change 50% to 50% (median)\n",
    "statistics_df = statistics_df.rename(index={'50%': 'median'})\n",
    "rounded_statistics_df = statistics_df.round(3)\n",
    "\n",
    "# Transposing the descriptive statistics table for horizontal display\n",
    "rounded_statistics_df = rounded_statistics_df.T\n",
    "\n",
    "# Adding the table below the histogram\n",
    "table = plt.table(cellText=rounded_statistics_df.values,\n",
    "                  colWidths=[0.1] * len(rounded_statistics_df.columns),\n",
    "                  rowLabels=rounded_statistics_df.index,\n",
    "                  colLabels=rounded_statistics_df.columns,\n",
    "                  cellLoc = 'center', rowLoc = 'center',\n",
    "                  loc='bottom', bbox=[0.0, -0.5, 1.0, 0.3])\n",
    "\n",
    "plt.subplots_adjust(left=0.2, bottom=0.3)\n",
    "plt.savefig(\"openai_v_vertex.png\", facecolor='white', bbox_inches='tight')\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}