Eval Board Syn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5214b543",
   "metadata": {},
   "source": [
    "- Load two eval_results\n",
    "\n",
    "EvalResult\n",
    "- example, label, result, item_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "314bb6b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import string\n",
    "import time\n",
    "import typing\n",
    "\n",
    "import weave\n",
    "from weave import weave_internal\n",
    "\n",
    "weave.use_frontend_devmode()\n",
    "from weave.legacy import ops_domain\n",
    "from weave.legacy.panels import panel_board"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "550daef6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rand_string_n(n: int) -> str:\n",
    "    return \"\".join(\n",
    "        random.choice(string.ascii_uppercase + string.digits) for _ in range(n)\n",
    "    )\n",
    "\n",
    "\n",
    "dataset_raw = [\n",
    "    {\n",
    "        \"id\": str(i),\n",
    "        \"example\": rand_string_n(10),\n",
    "        \"label\": random.choice(string.ascii_uppercase),\n",
    "    }\n",
    "    for i in range(50)\n",
    "]\n",
    "dataset = weave.save(dataset_raw, \"dataset\")\n",
    "# dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0d930d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(dataset_row, config):\n",
    "    if random.random() < config[\"correct_chance\"]:\n",
    "        return dataset_row[\"label\"]\n",
    "    return random.choice(string.ascii_uppercase)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb86b95c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(dataset, predict_config):\n",
    "    eval_result = []\n",
    "    correct_count = 0\n",
    "    count = 0\n",
    "    for dataset_row in dataset:\n",
    "        start_time = time.time()\n",
    "        result = predict(dataset_row, predict_config)\n",
    "        latency = time.time() - start_time\n",
    "        latency = random.gauss(\n",
    "            predict_config[\"latency_mu\"], predict_config[\"latency_sigma\"]\n",
    "        )\n",
    "        correct = dataset_row[\"label\"] == result\n",
    "        if correct:\n",
    "            correct_count += 1\n",
    "        count += 1\n",
    "        eval_result.append(\n",
    "            {\n",
    "                \"dataset_id\": dataset_row[\"id\"],\n",
    "                \"result\": result,\n",
    "                \"summary\": {\"latency\": latency, \"correct\": correct},\n",
    "            }\n",
    "        )\n",
    "    return {\n",
    "        \"config\": predict_config,\n",
    "        \"eval_table\": eval_result,\n",
    "        \"summary\": {\"accuracy\": correct_count / len(dataset)},\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05d16a5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_result_raw0 = evaluate(\n",
    "    dataset_raw, {\"correct_chance\": 0.5, \"latency_mu\": 0.3, \"latency_sigma\": 0.1}\n",
    ")\n",
    "eval_result_raw1 = evaluate(\n",
    "    dataset_raw, {\"correct_chance\": 0.5, \"latency_mu\": 0.4, \"latency_sigma\": 0.2}\n",
    ")\n",
    "eval_result0 = weave.save(eval_result_raw0, \"eval_result0\")\n",
    "eval_result1 = weave.save(eval_result_raw1, \"eval_result1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8065ad6",
   "metadata": {},
   "outputs": [],
   "source": [
    "varbar = panel_board.varbar()\n",
    "\n",
    "dataset_var = varbar.add(\"dataset\", dataset)\n",
    "eval_result0_var = varbar.add(\"eval_result0\", eval_result0)\n",
    "eval_result1_var = varbar.add(\"eval_result1\", eval_result1)\n",
    "\n",
    "summary = varbar.add(\n",
    "    \"summary\",\n",
    "    weave.legacy.ops.make_list(\n",
    "        a=weave.legacy.ops.TypedDict.merge(\n",
    "            weave.legacy.ops.dict_(name=\"res0\"), eval_result0_var[\"summary\"]\n",
    "        ),\n",
    "        b=weave.legacy.ops.TypedDict.merge(\n",
    "            weave.legacy.ops.dict_(name=\"res1\"), eval_result1_var[\"summary\"]\n",
    "        ),\n",
    "    ),\n",
    ")\n",
    "\n",
    "weave.legacy.ops.make_list(\n",
    "    a=eval_result0_var[\"eval_table\"], b=eval_result0_var[\"eval_table\"]\n",
    ")\n",
    "\n",
    "concatted_evals = varbar.add(\n",
    "    \"concatted_evals\",\n",
    "    weave.legacy.ops.List.concat(\n",
    "        weave.legacy.ops.make_list(\n",
    "            a=eval_result0_var[\"eval_table\"].map(\n",
    "                lambda row: weave.legacy.ops.TypedDict.merge(\n",
    "                    weave.legacy.ops.dict_(name=\"res0\"), row\n",
    "                )\n",
    "            ),\n",
    "            b=eval_result1_var[\"eval_table\"].map(\n",
    "                lambda row: weave.legacy.ops.TypedDict.merge(\n",
    "                    weave.legacy.ops.dict_(name=\"res1\"), row\n",
    "                )\n",
    "            ),\n",
    "        )\n",
    "    ),\n",
    ")\n",
    "\n",
    "# join evals together first\n",
    "joined_evals = varbar.add(\n",
    "    \"joined_evals\",\n",
    "    weave.legacy.ops.join_all(\n",
    "        weave.legacy.ops.make_list(\n",
    "            a=eval_result0_var[\"eval_table\"], b=eval_result1_var[\"eval_table\"]\n",
    "        ),\n",
    "        lambda row: row[\"dataset_id\"],\n",
    "        False,\n",
    "    ),\n",
    ")\n",
    "\n",
    "# then join dataset to evals\n",
    "dataset_evals = varbar.add(\n",
    "    \"dataset_evals\",\n",
    "    weave.legacy.ops.join_2(\n",
    "        dataset_var,\n",
    "        joined_evals,\n",
    "        lambda row: row[\"id\"],\n",
    "        lambda row: row[\"dataset_id\"][0],\n",
    "        \"dataset\",\n",
    "        \"evals\",\n",
    "        False,\n",
    "        False,\n",
    "    ),\n",
    ")\n",
    "\n",
    "\n",
    "main = weave.legacy.panels.Group(\n",
    "    layoutMode=\"grid\",\n",
    "    showExpressions=True,\n",
    "    enableAddPanel=True,\n",
    ")\n",
    "\n",
    "#### Run/config info TODO\n",
    "\n",
    "#### Summary info\n",
    "\n",
    "main.add(\n",
    "    \"accuracy\",\n",
    "    weave.legacy.panels.Plot(\n",
    "        summary,\n",
    "        x=lambda row: row[\"accuracy\"],\n",
    "        y=lambda row: row[\"name\"],\n",
    "        color=lambda row: row[\"name\"],\n",
    "    ),\n",
    "    layout=weave.legacy.panels.GroupPanelLayout(x=0, y=0, w=12, h=4),\n",
    ")\n",
    "\n",
    "\n",
    "main.add(\n",
    "    \"latency\",\n",
    "    weave.legacy.panels.Plot(\n",
    "        concatted_evals,\n",
    "        x=lambda row: row[\"summary\"][\"latency\"],\n",
    "        y=lambda row: row[\"name\"],\n",
    "        color=lambda row: row[\"name\"],\n",
    "        mark=\"boxplot\",\n",
    "    ),\n",
    "    layout=weave.legacy.panels.GroupPanelLayout(x=12, y=0, w=12, h=4),\n",
    ")\n",
    "\n",
    "# ct = main.add('concat_t', concatted_evals, layout=weave.legacy.panels.GroupPanelLayout(x=0, y=4, w=24, h=12))\n",
    "# main.add('dataset_table', dataset)\n",
    "# main.add('joined_evals', joined_evals)\n",
    "# main.add('dataset_evals', dataset_evals, layout=weave.legacy.panels.GroupPanelLayout(x=0, y=4, w=24, h=6))\n",
    "\n",
    "##### Example details\n",
    "\n",
    "# more ideas: show examples that all got wrong, or that are confusing\n",
    "\n",
    "faceted_view = weave.legacy.panels.Facet(\n",
    "    dataset_evals,\n",
    "    x=lambda row: row[\"evals.summary\"][0][\"correct\"],\n",
    "    y=lambda row: row[\"evals.summary\"][1][\"correct\"],\n",
    "    select=lambda row: row.count(),\n",
    ")\n",
    "\n",
    "faceted = main.add(\n",
    "    \"faceted\",\n",
    "    faceted_view,\n",
    "    layout=weave.legacy.panels.GroupPanelLayout(x=0, y=4, w=12, h=6),\n",
    ")\n",
    "\n",
    "main.add(\n",
    "    \"example_latencies\",\n",
    "    weave.legacy.panels.Plot(\n",
    "        dataset_evals,\n",
    "        x=lambda row: row[\"evals.summary\"][\"latency\"][0],\n",
    "        y=lambda row: row[\"evals.summary\"][\"latency\"][1],\n",
    "    ),\n",
    "    layout=weave.legacy.panels.GroupPanelLayout(x=12, y=4, w=12, h=6),\n",
    ")\n",
    "\n",
    "faceted_sel = weave.legacy.panels.Table(faceted.selected())\n",
    "faceted_sel.config.rowSize = 2\n",
    "faceted_sel.add_column(lambda row: row[\"dataset.id\"], \"id\")\n",
    "faceted_sel.add_column(lambda row: row[\"dataset.example\"], \"example\")\n",
    "faceted_sel.add_column(lambda row: row[\"dataset.label\"], \"label\")\n",
    "faceted_sel.add_column(\n",
    "    lambda row: weave.legacy.ops.dict_(\n",
    "        res0=row[\"evals.result\"][0], res1=row[\"evals.result\"][1]\n",
    "    ),\n",
    "    \"result\",\n",
    ")\n",
    "faceted_sel.add_column(\n",
    "    lambda row: weave.legacy.ops.dict_(\n",
    "        res0=row[\"evals.summary\"][0][\"correct\"], res1=row[\"evals.summary\"][1][\"correct\"]\n",
    "    ),\n",
    "    \"correct\",\n",
    ")\n",
    "faceted_sel.add_column(\n",
    "    lambda row: weave.legacy.ops.dict_(\n",
    "        res0=row[\"evals.summary\"][0][\"latency\"], res1=row[\"evals.summary\"][1][\"latency\"]\n",
    "    ),\n",
    "    \"latency\",\n",
    ")\n",
    "\n",
    "main.add(\n",
    "    \"faceted_sel\",\n",
    "    faceted_sel,\n",
    "    layout=weave.legacy.panels.GroupPanelLayout(x=0, y=10, w=24, h=12),\n",
    ")\n",
    "\n",
    "weave.legacy.panels.Board(vars=varbar, panels=main)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}