Text Extraction Eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04bba52c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import time\n",
    "import os\n",
    "import glob\n",
    "import weave\n",
    "import typing\n",
    "weave.use_frontend_devmode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8172d67c",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_labels = {\n",
    "    'Articles_of_Incorporation_Real_Example_3_txt': {\n",
    "        'name': 'HealthFirst Solutions LLC',\n",
    "        'shares': 500000\n",
    "    },\n",
    "    'Articles_of_Incorporation_Real_Example_2_txt': {\n",
    "        'name': 'GreenLeaf LLC',\n",
    "        'shares': None,\n",
    "        'directors': ['Sarah Miller', 'Daniel Lee']\n",
    "    },\n",
    "    'Articles_of_Incorporation_Real_Example_1_txt': {\n",
    "        'name': 'TechBoost Corp',\n",
    "        'shares': 1000000\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_10_txt': {\n",
    "        'name': 'Brown, Fernandez and Smith',\n",
    "        'shares': 41141\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_9_txt': {\n",
    "        'name': 'Ruiz-Goodman',\n",
    "        'shares': 31783\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_8_txt': {\n",
    "        'name': 'Gibson, Hunt and Davidson',\n",
    "        'shares': 96403\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_7_txt': {\n",
    "        'name': 'Boyd-Browning',\n",
    "        'shares': 41300\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_6_txt': {\n",
    "        'name': 'Newton, Moreno and Yang',\n",
    "        'shares': 73981\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_5_txt': {\n",
    "        'name': 'Matthews and Sons',\n",
    "        'shares': 98608\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_4_txt': {\n",
    "        'name': 'Moore LLC',\n",
    "        'shares': 5732\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_3_txt': {\n",
    "        'name': 'Mullen Inc',\n",
    "        'shares': 76197\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_2_txt': {\n",
    "        'name': 'Ellis and Sons',\n",
    "        'shares': 54183\n",
    "    },\n",
    "    'Highly_Varied_Article_of_Incorporation_1_txt': {\n",
    "        'name': 'French, Wyatt and Coleman',\n",
    "        'shares': 78821\n",
    "    },\n",
    "}\n",
    "\n",
    "def read_dataset(root):   \n",
    "    dataset_rows = []\n",
    "    for p in glob.glob(os.path.join(root, '*.txt')):\n",
    "        # Have to do replace here because of weave '.' access issues\n",
    "        example_id = os.path.basename(p).replace('.', '_')\n",
    "        label = raw_labels.get(example_id)\n",
    "        if label:\n",
    "            dataset_rows.append({'id': example_id, 'example': open(p).read(), 'label': label})\n",
    "    return dataset_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb93f2a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can't just make our own types, server won't deserialize.\n",
    "# A fairly easy fix.\n",
    "@weave.type()\n",
    "class Dataset:\n",
    "    rows: list[typing.Any]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92c6535c",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_dataset = read_dataset('/Users/shawn/datasets/aoi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e803efd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#raw_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc723706",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "dataset = weave.save(Dataset(raw_dataset), 'my_dataset5')\n",
    "# Now, here I really want to make my own labels in the UI immediately.\n",
    "# where should the added column go? A new version of this dataset?\n",
    "# yeah sure why not.\n",
    "# What's missing for editing to be good?\n",
    "#   - batch editing, ie make a bunch of changes and choose where/how to save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a99df38",
   "metadata": {},
   "outputs": [],
   "source": [
    "#published = weave.publish(Dataset(raw_dataset), 'weave-flow1/my_dataset1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77c71a23",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.rows\n",
    "# Here i went to render labels next to dataset.\n",
    "# I need access to labels in the notebook memory... would be easy enough to pass in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "331d88d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def split_paragraphs(doc):\n",
    "    lines = [l.strip() for l in doc.split('\\n')]\n",
    "    stripped_doc = '\\n'.join(lines)\n",
    "    return [p.strip() for p in stripped_doc.split('\\n\\n')]\n",
    "\n",
    "def find_first_numeric(s):\n",
    "    match = re.search(r'\\d+', s)\n",
    "    if match is None:\n",
    "        return None\n",
    "    return int(match.group().replace(',', ''))\n",
    "\n",
    "def predict(dataset_row, config):\n",
    "    paragraphs = split_paragraphs(dataset_row['example'])\n",
    "    capital_paragraph = None\n",
    "    name_paragraph = None\n",
    "    for p in paragraphs:\n",
    "        if 'name' in p.lower():\n",
    "            name_paragraph = p\n",
    "        if 'share' in p.lower():\n",
    "            capital_paragraph = p\n",
    "    result = {\n",
    "        'name': None,\n",
    "        'shares': None,\n",
    "    }\n",
    "    if capital_paragraph:\n",
    "        paragraph_start = config.get('shares_skip_chars', 0)\n",
    "        result['shares'] = find_first_numeric(capital_paragraph[paragraph_start:])\n",
    "    if name_paragraph:\n",
    "        match = re.search(r'is ', name_paragraph)\n",
    "        if match is not None:\n",
    "            result['name'] = name_paragraph[match.end():]\n",
    "        if result['name'] and config.get('name_up_to_period'):\n",
    "            match = re.search(r'\\.', result['name'])\n",
    "            if match is not None:\n",
    "                result['name'] = result['name'][:match.start()]\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2378174",
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset_row in weave.use(dataset.rows):\n",
    "    print(predict(dataset_row, {'shares_skip_chars': 4}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efc63aa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = ['name', 'shares', 'directors']\n",
    "\n",
    "def p_r_f1(tp, fp, fn):\n",
    "    # if any denom is zero, then zero. could use NaN instead...\n",
    "    precision = 0\n",
    "    if tp or fp:\n",
    "        precision = tp / (tp + fp)\n",
    "    recall = 0\n",
    "    if tp or fn:\n",
    "        recall = tp / (tp + fn)\n",
    "    f1 = 0\n",
    "    if precision or recall:\n",
    "        f1 = 2 * (precision * recall) / (precision + recall)\n",
    "    return precision, recall, f1\n",
    "\n",
    "def summarize_item(item_result, item_label):\n",
    "    item_summary = {}\n",
    "    for f in fields:\n",
    "        item_summary[f'{f}_negative'] = item_result.get(f) is None\n",
    "        item_summary[f'{f}_correct'] = item_result.get(f) == item_label.get(f)\n",
    "\n",
    "    item_correct = sum([item_summary[f'{f}_correct'] for f in fields])\n",
    "    item_tp = sum([item_label.get(f) is not None and item_summary[f'{f}_correct'] for f in fields])\n",
    "    item_fp = sum([item_label.get(f) is not None and not item_summary[f'{f}_correct'] for f in fields])\n",
    "    item_tn = sum([item_label.get(f) is None and item_summary[f'{f}_correct'] for f in fields])\n",
    "    item_fn = sum([item_label.get(f) is None and not item_summary[f'{f}_correct'] for f in fields])\n",
    "\n",
    "    item_precision, item_recall, item_f1 = p_r_f1(item_tp, item_fp, item_fn)\n",
    "\n",
    "    return {\n",
    "        **item_summary,\n",
    "        'correct': item_correct,\n",
    "        'tp': item_tp,\n",
    "        'fp': item_fp,\n",
    "        'tn': item_tn,\n",
    "        'fn': item_fn,\n",
    "        'precision': item_precision,\n",
    "        'recall': item_recall,\n",
    "        'f1': item_f1\n",
    "    }\n",
    "\n",
    "def field_pr(eval_result, field_name):\n",
    "    tp = sum(not item['summary'][f'{field_name}_negative']\n",
    "             and item['summary'][f'{field_name}_correct']\n",
    "             for item in eval_result)\n",
    "    fp = sum(not item['summary'][f'{field_name}_negative']\n",
    "             and not item['summary'][f'{field_name}_correct']\n",
    "             for item in eval_result)\n",
    "    tn = sum(item['summary'][f'{field_name}_negative']\n",
    "             and item['summary'][f'{field_name}_correct']\n",
    "             for item in eval_result)\n",
    "    fn = sum(item['summary'][f'{field_name}_negative']\n",
    "             and not item['summary'][f'{field_name}_correct']\n",
    "             for item in eval_result)\n",
    "\n",
    "    precision, recall, f1 = p_r_f1(tp, fp, fn)\n",
    "    return {\n",
    "        'tp': tp,\n",
    "        'fp': fp,\n",
    "        'tn': tn,\n",
    "        'fn': fn,\n",
    "        'precision': precision,\n",
    "        'recall': recall,\n",
    "        'f1': f1\n",
    "    }\n",
    "\n",
    "def summarize(eval_result):\n",
    "    summary = {}\n",
    "    for field in fields:\n",
    "        summary[f'field_{field}'] = field_pr(eval_result, field)\n",
    "    for metric in ['precision', 'recall', 'f1']:\n",
    "        summary[f'avg_{metric}'] = sum(summary[f'field_{f}'][metric] for f in fields) / len(fields)\n",
    "    return summary\n",
    "\n",
    "def evaluate(dataset, predict_config):\n",
    "    eval_result = []\n",
    "    correct_count = 0\n",
    "    count = 0\n",
    "    for dataset_row in dataset:\n",
    "        start_time = time.time()\n",
    "        result = predict(dataset_row, predict_config)\n",
    "        latency = time.time() - start_time\n",
    "        latency = random.gauss(predict_config['latency_mu'], predict_config['latency_sigma'])\n",
    "        item_summary = summarize_item(result, dataset_row['label'])\n",
    "        eval_result.append({\n",
    "            'dataset_id': dataset_row['id'],\n",
    "            'result': result,\n",
    "            'summary': {\n",
    "                'latency': latency,\n",
    "                **item_summary\n",
    "            }\n",
    "        })\n",
    "    return {\n",
    "        'config': predict_config,\n",
    "        'eval_table': eval_result,\n",
    "        'summary': summarize(eval_result)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2248364",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_results0 = weave.save(evaluate(\n",
    "    weave.use(dataset).rows, {'latency_mu': 0.3, 'latency_sigma': 0.1}),\n",
    "    'eval_results0')\n",
    "eval_results1 = weave.save(evaluate(\n",
    "    weave.use(dataset).rows, {'latency_mu': 0.3, 'latency_sigma': 0.1, 'name_up_to_period': True}),\n",
    "    'eval_results1')\n",
    "eval_results2 = weave.save(evaluate(\n",
    "    weave.use(dataset).rows, {'latency_mu': 0.3, 'latency_sigma': 0.1, 'name_up_to_period': True, 'shares_skip_chars': 4}),\n",
    "    'eval_results2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93727275",
   "metadata": {},
   "outputs": [],
   "source": [
    "from weave.panels_py import panel_eval\n",
    "panel_eval.eval_board(dataset.rows, eval_results0, eval_results2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "166e8607",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To add:\n",
    "#   - backed by W&B runs\n",
    "#   - show run code / config comparison\n",
    "#   - show traces of pipelines and compare them\n",
    "#   - add view of N runs instead of 2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}