tutorial_notebooks/tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8b787be6-acf0-46ab-8e97-3b7c1534fa14",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jannesklaas/miniconda3/envs/dxc_test/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import doxstractor as dxc\n",
    "import glob\n",
    "import os\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd961f37-860f-4213-a71f-87b16a5e2ce4",
   "metadata": {},
   "source": [
    "We will be using the Anthropic API as part of this tutorial. Make sure you have set the `ANTHROPIC_API_KEY` environment variable to your API key."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9ed9e5fa-a541-4d8f-b97a-ef7258d1e0a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"anthropic_key.txt\", \"r\") as f:\n",
    "    key = f.read()\n",
    "    os.environ['ANTHROPIC_API_KEY'] = key"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24bd462d-8f8e-4dc7-bb3b-aad3df084fea",
   "metadata": {},
   "source": [
    "## Creating our first extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dfcbf550-e2af-4572-ade2-626f8a391a8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"tutorial_data/EDGAR_lease_agreement_1.html\", \"r\") as f:\n",
    "    html = f.read()\n",
    "soup = BeautifulSoup(html, features=\"html.parser\")\n",
    "text = soup.get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ebd04ac4-dd46-476d-920a-6385c7ea6f99",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = dxc.TransformersQAModel(model=\"deepset/tinyroberta-squad2\", na_threshold = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "43b42faf-404e-46a2-8ab6-bdc62d339a84",
   "metadata": {},
   "outputs": [],
   "source": [
    "address_extractor = dxc.TextExtractor(\n",
    "    name=\"address\", \n",
    "    query=\"What is the address of the leased building?\", \n",
    "    model=model, \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4b177f3a-76b8-47fc-b7ab-4e8aecd527df",
   "metadata": {},
   "outputs": [],
   "source": [
    "address = address_extractor.extract(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1babd87a-86d1-45b0-8a91-c1589acd455c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'6335 1St – Avenue South'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "address"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f899c2f-ae22-4291-9652-41f0c0ef5cf8",
   "metadata": {},
   "source": [
    "## Creating a numeric extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a8e98898-a6b5-4753-9441-70ac2e497188",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"tutorial_data/EDGAR_employment_agreement_1.html\", \"r\") as f:\n",
    "    html = f.read()\n",
    "soup = BeautifulSoup(html, features=\"html.parser\")\n",
    "text = soup.get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bbf1071a-035d-4fa6-b8a6-899f7d96a457",
   "metadata": {},
   "outputs": [],
   "source": [
    "salary_extractor = dxc.NumericExtractor(\n",
    "    name=\"salary\", \n",
    "    query=\"What is the base salary?\", \n",
    "    model=model, \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "89333662-7fda-42d5-8199-04b9d1251f0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "salary = salary_extractor.extract(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5da81e2f-84a8-4292-bf58-8d0d732e4056",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'575,000'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fc37352-4425-4406-ae84-b282cc6cf86a",
   "metadata": {},
   "source": [
    "## Creating a categorical extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c394d0c3-be77-4df9-90da-81635dd56acc",
   "metadata": {},
   "outputs": [],
   "source": [
    "anthropic_model = dxc.AnthropicAPIModel(model=\"claude-3-haiku-20240307\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6dfe0fe2-06ce-478c-bb4e-d63a6e815770",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_classifier = dxc.CategoryExtractor(name=\"doctype\", \n",
    "                                       query=\"What type of agreement is this?\", \n",
    "                                       categories=[\"employment\", \"lease\", \"other\"],\n",
    "                                      model=anthropic_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3170569e-4a34-42ee-a964-326b83d0b1fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = doc_classifier.extract(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3edafb0d-2b52-4869-a9a5-6a9dc6487abf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'employment'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f91e165-a81b-46d9-a05c-3e81708d9bbb",
   "metadata": {},
   "source": [
    "## Chaining extractors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d13eabf7-6e5c-4fc7-9fe7-d421c1fa64f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = dxc.Node(extractor=doc_classifier, \n",
    "                 children={\n",
    "                     \"lease\":[dxc.Node(address_extractor)], \n",
    "                     \"employment\": [dxc.Node(salary_extractor)]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "84868906-9e1c-4ea5-be76-1a1588c1f0b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'doctype': 'employment', 'salary': '575,000'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chain.extract(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2149633-a37d-4706-95d2-2183176850d7",
   "metadata": {},
   "source": [
    "## Processing multiple documents into a table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2e786b52-460f-4c49-9a2f-fb208b2edf44",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"tutorial_data/\"\n",
    "file_paths = glob.glob(os.path.join(path, '*'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2af1359b-1789-491b-afd1-26933c1f0477",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'doctype': 'employment', 'salary': '18', 'file_path': 'tutorial_data/EDGAR_employment_agreement_2.html'}\n",
      "A 429 status code was received; sleeping 60s to reset rate limit\n",
      "{'doctype': 'lease', 'address': 'Annapolis Lane', 'file_path': 'tutorial_data/EDGAR_lease_agreement_2.html'}\n",
      "{'doctype': 'employment', 'salary': 'NA', 'file_path': 'tutorial_data/EDGAR_employment_agreement_3.html'}\n",
      "{'doctype': 'lease', 'address': '6335 1St – Avenue South', 'file_path': 'tutorial_data/EDGAR_lease_agreement_1.html'}\n",
      "{'doctype': 'employment', 'salary': '575,000', 'file_path': 'tutorial_data/EDGAR_employment_agreement_1.html'}\n"
     ]
    }
   ],
   "source": [
    "collector = []\n",
    "for fp in file_paths:\n",
    "    with open(fp, \"r\") as f:\n",
    "        html = f.read()\n",
    "    soup = BeautifulSoup(html, features=\"html.parser\")\n",
    "    text = soup.get_text()\n",
    "    data = chain.extract(text)\n",
    "    data.update({\"file_path\":fp})\n",
    "    print(data)\n",
    "    collector.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d344de65-8b29-4a4c-bfff-2fb55c729ddc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "89cdb163-639a-4380-847a-3be0ffa487c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>doctype</th>\n",
       "      <th>salary</th>\n",
       "      <th>address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tutorial_data/EDGAR_employment_agreement_2.html</td>\n",
       "      <td>employment</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tutorial_data/EDGAR_employment_agreement_3.html</td>\n",
       "      <td>employment</td>\n",
       "      <td>NA</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tutorial_data/EDGAR_employment_agreement_1.html</td>\n",
       "      <td>employment</td>\n",
       "      <td>575,000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tutorial_data/EDGAR_lease_agreement_2.html</td>\n",
       "      <td>lease</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Annapolis Lane</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tutorial_data/EDGAR_lease_agreement_1.html</td>\n",
       "      <td>lease</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6335 1St – Avenue South</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         file_path     doctype   salary  \\\n",
       "0  tutorial_data/EDGAR_employment_agreement_2.html  employment       18   \n",
       "2  tutorial_data/EDGAR_employment_agreement_3.html  employment       NA   \n",
       "4  tutorial_data/EDGAR_employment_agreement_1.html  employment  575,000   \n",
       "1       tutorial_data/EDGAR_lease_agreement_2.html       lease      NaN   \n",
       "3       tutorial_data/EDGAR_lease_agreement_1.html       lease      NaN   \n",
       "\n",
       "                   address  \n",
       "0                      NaN  \n",
       "2                      NaN  \n",
       "4                      NaN  \n",
       "1           Annapolis Lane  \n",
       "3  6335 1St – Avenue South  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(collector)[[\"file_path\", \"doctype\", \"salary\", \"address\"]].sort_values(\"doctype\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34afd6c4-bfe8-4921-9653-cb00a1d1bfc7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}