cohere_sm+split.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import Document\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "from llama_index.core.node_parser import LangchainNodeParser\n",
    "from bs4 import BeautifulSoup, NavigableString\n",
    "import cohere\n",
    "\n",
    "def generate_xpath(element, path=\"\"): # used to generate dict nodes\n",
    "    \"\"\" Recursive function to generate the xpath of an element \"\"\"\n",
    "    if element.parent is None:\n",
    "        return path\n",
    "    else:\n",
    "        siblings = [sib for sib in element.parent.children if sib.name == element.name]\n",
    "        if len(siblings) > 1:\n",
    "            count = siblings.index(element) + 1\n",
    "            path = f\"/{element.name}[{count}]{path}\"\n",
    "        else:\n",
    "            path = f\"/{element.name}{path}\"\n",
    "        return generate_xpath(element.parent, path)\n",
    "\n",
    "def create_xpath_dict(html, only_body=True, max_length=200): # used to generate dict nodes\n",
    "    ''' Create a list of xpaths and a list of dict of attributes of all elements in the html'''\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    if only_body:\n",
    "        root = soup.body\n",
    "    else:\n",
    "        root = soup.html\n",
    "    element_xpath_list = []\n",
    "    element_attributes_list = []\n",
    "    stack = [(root, '')]  # stack to keep track of elements and their paths\n",
    "    while stack:\n",
    "        element, path = stack.pop()\n",
    "        if element.name is not None:\n",
    "            current_path = generate_xpath(element)\n",
    "            element_attrs = dict(element.attrs)\n",
    "            direct_text_content = ''.join([str(content).strip() for content in element.contents if isinstance(content, NavigableString) and content.strip()])\n",
    "            if direct_text_content:\n",
    "                element_attrs['text'] = direct_text_content\n",
    "                element_attrs['element'] = element.name\n",
    "                for key in element_attrs:\n",
    "                    if len(element_attrs[key]) > max_length:\n",
    "                        element_attrs[key] = element_attrs[key][:max_length]\n",
    "                element_xpath_list.append(current_path)\n",
    "                element_attributes_list.append(element_attrs)\n",
    "            elif element_attrs != {}:\n",
    "                element_attrs['element'] = element.name\n",
    "                for key in element_attrs:\n",
    "                    if len(element_attrs[key]) > max_length:\n",
    "                        element_attrs[key] = element_attrs[key][:max_length]\n",
    "                element_xpath_list.append(current_path)\n",
    "                element_attributes_list.append(element_attrs)\n",
    "            for child in element.children:\n",
    "                if child.name is not None:\n",
    "                    stack.append((child, current_path))\n",
    "\n",
    "    return element_xpath_list, element_attributes_list\n",
    "\n",
    "def get_results(cohere, query, html, top_n=5, model=\"rerank-english-v3.0\", rank_fields=None):\n",
    "    xpath_list, attributes_list = create_xpath_dict(html)\n",
    "    assert len(xpath_list) == len(attributes_list)\n",
    "    l = len(xpath_list)\n",
    "    list_of_results = []\n",
    "    for j in range(0, l, 1000):\n",
    "        attr = attributes_list[j:j+1000]\n",
    "        results = cohere.rerank(model=model, query=query, documents=attr, top_n=top_n, return_documents=True, rank_fields=rank_fields)\n",
    "        results = [r.dict() for r in results.results]\n",
    "        for r in results:\n",
    "            r['index']+=j\n",
    "        list_of_results += results\n",
    "    list_of_results = sorted(list_of_results, key=lambda x: x['relevance_score'], reverse=True)\n",
    "    xpaths = [xpath_list[result['index']] for result in list_of_results[:top_n]]\n",
    "    return xpaths, list_of_results[:top_n]\n",
    "\n",
    "\n",
    "\n",
    "def match_element(attributes, element_specs):\n",
    "    i=0\n",
    "    for spec in element_specs:\n",
    "        matches = True\n",
    "        for key in spec:\n",
    "            if key in attributes:\n",
    "                if isinstance(attributes[key], list):\n",
    "                    if not set(spec[key]).issubset(set(attributes[key])):\n",
    "                        matches = False\n",
    "                        break\n",
    "                elif attributes[key] != spec[key]:\n",
    "                    matches = False\n",
    "                    break\n",
    "            else:\n",
    "                matches = False\n",
    "                break\n",
    "        if matches:\n",
    "            return i\n",
    "        i+=1\n",
    "    return None\n",
    "\n",
    "def return_nodes_with_xpath(nodes, xpaths, results_dict):\n",
    "    returned_nodes = []\n",
    "    for node in nodes:\n",
    "        node.metadata['xpath'] = []\n",
    "        node.metadata['element'] = []\n",
    "        split_html = node.text\n",
    "        soup = BeautifulSoup(split_html, 'html.parser')\n",
    "        for element  in soup.descendants:\n",
    "            try:\n",
    "                attribute = element.attrs\n",
    "                direct_text_content = ''.join([str(content).strip() for content in element.contents if isinstance(content, NavigableString) and content.strip()])\n",
    "                if direct_text_content:\n",
    "                    attribute['text'] = direct_text_content\n",
    "                attribute['element'] = element.name\n",
    "                indice = match_element(attribute, results_dict)\n",
    "                if indice is not None:\n",
    "                    node.metadata['xpath'].append(xpaths[indice])\n",
    "                    node.metadata['element'].append(results_dict[indice])\n",
    "                    returned_nodes.append(node)\n",
    "            except:\n",
    "                pass\n",
    "    return returned_nodes\n",
    "\n",
    "def get_nodes_sm(cohere, query, html, top_n=5, model=\"rerank-english-v3.0\", rank_fields=['element', 'placeholder', 'text', 'name']):\n",
    "    text_list = [html]\n",
    "    documents = [Document(text=t) for t in text_list]\n",
    "    splitter = LangchainNodeParser(lc_splitter=RecursiveCharacterTextSplitter.from_language(\n",
    "            language=\"html\",\n",
    "        ))\n",
    "    nodes = splitter.get_nodes_from_documents(documents)\n",
    "    xpaths, results = get_results(cohere, query, html, top_n=top_n, model=model, rank_fields=rank_fields)\n",
    "    results_dict = [r['document'] for r in results]\n",
    "    \n",
    "    returned_nodes = return_nodes_with_xpath(nodes, xpaths, results_dict)\n",
    "    return returned_nodes\n",
    "\n",
    "def get_nodes_sm_with_xpath(cohere, query, html, top_n=5, model=\"rerank-english-v3.0\", rank_fields=['element', 'placeholder', 'text', 'name']): # used to add xpaths to the returned nodes\n",
    "    nodes = get_nodes_sm(cohere, query, html, top_n, model, rank_fields)\n",
    "    returned_nodes = []\n",
    "    for node in nodes:\n",
    "       returned_nodes.append(node.text + f\"\"\"\\n\n",
    "        Here is a list of some xpaths of element of previous text:\n",
    "        {node.metadata['xpath']}\n",
    "        \\n\\n\n",
    "        \"\"\")\n",
    "    return returned_nodes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "cohere_api_key = #fill with your cohere api key\n",
    "\n",
    "html = requests.get('https://github.com').text\n",
    "query = 'Click on start a free entreprise trial'\n",
    "co = cohere.Client(cohere_api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<div class=\"HeaderMenu-dropdown dropdown-menu rounded m-0 p-0 py-2 py-lg-4 position-relative position-lg-absolute left-0 left-lg-n3 px-lg-4\">\\n          <div class=\"border-bottom pb-3 mb-3\">\\n              <span class=\"d-block h4 color-fg-default my-1\" id=\"solutions-for-heading\">For</span>\\n            <ul class=\"list-style-none f5\" aria-labelledby=\"solutions-for-heading\">\\n                <li>\\n  <a class=\"HeaderMenu-dropdown-link lh-condensed d-block no-underline position-relative py-2 Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Header dropdown (logged out), Solutions&quot;,&quot;action&quot;:&quot;click to go to Enterprise&quot;,&quot;label&quot;:&quot;ref_cta:Enterprise;&quot;}\" href=\"/enterprise\">\\n      Enterprise\\n\\n    \\n</a></li>\\n\\n                <li>\\n  <a class=\"HeaderMenu-dropdown-link lh-condensed d-block no-underline position-relative py-2 Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Header dropdown (logged out), Solutions&quot;,&quot;action&quot;:&quot;click to go to Teams&quot;,&quot;label&quot;:&quot;ref_cta:Teams;&quot;}\" href=\"/team\">\\n      Teams\\n\\n    \\n</a></li>\\n\\n                <li>\\n  <a class=\"HeaderMenu-dropdown-link lh-condensed d-block no-underline position-relative py-2 Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Header dropdown (logged out), Solutions&quot;,&quot;action&quot;:&quot;click to go to Startups&quot;,&quot;label&quot;:&quot;ref_cta:Startups;&quot;}\" href=\"/enterprise/startups\">\\n      Startups\\n\\n    \\n</a></li>\\n\\n                <li>\\n  <a class=\"HeaderMenu-dropdown-link lh-condensed d-block no-underline position-relative py-2 Link--secondary\" target=\"_blank\" data-analytics-event=\"{&quot;category&quot;:&quot;Header dropdown (logged out), Solutions&quot;,&quot;action&quot;:&quot;click to go to Education&quot;,&quot;label&quot;:&quot;ref_cta:Education;&quot;}\" href=\"https://education.github.com\">\\n      Education\\n\\n    <svg aria-hidden=\"true\" height=\"16\" viewBox=\"0 0 16 16\" version=\"1.1\" width=\"16\" data-view-component=\"true\" class=\"octicon octicon-link-external HeaderMenu-external-icon color-fg-subtle\">\\n    <path d=\"M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z\"></path>\\n</svg>\\n</a></li>\\n\\n            </ul>\\n          </div>\\n\\n        Here is a list of some xpaths of element of previous text:\\n        [\\'/html/body/div[1]/div[1]/header/div/div[2]/div/nav/ul/li[2]/div/div[1]/ul/li[1]/a\\']\\n        \\n\\n\\n        ',\n",
       " '<div class=\"border-top border-md-left mx-md-3 mb-3 mb-md-0\"></div>\\n\\n  <a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Start a free enterprise trial&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+hero&amp;ref_page=%2F\" data-test-selector=\"start-trial-button\">\\n  Start a free enterprise trial\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n</div>\\n\\n\\n        <div class=\"my-8 my-md-10 build-in-animate col-12\">\\n          <p class=\"d-block color-fg-subtle f3-mktg\">Trusted by the world’s leading&nbsp;organizations&nbsp;↘︎</p>\\n\\n          <div class=\"d-flex flex-wrap flex-justify-around flex-md-justify-between flex-items-center col-12 col-lg-11\">\\n            <img src=\"https://github.githubassets.com/assets/3m-0151c2fda0ce.svg\" alt=\"3M logo\" height=\"48\" class=\"my-3\">\\n            <img src=\"https://github.githubassets.com/assets/kpmg-c249f20c5173.svg\" alt=\"KPMG logo\" height=\"40\" class=\"my-3\">\\n            <div class=\"col-12 d-sm-none\"></div>\\n            <img src=\"https://github.githubassets.com/assets/mercedes-fcf97d2d6ec4.svg\" alt=\"Mercedes-Benz logo\" height=\"44\" class=\"my-3\">\\n            <img src=\"https://github.githubassets.com/assets/sap-96248a56d312.svg\" alt=\"SAP logo\" height=\"44\" class=\"my-3\">\\n            <div class=\"col-12 d-none d-sm-block d-md-none\"></div>\\n            <div class=\"col-12 d-sm-none\"></div>\\n            <img src=\"https://github.githubassets.com/assets/pg-f1f19955c4e4.svg\" alt=\"P&G logo\" height=\"32\" class=\"my-3\">\\n            <img src=\"https://github.githubassets.com/assets/telus-df0c2109df99.svg\" alt=\"Telus logo\" height=\"32\" class=\"my-3\">\\n          </div>\\n        </div>\\n      </div>\\n    </div>\\n  </div>\\n</div>\\n\\n  <div class=\"position-absolute\">\\n    <div class=\"sub-nav-mktg js-toggler-container z-3 js-sticky js-position-sticky top-0 width-full init-hidden scrollnav\">\\n    <div class=\"sub-nav-mktg-wrapper d-flex flex-items-center py-3 p-responsive\">\\n      \\n\\n      <button\\n        type=\"button\"\\n        class=\"btn-octicon sub-nav-mktg-toggle d-lg-none p-1 flex-order-1 float-right js-toggler-target mr-n1\"\\n        aria-controls=\"sub-nav-dropdown\"\\n        aria-label=\"Page navigation\"\\n        aria-expanded=\"false\"\\n      >\\n        <svg aria-hidden=\"true\" height=\"24\" viewBox=\"0 0 24 24\" version=\"1.1\" width=\"24\" data-view-component=\"true\" class=\"octicon octicon-chevron-down on-hidden\">\\n    <path d=\"M5.22 8.22a.749.749 0 0 0 0 1.06l6.25 6.25a.749.749 0 0 0 1.06 0l6.25-6.25a.749.749 0 1 0-1.06-1.06L12 13.939 6.28 8.22a.749.749 0 0 0-1.06 0Z\"></path>\\n</svg>\\n        <svg aria-hidden=\"true\" height=\"24\" viewBox=\"0 0 24 24\" version=\"1.1\" width=\"24\" data-view-component=\"true\" class=\"octicon octicon-chevron-up on-visible\">\\n    <path d=\"M18.78 15.78a.749.749 0 0 1-1.06 0L12 10.061 6.28 15.78a.749.749 0 1 1-1.06-1.06l6.25-6.25a.749.749 0 0 1 1.06 0l6.25 6.25a.749.749 0 0 1 0 1.06Z\"></path>\\n</svg>\\n      </button>\\n\\n        Here is a list of some xpaths of element of previous text:\\n        [\\'/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[2]/a\\']\\n        \\n\\n\\n        ',\n",
       " '<div class=\"sub-nav-mktg-links flex-auto f4-mktg d-flex flex-column flex-lg-row gap-4\" id=\"sub-nav-dropdown\">\\n          <a class=\"sub-nav-mktg-link Link--primary no-underline py-1 py-lg-2 js-scrollnav-item js-smoothscroll-anchor\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click item to scroll to Productivity&quot;,&quot;label&quot;:&quot;ref_cta:Productivity;&quot;}\" href=\"#productivity\">Productivity</a>\\n          <a class=\"sub-nav-mktg-link Link--primary no-underline py-1 py-lg-2 js-scrollnav-item js-smoothscroll-anchor\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click item to scroll to Security&quot;,&quot;label&quot;:&quot;ref_cta:Security;&quot;}\" href=\"#security\">Security</a>\\n          <a class=\"sub-nav-mktg-link Link--primary no-underline py-1 py-lg-2 js-scrollnav-item js-smoothscroll-anchor\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click item to scroll to Collaboration&quot;,&quot;label&quot;:&quot;ref_cta:Collaboration;&quot;}\" href=\"#collaboration\">Collaboration</a>\\n      </div>\\n\\n        <div class=\"sub-nav-mktg-ctas ml-lg-3\">\\n            <a class=\"btn-mktg ml-lg-2 mt-2 mt-lg-0 d-block d-lg-inline-block btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;ref_loc:Homepage;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=homepage+sticky+nav&amp;ref_page=%2F\">\\n  Start a free enterprise trial\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n            <a class=\"btn-mktg ml-lg-2 mt-2 mt-lg-0 d-block d-lg-inline-block\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click to Sign up for GitHub&quot;,&quot;label&quot;:&quot;ref_cta:Sign up for GitHub;ref_loc:Homepage;&quot;}\" href=\"/join?ref_cta=Sign+up+for+GitHub&amp;ref_loc=homepage+sticky+nav&amp;ref_page=%2F&amp;source=homepage-sticky-nav\">\\n  Sign up for GitHub\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n        </div>\\n    </div>\\n</div>\\n</div>\\n\\n  <div class=\"home-campaign-productivity js-section overflow-hidden px-3 pb-8 pt-5\" data-color-mode=\"dark\" data-light-theme=\"light\" data-dark-theme=\"dark\" id=\"productivity\">\\n  <div class=\"container-xl\">\\n\\n        Here is a list of some xpaths of element of previous text:\\n        [\\'/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[1]\\']\\n        \\n\\n\\n        ',\n",
       " '<div class=\"border-top border-md-left mx-md-3 mb-3 mb-md-0\"></div>\\n\\n  <a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Start a free enterprise trial&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+footer&amp;ref_page=%2F\" data-test-selector=\"start-trial-button\">\\n  Start a free enterprise trial\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n</div>\\n\\n        </div>\\n      </div>\\n    </div>\\n  </div>\\n\\n    <div class=\"position-relative z-2 user-select-none home-globe-scene js-webgl-globe-data\">\\n  <div class=\"home-globe-container home-globe-container-webgl mx-auto\">\\n\\n        Here is a list of some xpaths of element of previous text:\\n        [\\'/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/a\\']\\n        \\n\\n\\n        ',\n",
       " '<p class=\"f5 color-fg-muted mb-3\">Get tips, technical guides, and best practices. Twice a month. Right in your inbox.</p>\\n        <a class=\"btn-mktg mb-4 btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Subscribe&quot;,&quot;action&quot;:&quot;click to Subscribe&quot;,&quot;label&quot;:&quot;ref_cta:Subscribe;&quot;}\" href=\"https://resources.github.com/newsletter/\">\\n  Subscribe\\n  \\n  \\n</a>\\n\\n      </section>\\n\\n      <nav class=\"col-6 col-sm-3 col-lg-2 mb-6 mb-md-2 pr-3 pr-lg-0 pl-lg-4\" aria-label=\"Links related to product features\">\\n        <h3 class=\"h5 mb-3 text-mono color-fg-muted text-normal\">\\n          Product\\n        </h3>\\n\\n        <ul class=\"list-style-none color-fg-muted f5\">\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to features&quot;,&quot;label&quot;:&quot;text:features&quot;}\" href=\"/features\">Features</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to enterprise&quot;,&quot;label&quot;:&quot;text:enterprise&quot;}\" href=\"/enterprise\">Enterprise</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to copilot&quot;,&quot;label&quot;:&quot;text:copilot&quot;}\" href=\"/features/copilot\">Copilot</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to security&quot;,&quot;label&quot;:&quot;text:security&quot;}\" href=\"/security\">Security</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to pricing&quot;,&quot;label&quot;:&quot;text:pricing&quot;}\" href=\"/pricing\">Pricing</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to team&quot;,&quot;label&quot;:&quot;text:team&quot;}\" href=\"/team\">Team</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to resources&quot;,&quot;label&quot;:&quot;text:resources&quot;}\" href=\"https://resources.github.com\">Resources</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to roadmap&quot;,&quot;label&quot;:&quot;text:roadmap&quot;}\" href=\"https://github.com/github/roadmap\">Roadmap</a>\\n          </li>\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to compare github&quot;,&quot;label&quot;:&quot;text:compare github&quot;}\" href=\"https://resources.github.com/devops/tools/compare\">Compare GitHub</a>\\n          </li>\\n        </ul>\\n      </nav>\\n\\n      <nav class=\"col-6 col-sm-3 col-lg-2 mb-6 mb-md-2 pr-3 pr-md-0 pl-md-4\" aria-label=\"Links related to platform services and features\">\\n        <h3 class=\"h5 mb-3 text-mono color-fg-muted text-normal\">\\n          Platform\\n        </h3>\\n\\n        <ul class=\"list-style-none f5\">\\n          <li class=\"lh-condensed mb-3\">\\n            <a class=\"Link--secondary\" data-analytics-event=\"{&quot;category&quot;:&quot;Footer&quot;,&quot;action&quot;:&quot;go to api&quot;,&quot;label&quot;:&quot;text:api&quot;}\" href=\"https://docs.github.com/get-started/exploring-integrations/about-building-integrations\">Developer API</a>\\n          </li>\\n\\n        Here is a list of some xpaths of element of previous text:\\n        [\\'/html/body/div[1]/footer/div[1]/div/nav[1]/ul/li[2]/a\\']\\n        \\n\\n\\n        ']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes = get_nodes_sm_with_xpath(co, query, html, top_n=5, model=\"rerank-english-v3.0\", rank_fields=['element', 'placeholder', 'text', 'name'])\n",
    "nodes"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lavague_test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}