op_sm+split.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup, NavigableString\n",
    "import ast\n",
    "from llama_index.retrievers.bm25 import BM25Retriever\n",
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core.schema import TextNode\n",
    "from llama_index.core import Document\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "from llama_index.core.node_parser import LangchainNodeParser\n",
    "from llama_index.core.schema import NodeWithScore, QueryBundle\n",
    "\n",
    "def generate_xpath(element, path=\"\"): # used to generate dict nodes\n",
    "    \"\"\" Recursive function to generate the xpath of an element \"\"\"\n",
    "    if element.parent is None:\n",
    "        return path\n",
    "    else:\n",
    "        siblings = [sib for sib in element.parent.children if sib.name == element.name]\n",
    "        if len(siblings) > 1:\n",
    "            count = siblings.index(element) + 1\n",
    "            path = f\"/{element.name}[{count}]{path}\"\n",
    "        else:\n",
    "            path = f\"/{element.name}{path}\"\n",
    "        return generate_xpath(element.parent, path)\n",
    "\n",
    "def create_xpath_dict(html, only_body=True, max_length=200): # used to generate dict nodes\n",
    "    ''' Create a list of xpaths and a list of dict of attributes of all elements in the html'''\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    if only_body:\n",
    "        root = soup.body\n",
    "    else:\n",
    "        root = soup.html\n",
    "    element_xpath_list = []\n",
    "    element_attributes_list = []\n",
    "    stack = [(root, '')]  # stack to keep track of elements and their paths\n",
    "    while stack:\n",
    "        element, path = stack.pop()\n",
    "        if element.name is not None:\n",
    "            current_path = generate_xpath(element)\n",
    "            element_attrs = dict(element.attrs)\n",
    "            direct_text_content = ''.join([str(content).strip() for content in element.contents if isinstance(content, NavigableString) and content.strip()])\n",
    "            if direct_text_content:\n",
    "                element_attrs['text'] = direct_text_content\n",
    "                element_attrs['element'] = element.name\n",
    "                for key in element_attrs:\n",
    "                    if len(element_attrs[key]) > max_length:\n",
    "                        element_attrs[key] = element_attrs[key][:max_length]\n",
    "                element_xpath_list.append(current_path)\n",
    "                element_attributes_list.append(element_attrs)\n",
    "            elif element_attrs != {}:\n",
    "                element_attrs['element'] = element.name\n",
    "                for key in element_attrs:\n",
    "                    if len(element_attrs[key]) > max_length:\n",
    "                        element_attrs[key] = element_attrs[key][:max_length]\n",
    "                element_xpath_list.append(current_path)\n",
    "                element_attributes_list.append(element_attrs)\n",
    "            for child in element.children:\n",
    "                if child.name is not None:\n",
    "                    stack.append((child, current_path))\n",
    "\n",
    "    return element_xpath_list, element_attributes_list\n",
    "\n",
    "def clean_attributes(attributes_list, xpath_list, rank_fields): # used to generate dict nodes\n",
    "    for attr, xpath in zip(attributes_list, xpath_list):\n",
    "        attr['xpath'] = xpath\n",
    "    if rank_fields:\n",
    "        rank_fields.append('xpath')\n",
    "        attributes_list = [{k: v for k, v in d.items() if k in rank_fields} for d in attributes_list]\n",
    "    attributes_list = [d for d in attributes_list if (not((len(list(d.keys()))==2) and (('element' in list(d.keys())) and 'xpath' in list(d.keys())))) or d=={}]\n",
    "    return attributes_list\n",
    "\n",
    "def chunk_dicts(dicts, chunk_size=10):\n",
    "    def chunks(lst, n):\n",
    "        for i in range(0, len(lst), n):\n",
    "            yield lst[i:i + n]\n",
    "    grouped_chunks = []\n",
    "    for chunk in chunks(dicts, chunk_size):\n",
    "        all_keys = set(key for d in chunk for key in d.keys()) \n",
    "        grouped = {key: [] for key in all_keys}\n",
    "        for d in chunk:\n",
    "            for key in all_keys:\n",
    "                grouped[key].append(d.get(key, ''))  \n",
    "        grouped_chunks.append(grouped)   \n",
    "    return grouped_chunks\n",
    "\n",
    "def unchunk_dicts(grouped_chunks):\n",
    "    flat_list = []\n",
    "    for group in grouped_chunks:\n",
    "        max_length = max(len(v) for v in group.values())\n",
    "        for i in range(max_length):\n",
    "            new_dict = {}\n",
    "            for key, values in group.items():\n",
    "                if i < len(values):\n",
    "                    if values[i] != '':\n",
    "                        new_dict[key] = values[i]\n",
    "            if new_dict:\n",
    "                flat_list.append(new_dict)\n",
    "    return flat_list\n",
    "\n",
    "\n",
    "def get_results(query, html, embedder, top_n=5, group_by=10, rank_fields=None): # used to generate and retrieve dict nodes\n",
    "    ''' Return the top_n elements of the html that are the most relevant to the query as Node objects with xpath in their metadata'''\n",
    "    xpath_list, attributes_list = create_xpath_dict(html)\n",
    "    assert len(xpath_list) == len(attributes_list)\n",
    "    assert group_by > 0\n",
    "    #cleaning the attributes_list\n",
    "    attributes_list = clean_attributes(attributes_list, xpath_list, rank_fields)\n",
    "    #retrieving the top_n results\n",
    "    \n",
    "    list_of_results = []\n",
    "    attributes_list = chunk_dicts(attributes_list, group_by)\n",
    "    l = len(attributes_list)\n",
    "    #grouping the attributes_list in groups of 1000 to avoid memory errors\n",
    "    list_of_grouped_results = []\n",
    "    for j in range(0, l, 1000):\n",
    "        nodes = []\n",
    "        attr = attributes_list[j:j+1000]\n",
    "        for d in attr:\n",
    "            xpath = d.pop('xpath')\n",
    "            nodes.append(TextNode(text=str(d), metadata={'xpath': xpath}))\n",
    "        index = VectorStoreIndex(nodes, embed_model=embedder)\n",
    "        retriever = BM25Retriever.from_defaults(index = index, similarity_top_k=top_n)\n",
    "        results = retriever.retrieve(query)\n",
    "        list_of_grouped_results += results\n",
    "    nodes = []\n",
    "    for grouped_results in list_of_grouped_results:\n",
    "        xpaths = grouped_results.metadata['xpath']\n",
    "        ds = unchunk_dicts([ast.literal_eval(grouped_results.text)])\n",
    "        assert len(xpaths) == len(ds)\n",
    "        for xpath, d in zip(xpaths, ds):\n",
    "            nodes.append(TextNode(text=str(d), metadata={'xpath': xpath}))\n",
    "    l2 = len(nodes)\n",
    "    for j in range(0, l2, 1000):\n",
    "        index = VectorStoreIndex(nodes[j:j+1000], embed_model=embedder)\n",
    "        retriever = BM25Retriever.from_defaults(index = index, similarity_top_k=top_n)\n",
    "        results = retriever.retrieve(query)\n",
    "        list_of_results += results\n",
    "    list_of_results = sorted(list_of_results, key=lambda x: x.score, reverse=True)\n",
    "    return list_of_results[:top_n]\n",
    "\n",
    "def match_element(attributes, element_specs): # used to find chunk nodes corresponding to retrieve dict nodes\n",
    "    ''' Return the index of the element in element_specs that matches the attributes of the element'''\n",
    "    i=0\n",
    "    for spec in element_specs:\n",
    "        matches = True\n",
    "        for key in spec:\n",
    "            if key in attributes:\n",
    "                if isinstance(attributes[key], list):\n",
    "                    if not set(spec[key]).issubset(set(attributes[key])):\n",
    "                        matches = False\n",
    "                        break\n",
    "                elif attributes[key] != spec[key]:\n",
    "                    matches = False\n",
    "                    break\n",
    "            else:\n",
    "                matches = False\n",
    "                break\n",
    "        if matches:\n",
    "            return i\n",
    "        i+=1\n",
    "    return None\n",
    "\n",
    "def return_nodes_with_xpath(nodes, results_dict, score): # used to find chunk nodes corresponding to retrieve dict nodes\n",
    "    ''' Return the chunk nodes that have an element that matches the attributes of the elements in results_dict'''\n",
    "    returned_nodes = []\n",
    "    for node in nodes:\n",
    "        split_html = node.text\n",
    "        soup = BeautifulSoup(split_html, 'html.parser')\n",
    "        for element  in soup.descendants:\n",
    "            try:\n",
    "                attribute = element.attrs\n",
    "                direct_text_content = ''.join([str(content).strip() for content in element.contents if isinstance(content, NavigableString) and content.strip()])\n",
    "                if direct_text_content:\n",
    "                    attribute['text'] = direct_text_content\n",
    "                attribute['element'] = element.name\n",
    "                indice = match_element(attribute, results_dict)\n",
    "                if indice is not None:\n",
    "                    node.metadata['score'] = score[indice]\n",
    "                    node.metadata['element'] = results_dict[indice]\n",
    "                    returned_nodes.append(node)\n",
    "            except:\n",
    "                pass\n",
    "    return returned_nodes\n",
    "\n",
    "def get_nodes_sm(query, html, embedder, top_n=5, group_by=10, rank_fields=['element', 'placeholder', 'text', 'name']):\n",
    "    if isinstance(query, QueryBundle):\n",
    "        query = query.query_str\n",
    "    text_list = [html]\n",
    "    documents = [Document(text=t) for t in text_list]\n",
    "    splitter = LangchainNodeParser(lc_splitter=RecursiveCharacterTextSplitter.from_language(\n",
    "            language=\"html\",\n",
    "        ))\n",
    "    #chunk nodes\n",
    "    nodes = splitter.get_nodes_from_documents(documents)\n",
    "    #dict nodes\n",
    "    results = get_results(query, html, embedder=embedder, top_n=top_n, group_by=group_by, rank_fields = rank_fields)\n",
    "    results_dict = [ast.literal_eval(r.text) for r in results]\n",
    "    score = [r.score for r in results]\n",
    "    #find chunk nodes corresponding to retrieve dict nodes\n",
    "    returned_nodes = return_nodes_with_xpath(nodes, results_dict, score)\n",
    "    returned_nodes = [NodeWithScore(node=node, score=node.metadata['score']) for node in returned_nodes]\n",
    "    \n",
    "    return returned_nodes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
    "import os\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] =  #to fill \n",
    "\n",
    "html = requests.get('https://github.com').text\n",
    "query = 'Click on start a free entreprise trial'\n",
    "embedder = OpenAIEmbedding(model= \"text-embedding-3-large\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = get_nodes_sm(QueryBundle(query), html, embedder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[NodeWithScore(node=TextNode(id_='629fa3ea-4a9d-49f3-b23e-c31ada983183', embedding=None, metadata={'score': 1.2890889229649445, 'element': {'element': 'script', 'text': '{\"props\":{\"docsUrl\":\"https://docs.github.com/get-started/accessibility/keyboard-shortcuts\"}}'}}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='be7a85c0-08f0-477b-9cc0-3d802cbb1aa4', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='f466d0bc4408e01ffe7d53ee0132e2d089420ecf5481d96612e231ea65a147ef'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='7a1597e5-85e0-4aed-8a1e-b0e9bf306af1', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='468cca38ca3717daeae0d277bce1417aecbcdd791a41e2b1162bffb91a816948'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='9c870601-ccf8-402b-bef7-6cf709c3cf5c', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='bdfecd730cfc5eb108c6eecf454eee33ef5827a2d83f3f93106cff1ed29bf6f4')}, text='<body class=\"logged-out env-production page-responsive header-overlay home-campaign\" style=\"word-wrap: break-word;\">\\n    <div data-turbo-body class=\"logged-out env-production page-responsive header-overlay home-campaign\" style=\"word-wrap: break-word;\">\\n      \\n\\n\\n    <div class=\"position-relative js-header-wrapper \">\\n      <h2 class=\"sr-only\">Navigation Menu</h2>\\n      <a href=\"#start-of-content\" data-skip-target-assigned=\"false\" class=\"px-2 py-4 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content\">Skip to content</a>\\n\\n      <span data-view-component=\"true\" class=\"progress-pjax-loader Progress position-fixed width-full\">\\n    <span style=\"width: 0%;\" data-view-component=\"true\" class=\"Progress-item progress-pjax-loader-bar left-0 top-0 color-bg-accent-emphasis\"></span>\\n</span>      \\n      \\n      \\n\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/react-lib-1fbfc5be2c18.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_octicons-react_dist_index_esm_js-node_modules_primer_react_lib-es-541a38-ade861844008.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_Box_Box_js-8f8c5e2a2cbf.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_Button_Button_js-95a7748e3c39.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_TooltipV2_Tooltip_js-5c105bd4b6bc.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_node_modules_primer_octicons-react_dist_index_esm_mjs-cb996b1b8e38.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_ActionList_index_js-f41028bf9254.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_Button_IconButton_js-node_modules_primer_react_lib--b964b4-f8441db8b94b.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/ui_packages_react-core_create-browser-history_ts-ui_packages_safe-storage_safe-storage_ts-ui_-682c2c-e45e451173ec.js\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" type=\"application/javascript\" src=\"https://github.githubassets.com/assets/keyboard-shortcuts-dialog-b4f13290b41c.js\"></script>\\n\\n<react-partial\\n  partial-name=\"keyboard-shortcuts-dialog\"\\n  data-ssr=\"false\"\\n>\\n  \\n  <script type=\"application/json\" data-target=\"react-partial.embeddedData\">{\"props\":{\"docsUrl\":\"https://docs.github.com/get-started/accessibility/keyboard-shortcuts\"}}</script>', start_char_idx=19215, end_char_idx=22440, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=1.2890889229649445),\n",
       " NodeWithScore(node=TextNode(id_='ecfdcd13-db53-4cf4-b02c-7cb78df208c6', embedding=None, metadata={'score': 7.196651180314209, 'element': {'element': 'a', 'text': 'Start a free enterprise trial'}}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='be7a85c0-08f0-477b-9cc0-3d802cbb1aa4', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='f466d0bc4408e01ffe7d53ee0132e2d089420ecf5481d96612e231ea65a147ef'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='8a8fa38c-f398-426e-b998-1623e2d94b1e', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='86d0b1155b68ce58b95ae9fec2d51911f7b5c144d8ff06a97e1ce59ad26bbcdc'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='b8add2d4-6ec0-463f-8bdb-1f1d9c179ccf', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='0bb078e6405bbe0d04b0e2247e0fec1c2dd2b32f6f787db2f22b92f9b222f7e6')}, text='<div class=\"border-top border-md-left mx-md-3 mb-3 mb-md-0\"></div>\\n\\n  <a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Start a free enterprise trial&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+hero&amp;ref_page=%2F\" data-test-selector=\"start-trial-button\">\\n  Start a free enterprise trial\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n</div>\\n\\n\\n        <div class=\"my-8 my-md-10 build-in-animate col-12\">\\n          <p class=\"d-block color-fg-subtle f3-mktg\">Trusted by the world’s leading&nbsp;organizations&nbsp;↘︎</p>\\n\\n          <div class=\"d-flex flex-wrap flex-justify-around flex-md-justify-between flex-items-center col-12 col-lg-11\">\\n            <img src=\"https://github.githubassets.com/assets/3m-0151c2fda0ce.svg\" alt=\"3M logo\" height=\"48\" class=\"my-3\">\\n            <img src=\"https://github.githubassets.com/assets/kpmg-c249f20c5173.svg\" alt=\"KPMG logo\" height=\"40\" class=\"my-3\">\\n            <div class=\"col-12 d-sm-none\"></div>\\n            <img src=\"https://github.githubassets.com/assets/mercedes-fcf97d2d6ec4.svg\" alt=\"Mercedes-Benz logo\" height=\"44\" class=\"my-3\">\\n            <img src=\"https://github.githubassets.com/assets/sap-96248a56d312.svg\" alt=\"SAP logo\" height=\"44\" class=\"my-3\">\\n            <div class=\"col-12 d-none d-sm-block d-md-none\"></div>\\n            <div class=\"col-12 d-sm-none\"></div>\\n            <img src=\"https://github.githubassets.com/assets/pg-f1f19955c4e4.svg\" alt=\"P&G logo\" height=\"32\" class=\"my-3\">\\n            <img src=\"https://github.githubassets.com/assets/telus-df0c2109df99.svg\" alt=\"Telus logo\" height=\"32\" class=\"my-3\">\\n          </div>\\n        </div>\\n      </div>\\n    </div>\\n  </div>\\n</div>\\n\\n  <div class=\"position-absolute\">\\n    <div class=\"sub-nav-mktg js-toggler-container z-3 js-sticky js-position-sticky top-0 width-full init-hidden scrollnav\">\\n    <div class=\"sub-nav-mktg-wrapper d-flex flex-items-center py-3 p-responsive\">\\n      \\n\\n      <button\\n        type=\"button\"\\n        class=\"btn-octicon sub-nav-mktg-toggle d-lg-none p-1 flex-order-1 float-right js-toggler-target mr-n1\"\\n        aria-controls=\"sub-nav-dropdown\"\\n        aria-label=\"Page navigation\"\\n        aria-expanded=\"false\"\\n      >\\n        <svg aria-hidden=\"true\" height=\"24\" viewBox=\"0 0 24 24\" version=\"1.1\" width=\"24\" data-view-component=\"true\" class=\"octicon octicon-chevron-down on-hidden\">\\n    <path d=\"M5.22 8.22a.749.749 0 0 0 0 1.06l6.25 6.25a.749.749 0 0 0 1.06 0l6.25-6.25a.749.749 0 1 0-1.06-1.06L12 13.939 6.28 8.22a.749.749 0 0 0-1.06 0Z\"></path>\\n</svg>\\n        <svg aria-hidden=\"true\" height=\"24\" viewBox=\"0 0 24 24\" version=\"1.1\" width=\"24\" data-view-component=\"true\" class=\"octicon octicon-chevron-up on-visible\">\\n    <path d=\"M18.78 15.78a.749.749 0 0 1-1.06 0L12 10.061 6.28 15.78a.749.749 0 1 1-1.06-1.06l6.25-6.25a.749.749 0 0 1 1.06 0l6.25 6.25a.749.749 0 0 1 0 1.06Z\"></path>\\n</svg>\\n      </button>', start_char_idx=107354, end_char_idx=111214, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=7.196651180314209),\n",
       " NodeWithScore(node=TextNode(id_='b8add2d4-6ec0-463f-8bdb-1f1d9c179ccf', embedding=None, metadata={'score': 7.196651180314209, 'element': {'element': 'a', 'text': 'Start a free enterprise trial'}}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='be7a85c0-08f0-477b-9cc0-3d802cbb1aa4', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='f466d0bc4408e01ffe7d53ee0132e2d089420ecf5481d96612e231ea65a147ef'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='ecfdcd13-db53-4cf4-b02c-7cb78df208c6', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='6fd96581df61a170279a70e4037542c46ca90cefd787cfc8f051457cc880eec9'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='6f0af55a-2758-4e0c-ba27-52c15824fb1b', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='3d7dca66b2599ca22b2917d82fd1da9fc67aa97aadc928ad84833e29421375b1')}, text='<div class=\"sub-nav-mktg-links flex-auto f4-mktg d-flex flex-column flex-lg-row gap-4\" id=\"sub-nav-dropdown\">\\n          <a class=\"sub-nav-mktg-link Link--primary no-underline py-1 py-lg-2 js-scrollnav-item js-smoothscroll-anchor\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click item to scroll to Productivity&quot;,&quot;label&quot;:&quot;ref_cta:Productivity;&quot;}\" href=\"#productivity\">Productivity</a>\\n          <a class=\"sub-nav-mktg-link Link--primary no-underline py-1 py-lg-2 js-scrollnav-item js-smoothscroll-anchor\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click item to scroll to Security&quot;,&quot;label&quot;:&quot;ref_cta:Security;&quot;}\" href=\"#security\">Security</a>\\n          <a class=\"sub-nav-mktg-link Link--primary no-underline py-1 py-lg-2 js-scrollnav-item js-smoothscroll-anchor\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click item to scroll to Collaboration&quot;,&quot;label&quot;:&quot;ref_cta:Collaboration;&quot;}\" href=\"#collaboration\">Collaboration</a>\\n      </div>\\n\\n        <div class=\"sub-nav-mktg-ctas ml-lg-3\">\\n            <a class=\"btn-mktg ml-lg-2 mt-2 mt-lg-0 d-block d-lg-inline-block btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;ref_loc:Homepage;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=homepage+sticky+nav&amp;ref_page=%2F\">\\n  Start a free enterprise trial\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n            <a class=\"btn-mktg ml-lg-2 mt-2 mt-lg-0 d-block d-lg-inline-block\" data-analytics-event=\"{&quot;category&quot;:&quot;Homepage&quot;,&quot;action&quot;:&quot;click to Sign up for GitHub&quot;,&quot;label&quot;:&quot;ref_cta:Sign up for GitHub;ref_loc:Homepage;&quot;}\" href=\"/join?ref_cta=Sign+up+for+GitHub&amp;ref_loc=homepage+sticky+nav&amp;ref_page=%2F&amp;source=homepage-sticky-nav\">\\n  Sign up for GitHub\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n        </div>\\n    </div>\\n</div>\\n</div>\\n\\n  <div class=\"home-campaign-productivity js-section overflow-hidden px-3 pb-8 pt-5\" data-color-mode=\"dark\" data-light-theme=\"light\" data-dark-theme=\"dark\" id=\"productivity\">\\n  <div class=\"container-xl\">', start_char_idx=111222, end_char_idx=115159, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=7.196651180314209),\n",
       " NodeWithScore(node=TextNode(id_='67055319-2dfe-47d5-8b92-2cce6602e8f9', embedding=None, metadata={'score': 2.0713224489746325, 'element': {'element': 'a', 'text': 'Get started with GitHub Issues'}}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='be7a85c0-08f0-477b-9cc0-3d802cbb1aa4', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='f466d0bc4408e01ffe7d53ee0132e2d089420ecf5481d96612e231ea65a147ef'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='1dd85e19-691a-4b4a-b1ec-99615a23c3b6', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='eae3c3b34bc602e33d587757309c795026435016cf64f56981dfc4a752d2cb1b'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='771c4c85-7e63-4b53-9a45-0c08cff71954', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='d4a775b820d84bd8892d61dbd076d231ce365c107d332d83f29fb1739c612c53')}, text='<div class=\"col-12 py-3 mb-2 js-build-in-item col-sm-10 col-md-6 text-left col-lg-6 build-in-slideX-right\">\\n      <div class=\"pr-md-4\">\\n        \\n        \\n        \\n        \\n        <h2 class=\"color-fg-default mb-3 f2-mktg color-fg-muted\"><em>GitHub Issues and GitHub Projects</em> supply project management tools that adapt to your team alongside your code.</h2>\\n        \\n          <a class=\"link-mktg text-semibold color-fg-default py-1 f3-mktg\" href=\"/features/issues\" data-analytics-event=\"{&quot;category&quot;:&quot;Get started with GitHub Issues&quot;,&quot;action&quot;:&quot;click to Get started with GitHub Issues&quot;,&quot;label&quot;:&quot;ref_cta:Get started with GitHub Issues;&quot;}\">\\n    Get started with GitHub Issues <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n        \\n      </div>\\n    </div>\\n\\n      <div class=\"col-12 py-3 col-md-6 col-lg-6\">\\n        <div class=\"position-relative\">\\n          \\n        <div class=\"d-flex flex-column position-relative\">\\n          <div class=\"js-build-in-item build-in-slideY border box-shadow-mktg-xl overflow-hidden rounded-2 position-relative left-4 z-1\" style=\"max-width: 595px; margin-top: -50%;\">\\n            <img src=\"https://github.githubassets.com/assets/illu-projects-2-26077f1dd188.png\" alt=\"\" loading=\"lazy\" decoding=\"async\" width=\"1190\" height=\"964\" class=\"width-full height-auto d-block\">\\n          </div>\\n\\n          <img alt=\"\" aria-hidden=\"true\" width=\"1000\" height=\"1000\" class=\"position-absolute d-none d-md-block events-none js-build-in-item build-in-fade\" style=\"bottom: -300px; right: -240px;\" loading=\"lazy\" decoding=\"async\" src=\"https://github.githubassets.com/assets/bg-glow-purple-6e9a6a96cb04.png\" />\\n        </div>\\n\\n        </div>\\n      </div>\\n</div>\\n      <div class=\"mb-6 mb-md-9 pb-4 js-build-in-trigger\">\\n        <div class=\"py-3 text-left col-sm-10 col-md-6 col-lg-3 js-build-in-item build-in-slideX-left\">\\n          <div class=\"gradient-border-mktg d-inline-block z-1 position-relative px-2 lh-condensed f6-mktg\" style=\"top: -1px\">\\n  <span class=\"text-gradient-mktg text-semibold\">Did you know?</span>\\n</div>\\n\\n          <h3 class=\"text-accent-primary mb-2 mt-3 h1-mktg text-medium\">80%</h3>\\n          <p class=\"mb-3 f2-mktg text-medium col-6-max\">reduction in onboarding time with GitHub<sup>2</sup></p>\\n        </div>\\n      </div>\\n    </div>\\n  </div>\\n\\n  <div class=\"d-flex flex-column flex-md-row gutter js-build-in-trigger \" >\\n  \\n    <div class=\"position-relative z-1 col-md-6 d-flex\">\\n        <card-skew class=\"overflow-hidden rounded-3 mb-3 mb-md-5\">\\n    <div class=\"feature-card-mktg d-md-flex rounded-3 color-bg-subtle border position-relative flex-justify-between z-1 height-full flex-column\" data-target=\"card-skew.bounding\">\\n      <div class=\"d-md-flex flex-column flex-1 p-5 p-sm-6 py-lg-8 pl-lg-8 pr-lg-12 flex-justify-start\">\\n        <p class=\"f2-mktg text-medium color-fg-muted mb-4\"><em>GitHub Discussions</em> creates space to ask questions and have open-ended conversations.</p>', start_char_idx=161522, end_char_idx=165254, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=2.0713224489746325),\n",
       " NodeWithScore(node=TextNode(id_='447d4e9f-edce-47bd-9284-510358efef62', embedding=None, metadata={'score': 7.196651180314209, 'element': {'element': 'a', 'text': 'Start a free enterprise trial'}}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='be7a85c0-08f0-477b-9cc0-3d802cbb1aa4', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='f466d0bc4408e01ffe7d53ee0132e2d089420ecf5481d96612e231ea65a147ef'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='93b261d5-929f-49d2-8c4e-3034a12030a0', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='cfd262bf9b471b98ef0b4dd3f9572e5be9cc9636b33d4a6fbb0d8ae79592b2c2'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='01ce8d0f-6a50-4d0e-b1a9-2ce67eb1f0a4', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='1262c38e63d5030fd702cee25b9cce6419711ed4428be3f4e29a91a844df7c80')}, text='<div class=\"border-top border-md-left mx-md-3 mb-3 mb-md-0\"></div>\\n\\n  <a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Start a free enterprise trial&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+footer&amp;ref_page=%2F\" data-test-selector=\"start-trial-button\">\\n  Start a free enterprise trial\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>\\n</div>\\n\\n        </div>\\n      </div>\\n    </div>\\n  </div>\\n\\n    <div class=\"position-relative z-2 user-select-none home-globe-scene js-webgl-globe-data\">\\n  <div class=\"home-globe-container home-globe-container-webgl mx-auto\">', start_char_idx=191541, end_char_idx=193120, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=7.196651180314209)]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lavague_test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}