Modify the load_data method to avoid duplicates and reduce the number… (

#12977)

llama-index-integrations/readers/llama-index-readers-web/llama_index/readers/web/whole_site/base.py

@@ -100,29 +100,30 @@ def load_data(self, base_url: str) -> List[Document]:
 
         while urls_to_visit:
             current_url, depth = urls_to_visit.pop(0)
-
             print(f"Visiting: {current_url}, {len(urls_to_visit)} left")
 
-            if depth > self.max_depth:
-                continue
             try:
                 self.driver.get(current_url)
                 page_content = self.extract_content()
-
-                # links = self.driver.find_elements(By.TAG_NAME, 'a')
-                links = self.extract_links()
-                # clean all urls
-                links = [self.clean_url(link) for link in links]
-                # extract new links
-                links = [link for link in links if link not in added_urls]
-                print(f"Found {len(links)} new potential links")
-                for href in links:
-                    try:
-                        if href.startswith(self.prefix) and href not in added_urls:
-                            urls_to_visit.append((href, depth + 1))
-                            added_urls.add(href)
-                    except Exception:
-                        continue
+                added_urls.add(current_url)
+
+                next_depth = depth + 1
+                if next_depth <= self.max_depth:
+                    # links = self.driver.find_elements(By.TAG_NAME, 'a')
+                    links = self.extract_links()
+                    # clean all urls
+                    links = [self.clean_url(link) for link in links]
+                    # extract new links
+                    links = [link for link in links if link not in added_urls]
+                    print(f"Found {len(links)} new potential links")
+
+                    for href in links:
+                        try:
+                            if href.startswith(self.prefix) and href not in added_urls:
+                                urls_to_visit.append((href, next_depth))
+                                added_urls.add(href)
+                        except Exception:
+                            continue
 
                 documents.append(
                     Document(text=page_content, extra_info={"URL": current_url})

llama-index-integrations/readers/llama-index-readers-web/pyproject.toml

@@ -41,7 +41,7 @@ license = "MIT"
 maintainers = ["HawkClaws", "Hironsan", "NA", "an-bluecat", "bborn", "jasonwcfan", "kravetsmic", "pandazki", "ruze00", "selamanse", "thejessezhang"]
 name = "llama-index-readers-web"
 readme = "README.md"
-version = "0.1.9"
+version = "0.1.10"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"