Skip to content

Commit

Permalink
community[patch]: Fix source path mismatch in PebbloSafeLoader (#23857)
Browse files Browse the repository at this point in the history
**Description:** Fix for source path mismatch in PebbloSafeLoader. The
fix involves storing the full path in the doc metadata in VectorDB
**Issue:** NA, caught in internal testing
**Dependencies:** NA
**Add tests**:  Updated tests
  • Loading branch information
Raj725 committed Jul 5, 2024
1 parent 5b7d5f7 commit ee8aa54
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 2 deletions.
14 changes: 14 additions & 0 deletions libs/community/langchain_community/document_loaders/pebblo.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ def load(self) -> List[Document]:
list: Documents fetched from load method of the wrapped `loader`.
"""
self.docs = self.loader.load()
# Add pebblo-specific metadata to docs
self._add_pebblo_specific_metadata()
if not self.load_semantic:
self._classify_doc(self.docs, loading_end=True)
return self.docs
Expand Down Expand Up @@ -123,6 +125,8 @@ def lazy_load(self) -> Iterator[Document]:
self.docs = []
break
self.docs = list((doc,))
# Add pebblo-specific metadata to docs
self._add_pebblo_specific_metadata()
if not self.load_semantic:
self._classify_doc(self.docs, loading_end=True)
yield self.docs[0]
Expand Down Expand Up @@ -517,3 +521,13 @@ def _add_semantic_to_doc(self, doc: Document, classified_doc: dict) -> Document:
classified_doc.get("topics", {}).keys()
)
return doc

def _add_pebblo_specific_metadata(self) -> None:
"""Add Pebblo specific metadata to documents."""
for doc in self.docs:
doc_metadata = doc.metadata
doc_metadata["full_path"] = get_full_path(
doc_metadata.get(
"full_path", doc_metadata.get("source", self.source_path)
)
)
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,15 @@ def test_csv_loader_load_valid_data(mocker: MockerFixture) -> None:
post=MockResponse(json_data={"data": ""}, status_code=200),
)
file_path = os.path.join(EXAMPLE_DOCS_DIRECTORY, "test_nominal.csv")
full_file_path = os.path.abspath(file_path)
expected_docs = [
Document(
page_content="column1: value1\ncolumn2: value2\ncolumn3: value3",
metadata={"source": file_path, "row": 0},
metadata={"source": file_path, "row": 0, "full_path": full_file_path},
),
Document(
page_content="column1: value4\ncolumn2: value5\ncolumn3: value6",
metadata={"source": file_path, "row": 1},
metadata={"source": file_path, "row": 1, "full_path": full_file_path},
),
]

Expand Down

0 comments on commit ee8aa54

Please sign in to comment.