Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

langchain[patch]: Migrate retrievers to use optional langchain community imports #21155

Merged
merged 7 commits into from
May 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 49 additions & 42 deletions libs/community/langchain_community/retrievers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
from langchain_community.retrievers.arxiv import (
ArxivRetriever,
)
from langchain_community.retrievers.azure_cognitive_search import (
from langchain_community.retrievers.azure_ai_search import (
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
)
from langchain_community.retrievers.bedrock import (
Expand Down Expand Up @@ -100,6 +101,9 @@
from langchain_community.retrievers.qdrant_sparse_vector_retriever import (
QdrantSparseVectorRetriever,
)
from langchain_community.retrievers.rememberizer import (
RememberizerRetriever,
)
from langchain_community.retrievers.remote_retriever import (
RemoteLangChainRetriever,
)
Expand All @@ -112,6 +116,7 @@
from langchain_community.retrievers.tfidf import (
TFIDFRetriever,
)
from langchain_community.retrievers.thirdai_neuraldb import NeuralDBRetriever
from langchain_community.retrievers.vespa_retriever import (
VespaRetriever,
)
Expand All @@ -131,46 +136,6 @@
ZillizRetriever,
)

__all__ = [
"AmazonKendraRetriever",
"AmazonKnowledgeBasesRetriever",
"ArceeRetriever",
"ArxivRetriever",
"AzureCognitiveSearchRetriever",
"BM25Retriever",
"BreebsRetriever",
"ChaindeskRetriever",
"ChatGPTPluginRetriever",
"CohereRagRetriever",
"DocArrayRetriever",
"DriaRetriever",
"ElasticSearchBM25Retriever",
"EmbedchainRetriever",
"GoogleCloudEnterpriseSearchRetriever",
"GoogleDocumentAIWarehouseRetriever",
"GoogleVertexAIMultiTurnSearchRetriever",
"GoogleVertexAISearchRetriever",
"KNNRetriever",
"KayAiRetriever",
"LlamaIndexGraphRetriever",
"LlamaIndexRetriever",
"MetalRetriever",
"MilvusRetriever",
"OutlineRetriever",
"PineconeHybridSearchRetriever",
"PubMedRetriever",
"QdrantSparseVectorRetriever",
"RemoteLangChainRetriever",
"SVMRetriever",
"TFIDFRetriever",
"TavilySearchAPIRetriever",
"VespaRetriever",
"WeaviateHybridSearchRetriever",
"WikipediaRetriever",
"YouRetriever",
"ZepRetriever",
"ZillizRetriever",
]

_module_lookup = {
"AmazonKendraRetriever": "langchain_community.retrievers.kendra",
Expand Down Expand Up @@ -224,4 +189,46 @@ def __getattr__(name: str) -> Any:
raise AttributeError(f"module {__name__} has no attribute {name}")


__all__ = list(_module_lookup.keys())
__all__ = [
"AmazonKendraRetriever",
"AmazonKnowledgeBasesRetriever",
"ArceeRetriever",
"ArxivRetriever",
"AzureCognitiveSearchRetriever",
"AzureAISearchRetriever",
"BM25Retriever",
"BreebsRetriever",
"ChaindeskRetriever",
"ChatGPTPluginRetriever",
"CohereRagRetriever",
"DocArrayRetriever",
"DriaRetriever",
"ElasticSearchBM25Retriever",
"EmbedchainRetriever",
"GoogleCloudEnterpriseSearchRetriever",
"GoogleDocumentAIWarehouseRetriever",
"GoogleVertexAIMultiTurnSearchRetriever",
"GoogleVertexAISearchRetriever",
"KNNRetriever",
"KayAiRetriever",
"LlamaIndexGraphRetriever",
"LlamaIndexRetriever",
"MetalRetriever",
"MilvusRetriever",
"NeuralDBRetriever",
"OutlineRetriever",
"PineconeHybridSearchRetriever",
"PubMedRetriever",
"QdrantSparseVectorRetriever",
"RememberizerRetriever",
"RemoteLangChainRetriever",
"SVMRetriever",
"TFIDFRetriever",
"TavilySearchAPIRetriever",
"VespaRetriever",
"WeaviateHybridSearchRetriever",
"WikipediaRetriever",
"YouRetriever",
"ZepRetriever",
"ZillizRetriever",
]
2 changes: 2 additions & 0 deletions libs/community/tests/unit_tests/retrievers/test_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
"RememberizerRetriever",
"SVMRetriever",
"TavilySearchAPIRetriever",
"NeuralDBRetriever",
"RememberizerRetriever",
"TFIDFRetriever",
"BM25Retriever",
"VespaRetriever",
Expand Down
118 changes: 101 additions & 17 deletions libs/langchain/langchain/retrievers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
Document, Serializable, Callbacks,
CallbackManagerForRetrieverRun, AsyncCallbackManagerForRetrieverRun
"""
from typing import Any
from typing import TYPE_CHECKING, Any

from langchain._api.module_import import create_importer
from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
Expand All @@ -34,14 +34,96 @@
)
from langchain.retrievers.web_research import WebResearchRetriever

import_lookup = create_importer(
__package__, fallback_module="langchain_community.retrievers"
)
if TYPE_CHECKING:
from langchain_community.retrievers import (
AmazonKendraRetriever,
AmazonKnowledgeBasesRetriever,
ArceeRetriever,
ArxivRetriever,
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
BM25Retriever,
ChaindeskRetriever,
ChatGPTPluginRetriever,
CohereRagRetriever,
DocArrayRetriever,
DriaRetriever,
ElasticSearchBM25Retriever,
EmbedchainRetriever,
GoogleCloudEnterpriseSearchRetriever,
GoogleDocumentAIWarehouseRetriever,
GoogleVertexAIMultiTurnSearchRetriever,
GoogleVertexAISearchRetriever,
KayAiRetriever,
KNNRetriever,
LlamaIndexGraphRetriever,
LlamaIndexRetriever,
MetalRetriever,
MilvusRetriever,
NeuralDBRetriever,
OutlineRetriever,
PineconeHybridSearchRetriever,
PubMedRetriever,
RemoteLangChainRetriever,
SVMRetriever,
TavilySearchAPIRetriever,
TFIDFRetriever,
VespaRetriever,
WeaviateHybridSearchRetriever,
WikipediaRetriever,
ZepRetriever,
ZillizRetriever,
)

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {
"AmazonKendraRetriever": "langchain_community.retrievers",
"AmazonKnowledgeBasesRetriever": "langchain_community.retrievers",
"ArceeRetriever": "langchain_community.retrievers",
"ArxivRetriever": "langchain_community.retrievers",
"AzureAISearchRetriever": "langchain_community.retrievers",
"AzureCognitiveSearchRetriever": "langchain_community.retrievers",
"ChatGPTPluginRetriever": "langchain_community.retrievers",
"ChaindeskRetriever": "langchain_community.retrievers",
"CohereRagRetriever": "langchain_community.retrievers",
"ElasticSearchBM25Retriever": "langchain_community.retrievers",
"EmbedchainRetriever": "langchain_community.retrievers",
"GoogleDocumentAIWarehouseRetriever": "langchain_community.retrievers",
"GoogleCloudEnterpriseSearchRetriever": "langchain_community.retrievers",
"GoogleVertexAIMultiTurnSearchRetriever": "langchain_community.retrievers",
"GoogleVertexAISearchRetriever": "langchain_community.retrievers",
"KayAiRetriever": "langchain_community.retrievers",
"KNNRetriever": "langchain_community.retrievers",
"LlamaIndexGraphRetriever": "langchain_community.retrievers",
"LlamaIndexRetriever": "langchain_community.retrievers",
"MetalRetriever": "langchain_community.retrievers",
"MilvusRetriever": "langchain_community.retrievers",
"OutlineRetriever": "langchain_community.retrievers",
"PineconeHybridSearchRetriever": "langchain_community.retrievers",
"PubMedRetriever": "langchain_community.retrievers",
"RemoteLangChainRetriever": "langchain_community.retrievers",
"SVMRetriever": "langchain_community.retrievers",
"TavilySearchAPIRetriever": "langchain_community.retrievers",
"TFIDFRetriever": "langchain_community.retrievers",
"BM25Retriever": "langchain_community.retrievers",
"VespaRetriever": "langchain_community.retrievers",
"NeuralDBRetriever": "langchain_community.retrievers",
"DriaRetriever": "langchain_community.retrievers",
"WeaviateHybridSearchRetriever": "langchain_community.retrievers",
"WikipediaRetriever": "langchain_community.retrievers",
"ZepRetriever": "langchain_community.retrievers",
"ZillizRetriever": "langchain_community.retrievers",
"DocArrayRetriever": "langchain_community.retrievers",
}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Import retrievers from langchain_community."""
return import_lookup(name)
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
Expand All @@ -51,14 +133,18 @@ def __getattr__(name: str) -> Any:
"ArxivRetriever",
"AzureAISearchRetriever",
"AzureCognitiveSearchRetriever",
"ChatGPTPluginRetriever",
"ContextualCompressionRetriever",
"BM25Retriever",
"ChaindeskRetriever",
"ChatGPTPluginRetriever",
"CohereRagRetriever",
"ContextualCompressionRetriever",
"DocArrayRetriever",
"DriaRetriever",
"ElasticSearchBM25Retriever",
"EmbedchainRetriever",
"GoogleDocumentAIWarehouseRetriever",
"EnsembleRetriever",
"GoogleCloudEnterpriseSearchRetriever",
"GoogleDocumentAIWarehouseRetriever",
"GoogleVertexAIMultiTurnSearchRetriever",
"GoogleVertexAISearchRetriever",
"KayAiRetriever",
Expand All @@ -69,25 +155,23 @@ def __getattr__(name: str) -> Any:
"MetalRetriever",
"MilvusRetriever",
"MultiQueryRetriever",
"MultiVectorRetriever",
"OutlineRetriever",
"ParentDocumentRetriever",
"PineconeHybridSearchRetriever",
"PubMedRetriever",
"RemoteLangChainRetriever",
"SVMRetriever",
"RePhraseQueryRetriever",
"SelfQueryRetriever",
"SVMRetriever",
"TavilySearchAPIRetriever",
"TFIDFRetriever",
"BM25Retriever",
"TimeWeightedVectorStoreRetriever",
"VespaRetriever",
"WeaviateHybridSearchRetriever",
"WebResearchRetriever",
"WikipediaRetriever",
"ZepRetriever",
"NeuralDBRetriever",
"ZillizRetriever",
"DocArrayRetriever",
"RePhraseQueryRetriever",
"WebResearchRetriever",
"EnsembleRetriever",
"ParentDocumentRetriever",
"MultiVectorRetriever",
]
24 changes: 22 additions & 2 deletions libs/langchain/langchain/retrievers/arcee.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
from langchain_community.retrievers.arcee import ArceeRetriever
from typing import TYPE_CHECKING, Any

__all__ = ["ArceeRetriever"]
from langchain._api import create_importer

if TYPE_CHECKING:
from langchain_community.retrievers import ArceeRetriever

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {"ArceeRetriever": "langchain_community.retrievers"}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
"ArceeRetriever",
]
24 changes: 22 additions & 2 deletions libs/langchain/langchain/retrievers/arxiv.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
from langchain_community.retrievers.arxiv import ArxivRetriever
from typing import TYPE_CHECKING, Any

__all__ = ["ArxivRetriever"]
from langchain._api import create_importer

if TYPE_CHECKING:
from langchain_community.retrievers import ArxivRetriever

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {"ArxivRetriever": "langchain_community.retrievers"}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
"ArxivRetriever",
]
34 changes: 29 additions & 5 deletions libs/langchain/langchain/retrievers/azure_ai_search.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,30 @@
from langchain_community.retrievers.azure_ai_search import (
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
)
from typing import TYPE_CHECKING, Any

__all__ = ["AzureAISearchRetriever", "AzureCognitiveSearchRetriever"]
from langchain._api import create_importer

if TYPE_CHECKING:
from langchain_community.retrievers import (
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
)

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {
"AzureAISearchRetriever": "langchain_community.retrievers",
"AzureCognitiveSearchRetriever": "langchain_community.retrievers",
}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
"AzureAISearchRetriever",
"AzureCognitiveSearchRetriever",
]
Loading
Loading