Skip to content

Commit

Permalink
langchain[patch]: Migrate retrievers to use optional langchain commun…
Browse files Browse the repository at this point in the history
…ity imports (#21155)
  • Loading branch information
eyurtsev committed May 1, 2024
1 parent 43110da commit bec3eee
Show file tree
Hide file tree
Showing 38 changed files with 1,000 additions and 183 deletions.
91 changes: 49 additions & 42 deletions libs/community/langchain_community/retrievers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
from langchain_community.retrievers.arxiv import (
ArxivRetriever,
)
from langchain_community.retrievers.azure_cognitive_search import (
from langchain_community.retrievers.azure_ai_search import (
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
)
from langchain_community.retrievers.bedrock import (
Expand Down Expand Up @@ -100,6 +101,9 @@
from langchain_community.retrievers.qdrant_sparse_vector_retriever import (
QdrantSparseVectorRetriever,
)
from langchain_community.retrievers.rememberizer import (
RememberizerRetriever,
)
from langchain_community.retrievers.remote_retriever import (
RemoteLangChainRetriever,
)
Expand All @@ -112,6 +116,7 @@
from langchain_community.retrievers.tfidf import (
TFIDFRetriever,
)
from langchain_community.retrievers.thirdai_neuraldb import NeuralDBRetriever
from langchain_community.retrievers.vespa_retriever import (
VespaRetriever,
)
Expand All @@ -131,46 +136,6 @@
ZillizRetriever,
)

__all__ = [
"AmazonKendraRetriever",
"AmazonKnowledgeBasesRetriever",
"ArceeRetriever",
"ArxivRetriever",
"AzureCognitiveSearchRetriever",
"BM25Retriever",
"BreebsRetriever",
"ChaindeskRetriever",
"ChatGPTPluginRetriever",
"CohereRagRetriever",
"DocArrayRetriever",
"DriaRetriever",
"ElasticSearchBM25Retriever",
"EmbedchainRetriever",
"GoogleCloudEnterpriseSearchRetriever",
"GoogleDocumentAIWarehouseRetriever",
"GoogleVertexAIMultiTurnSearchRetriever",
"GoogleVertexAISearchRetriever",
"KNNRetriever",
"KayAiRetriever",
"LlamaIndexGraphRetriever",
"LlamaIndexRetriever",
"MetalRetriever",
"MilvusRetriever",
"OutlineRetriever",
"PineconeHybridSearchRetriever",
"PubMedRetriever",
"QdrantSparseVectorRetriever",
"RemoteLangChainRetriever",
"SVMRetriever",
"TFIDFRetriever",
"TavilySearchAPIRetriever",
"VespaRetriever",
"WeaviateHybridSearchRetriever",
"WikipediaRetriever",
"YouRetriever",
"ZepRetriever",
"ZillizRetriever",
]

_module_lookup = {
"AmazonKendraRetriever": "langchain_community.retrievers.kendra",
Expand Down Expand Up @@ -224,4 +189,46 @@ def __getattr__(name: str) -> Any:
raise AttributeError(f"module {__name__} has no attribute {name}")


__all__ = list(_module_lookup.keys())
__all__ = [
"AmazonKendraRetriever",
"AmazonKnowledgeBasesRetriever",
"ArceeRetriever",
"ArxivRetriever",
"AzureCognitiveSearchRetriever",
"AzureAISearchRetriever",
"BM25Retriever",
"BreebsRetriever",
"ChaindeskRetriever",
"ChatGPTPluginRetriever",
"CohereRagRetriever",
"DocArrayRetriever",
"DriaRetriever",
"ElasticSearchBM25Retriever",
"EmbedchainRetriever",
"GoogleCloudEnterpriseSearchRetriever",
"GoogleDocumentAIWarehouseRetriever",
"GoogleVertexAIMultiTurnSearchRetriever",
"GoogleVertexAISearchRetriever",
"KNNRetriever",
"KayAiRetriever",
"LlamaIndexGraphRetriever",
"LlamaIndexRetriever",
"MetalRetriever",
"MilvusRetriever",
"NeuralDBRetriever",
"OutlineRetriever",
"PineconeHybridSearchRetriever",
"PubMedRetriever",
"QdrantSparseVectorRetriever",
"RememberizerRetriever",
"RemoteLangChainRetriever",
"SVMRetriever",
"TFIDFRetriever",
"TavilySearchAPIRetriever",
"VespaRetriever",
"WeaviateHybridSearchRetriever",
"WikipediaRetriever",
"YouRetriever",
"ZepRetriever",
"ZillizRetriever",
]
2 changes: 2 additions & 0 deletions libs/community/tests/unit_tests/retrievers/test_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
"RememberizerRetriever",
"SVMRetriever",
"TavilySearchAPIRetriever",
"NeuralDBRetriever",
"RememberizerRetriever",
"TFIDFRetriever",
"BM25Retriever",
"VespaRetriever",
Expand Down
118 changes: 101 additions & 17 deletions libs/langchain/langchain/retrievers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
Document, Serializable, Callbacks,
CallbackManagerForRetrieverRun, AsyncCallbackManagerForRetrieverRun
"""
from typing import Any
from typing import TYPE_CHECKING, Any

from langchain._api.module_import import create_importer
from langchain.retrievers.contextual_compression import ContextualCompressionRetriever
Expand All @@ -34,14 +34,96 @@
)
from langchain.retrievers.web_research import WebResearchRetriever

import_lookup = create_importer(
__package__, fallback_module="langchain_community.retrievers"
)
if TYPE_CHECKING:
from langchain_community.retrievers import (
AmazonKendraRetriever,
AmazonKnowledgeBasesRetriever,
ArceeRetriever,
ArxivRetriever,
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
BM25Retriever,
ChaindeskRetriever,
ChatGPTPluginRetriever,
CohereRagRetriever,
DocArrayRetriever,
DriaRetriever,
ElasticSearchBM25Retriever,
EmbedchainRetriever,
GoogleCloudEnterpriseSearchRetriever,
GoogleDocumentAIWarehouseRetriever,
GoogleVertexAIMultiTurnSearchRetriever,
GoogleVertexAISearchRetriever,
KayAiRetriever,
KNNRetriever,
LlamaIndexGraphRetriever,
LlamaIndexRetriever,
MetalRetriever,
MilvusRetriever,
NeuralDBRetriever,
OutlineRetriever,
PineconeHybridSearchRetriever,
PubMedRetriever,
RemoteLangChainRetriever,
SVMRetriever,
TavilySearchAPIRetriever,
TFIDFRetriever,
VespaRetriever,
WeaviateHybridSearchRetriever,
WikipediaRetriever,
ZepRetriever,
ZillizRetriever,
)

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {
"AmazonKendraRetriever": "langchain_community.retrievers",
"AmazonKnowledgeBasesRetriever": "langchain_community.retrievers",
"ArceeRetriever": "langchain_community.retrievers",
"ArxivRetriever": "langchain_community.retrievers",
"AzureAISearchRetriever": "langchain_community.retrievers",
"AzureCognitiveSearchRetriever": "langchain_community.retrievers",
"ChatGPTPluginRetriever": "langchain_community.retrievers",
"ChaindeskRetriever": "langchain_community.retrievers",
"CohereRagRetriever": "langchain_community.retrievers",
"ElasticSearchBM25Retriever": "langchain_community.retrievers",
"EmbedchainRetriever": "langchain_community.retrievers",
"GoogleDocumentAIWarehouseRetriever": "langchain_community.retrievers",
"GoogleCloudEnterpriseSearchRetriever": "langchain_community.retrievers",
"GoogleVertexAIMultiTurnSearchRetriever": "langchain_community.retrievers",
"GoogleVertexAISearchRetriever": "langchain_community.retrievers",
"KayAiRetriever": "langchain_community.retrievers",
"KNNRetriever": "langchain_community.retrievers",
"LlamaIndexGraphRetriever": "langchain_community.retrievers",
"LlamaIndexRetriever": "langchain_community.retrievers",
"MetalRetriever": "langchain_community.retrievers",
"MilvusRetriever": "langchain_community.retrievers",
"OutlineRetriever": "langchain_community.retrievers",
"PineconeHybridSearchRetriever": "langchain_community.retrievers",
"PubMedRetriever": "langchain_community.retrievers",
"RemoteLangChainRetriever": "langchain_community.retrievers",
"SVMRetriever": "langchain_community.retrievers",
"TavilySearchAPIRetriever": "langchain_community.retrievers",
"TFIDFRetriever": "langchain_community.retrievers",
"BM25Retriever": "langchain_community.retrievers",
"VespaRetriever": "langchain_community.retrievers",
"NeuralDBRetriever": "langchain_community.retrievers",
"DriaRetriever": "langchain_community.retrievers",
"WeaviateHybridSearchRetriever": "langchain_community.retrievers",
"WikipediaRetriever": "langchain_community.retrievers",
"ZepRetriever": "langchain_community.retrievers",
"ZillizRetriever": "langchain_community.retrievers",
"DocArrayRetriever": "langchain_community.retrievers",
}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Import retrievers from langchain_community."""
return import_lookup(name)
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
Expand All @@ -51,14 +133,18 @@ def __getattr__(name: str) -> Any:
"ArxivRetriever",
"AzureAISearchRetriever",
"AzureCognitiveSearchRetriever",
"ChatGPTPluginRetriever",
"ContextualCompressionRetriever",
"BM25Retriever",
"ChaindeskRetriever",
"ChatGPTPluginRetriever",
"CohereRagRetriever",
"ContextualCompressionRetriever",
"DocArrayRetriever",
"DriaRetriever",
"ElasticSearchBM25Retriever",
"EmbedchainRetriever",
"GoogleDocumentAIWarehouseRetriever",
"EnsembleRetriever",
"GoogleCloudEnterpriseSearchRetriever",
"GoogleDocumentAIWarehouseRetriever",
"GoogleVertexAIMultiTurnSearchRetriever",
"GoogleVertexAISearchRetriever",
"KayAiRetriever",
Expand All @@ -69,25 +155,23 @@ def __getattr__(name: str) -> Any:
"MetalRetriever",
"MilvusRetriever",
"MultiQueryRetriever",
"MultiVectorRetriever",
"OutlineRetriever",
"ParentDocumentRetriever",
"PineconeHybridSearchRetriever",
"PubMedRetriever",
"RemoteLangChainRetriever",
"SVMRetriever",
"RePhraseQueryRetriever",
"SelfQueryRetriever",
"SVMRetriever",
"TavilySearchAPIRetriever",
"TFIDFRetriever",
"BM25Retriever",
"TimeWeightedVectorStoreRetriever",
"VespaRetriever",
"WeaviateHybridSearchRetriever",
"WebResearchRetriever",
"WikipediaRetriever",
"ZepRetriever",
"NeuralDBRetriever",
"ZillizRetriever",
"DocArrayRetriever",
"RePhraseQueryRetriever",
"WebResearchRetriever",
"EnsembleRetriever",
"ParentDocumentRetriever",
"MultiVectorRetriever",
]
24 changes: 22 additions & 2 deletions libs/langchain/langchain/retrievers/arcee.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
from langchain_community.retrievers.arcee import ArceeRetriever
from typing import TYPE_CHECKING, Any

__all__ = ["ArceeRetriever"]
from langchain._api import create_importer

if TYPE_CHECKING:
from langchain_community.retrievers import ArceeRetriever

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {"ArceeRetriever": "langchain_community.retrievers"}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
"ArceeRetriever",
]
24 changes: 22 additions & 2 deletions libs/langchain/langchain/retrievers/arxiv.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
from langchain_community.retrievers.arxiv import ArxivRetriever
from typing import TYPE_CHECKING, Any

__all__ = ["ArxivRetriever"]
from langchain._api import create_importer

if TYPE_CHECKING:
from langchain_community.retrievers import ArxivRetriever

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {"ArxivRetriever": "langchain_community.retrievers"}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
"ArxivRetriever",
]
34 changes: 29 additions & 5 deletions libs/langchain/langchain/retrievers/azure_ai_search.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,30 @@
from langchain_community.retrievers.azure_ai_search import (
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
)
from typing import TYPE_CHECKING, Any

__all__ = ["AzureAISearchRetriever", "AzureCognitiveSearchRetriever"]
from langchain._api import create_importer

if TYPE_CHECKING:
from langchain_community.retrievers import (
AzureAISearchRetriever,
AzureCognitiveSearchRetriever,
)

# Create a way to dynamically look up deprecated imports.
# Used to consolidate logic for raising deprecation warnings and
# handling optional imports.
DEPRECATED_LOOKUP = {
"AzureAISearchRetriever": "langchain_community.retrievers",
"AzureCognitiveSearchRetriever": "langchain_community.retrievers",
}

_import_attribute = create_importer(__package__, deprecated_lookups=DEPRECATED_LOOKUP)


def __getattr__(name: str) -> Any:
"""Look up attributes dynamically."""
return _import_attribute(name)


__all__ = [
"AzureAISearchRetriever",
"AzureCognitiveSearchRetriever",
]

0 comments on commit bec3eee

Please sign in to comment.