From e6806a08d4126a32484dd25abfd15a1a32a1171a Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Thu, 11 Apr 2024 18:23:13 -0700 Subject: [PATCH] multiple: standard chat model tests (#20359) --- libs/partners/ai21/poetry.lock | 22 ++++++++++++++++--- libs/partners/ai21/pyproject.toml | 1 + .../tests/integration_tests/test_standard.py | 21 ++++++++++++++++++ .../ai21/tests/unit_tests/test_standard.py | 22 +++++++++++++++++++ libs/partners/anthropic/poetry.lock | 21 ++++++++++++++++-- libs/partners/anthropic/pyproject.toml | 1 + .../tests/integration_tests/test_standard.py | 21 ++++++++++++++++++ .../tests/unit_tests/test_standard.py | 21 ++++++++++++++++++ libs/partners/fireworks/poetry.lock | 21 ++++++++++++++++-- libs/partners/fireworks/pyproject.toml | 1 + .../tests/integration_tests/test_standard.py | 15 +++++++++++++ .../tests/unit_tests/test_standard.py | 21 ++++++++++++++++++ libs/partners/groq/poetry.lock | 21 ++++++++++++++++-- libs/partners/groq/pyproject.toml | 1 + .../tests/integration_tests/test_standard.py | 15 +++++++++++++ .../groq/tests/unit_tests/test_standard.py | 15 +++++++++++++ .../unit_tests/chat_models.py | 8 +++++-- 17 files changed, 237 insertions(+), 11 deletions(-) create mode 100644 libs/partners/ai21/tests/integration_tests/test_standard.py create mode 100644 libs/partners/ai21/tests/unit_tests/test_standard.py create mode 100644 libs/partners/anthropic/tests/integration_tests/test_standard.py create mode 100644 libs/partners/anthropic/tests/unit_tests/test_standard.py create mode 100644 libs/partners/fireworks/tests/integration_tests/test_standard.py create mode 100644 libs/partners/fireworks/tests/unit_tests/test_standard.py create mode 100644 libs/partners/groq/tests/integration_tests/test_standard.py create mode 100644 libs/partners/groq/tests/unit_tests/test_standard.py diff --git a/libs/partners/ai21/poetry.lock b/libs/partners/ai21/poetry.lock index a7a14a813b83e2..6e67fe665dac41 100644 --- a/libs/partners/ai21/poetry.lock +++ b/libs/partners/ai21/poetry.lock @@ -278,7 +278,7 @@ files = [ [[package]] name = "langchain-core" -version = "0.1.37" +version = "0.1.42" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" @@ -291,7 +291,6 @@ langsmith = "^0.1.0" packaging = "^23.2" pydantic = ">=1,<3" PyYAML = ">=5.3" -requests = "^2" tenacity = "^8.1.0" [package.extras] @@ -301,6 +300,23 @@ extended-testing = ["jinja2 (>=3,<4)"] type = "directory" url = "../../core" +[[package]] +name = "langchain-standard-tests" +version = "0.1.0" +description = "Standard tests for LangChain implementations" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [] +develop = true + +[package.dependencies] +langchain-core = "^0.1.40" +pytest = ">=7,<9" + +[package.source] +type = "directory" +url = "../../standard-tests" + [[package]] name = "langchain-text-splitters" version = "0.0.1" @@ -994,4 +1010,4 @@ watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "d0e6ec94729c40ea458eead2404d2b501f18dd67c211c832146352410223276e" +content-hash = "446d53423f89a3a378db9cff7ce8cb392e146e294d31e9d1bbfc23108a571097" diff --git a/libs/partners/ai21/pyproject.toml b/libs/partners/ai21/pyproject.toml index 94135fb0b76561..845d77cb8e4f4e 100644 --- a/libs/partners/ai21/pyproject.toml +++ b/libs/partners/ai21/pyproject.toml @@ -22,6 +22,7 @@ syrupy = "^4.0.2" pytest-watcher = "^0.3.4" pytest-asyncio = "^0.21.1" langchain-core = { path = "../../core", develop = true } +langchain-standard-tests = {path = "../../standard-tests", develop = true} [tool.poetry.group.codespell] optional = true diff --git a/libs/partners/ai21/tests/integration_tests/test_standard.py b/libs/partners/ai21/tests/integration_tests/test_standard.py new file mode 100644 index 00000000000000..5c62b025999d17 --- /dev/null +++ b/libs/partners/ai21/tests/integration_tests/test_standard.py @@ -0,0 +1,21 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.integration_tests import ChatModelIntegrationTests + +from langchain_ai21 import ChatAI21 + + +class TestAI21Standard(ChatModelIntegrationTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatAI21 + + @pytest.fixture + def chat_model_params(self) -> dict: + return { + "model": "j2-ultra", + } diff --git a/libs/partners/ai21/tests/unit_tests/test_standard.py b/libs/partners/ai21/tests/unit_tests/test_standard.py new file mode 100644 index 00000000000000..6b9ebf71b36833 --- /dev/null +++ b/libs/partners/ai21/tests/unit_tests/test_standard.py @@ -0,0 +1,22 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.unit_tests import ChatModelUnitTests + +from langchain_ai21 import ChatAI21 + + +class TestAI21Standard(ChatModelUnitTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatAI21 + + @pytest.fixture + def chat_model_params(self) -> dict: + return { + "model": "j2-ultra", + "api_key": "test_api_key", + } diff --git a/libs/partners/anthropic/poetry.lock b/libs/partners/anthropic/poetry.lock index f567a305ef951a..5525b16ed24632 100644 --- a/libs/partners/anthropic/poetry.lock +++ b/libs/partners/anthropic/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -460,6 +460,23 @@ extended-testing = ["jinja2 (>=3,<4)"] type = "directory" url = "../../core" +[[package]] +name = "langchain-standard-tests" +version = "0.1.0" +description = "Standard tests for LangChain implementations" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [] +develop = true + +[package.dependencies] +langchain-core = "^0.1.40" +pytest = ">=7,<9" + +[package.source] +type = "directory" +url = "../../standard-tests" + [[package]] name = "langsmith" version = "0.1.42" @@ -1206,4 +1223,4 @@ watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "dce6101bb01cd9ab521a8b8df906db7b4631d8305952a442c78db9a77f2b2f1f" +content-hash = "adc0024beed52fca8f0f9e1786aaf5c25b3bf6fd138fb9668b44ef36c6bcf23f" diff --git a/libs/partners/anthropic/pyproject.toml b/libs/partners/anthropic/pyproject.toml index 74bbca52056a3d..e7320a9075d40d 100644 --- a/libs/partners/anthropic/pyproject.toml +++ b/libs/partners/anthropic/pyproject.toml @@ -28,6 +28,7 @@ pytest-watcher = "^0.3.4" pytest-asyncio = "^0.21.1" langchain-core = { path = "../../core", develop = true } defusedxml = "^0.7.1" +langchain-standard-tests = {path = "../../standard-tests", develop = true} [tool.poetry.group.codespell] optional = true diff --git a/libs/partners/anthropic/tests/integration_tests/test_standard.py b/libs/partners/anthropic/tests/integration_tests/test_standard.py new file mode 100644 index 00000000000000..464f5f947e5f6c --- /dev/null +++ b/libs/partners/anthropic/tests/integration_tests/test_standard.py @@ -0,0 +1,21 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.integration_tests import ChatModelIntegrationTests + +from langchain_anthropic import ChatAnthropic + + +class TestAnthropicStandard(ChatModelIntegrationTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatAnthropic + + @pytest.fixture + def chat_model_params(self) -> dict: + return { + "model": "claude-3-haiku-20240307", + } diff --git a/libs/partners/anthropic/tests/unit_tests/test_standard.py b/libs/partners/anthropic/tests/unit_tests/test_standard.py new file mode 100644 index 00000000000000..2650554e79bd36 --- /dev/null +++ b/libs/partners/anthropic/tests/unit_tests/test_standard.py @@ -0,0 +1,21 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.unit_tests import ChatModelUnitTests + +from langchain_anthropic import ChatAnthropic + + +class TestAnthropicStandard(ChatModelUnitTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatAnthropic + + @pytest.fixture + def chat_model_params(self) -> dict: + return { + "model": "claude-3-haiku-20240307", + } diff --git a/libs/partners/fireworks/poetry.lock b/libs/partners/fireworks/poetry.lock index 2e26da35df0c6e..41d5dd1bb9ce3a 100644 --- a/libs/partners/fireworks/poetry.lock +++ b/libs/partners/fireworks/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -594,6 +594,23 @@ extended-testing = ["jinja2 (>=3,<4)"] type = "directory" url = "../../core" +[[package]] +name = "langchain-standard-tests" +version = "0.1.0" +description = "Standard tests for LangChain implementations" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [] +develop = true + +[package.dependencies] +langchain-core = "^0.1.40" +pytest = ">=7,<9" + +[package.source] +type = "directory" +url = "../../standard-tests" + [[package]] name = "langsmith" version = "0.1.10" @@ -1536,4 +1553,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "5f4c474fcbc2ef84c95a0e4992c0acdb08e4fa5817909576d31caf793e54109b" +content-hash = "fbf305613a6134e08c9efec406928b30ba7830a13a87c9a523708699b7efc9a3" diff --git a/libs/partners/fireworks/pyproject.toml b/libs/partners/fireworks/pyproject.toml index 94187b5f9e9722..8cbc5d0766f00b 100644 --- a/libs/partners/fireworks/pyproject.toml +++ b/libs/partners/fireworks/pyproject.toml @@ -29,6 +29,7 @@ syrupy = "^4.0.2" pytest-watcher = "^0.3.4" pytest-asyncio = "^0.21.1" langchain-core = { path = "../../core", develop = true } +langchain-standard-tests = {path = "../../standard-tests", develop = true} [tool.poetry.group.codespell] optional = true diff --git a/libs/partners/fireworks/tests/integration_tests/test_standard.py b/libs/partners/fireworks/tests/integration_tests/test_standard.py new file mode 100644 index 00000000000000..c8f9c05c1bd7b4 --- /dev/null +++ b/libs/partners/fireworks/tests/integration_tests/test_standard.py @@ -0,0 +1,15 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.integration_tests import ChatModelIntegrationTests + +from langchain_fireworks import ChatFireworks + + +class TestFireworksStandard(ChatModelIntegrationTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatFireworks diff --git a/libs/partners/fireworks/tests/unit_tests/test_standard.py b/libs/partners/fireworks/tests/unit_tests/test_standard.py new file mode 100644 index 00000000000000..455af60288d2fd --- /dev/null +++ b/libs/partners/fireworks/tests/unit_tests/test_standard.py @@ -0,0 +1,21 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.unit_tests import ChatModelUnitTests + +from langchain_fireworks import ChatFireworks + + +class TestFireworksStandard(ChatModelUnitTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatFireworks + + @pytest.fixture + def chat_model_params(self) -> dict: + return { + "api_key": "test_api_key", + } diff --git a/libs/partners/groq/poetry.lock b/libs/partners/groq/poetry.lock index 2fb25039847de5..6fb18686953f53 100644 --- a/libs/partners/groq/poetry.lock +++ b/libs/partners/groq/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -345,6 +345,23 @@ extended-testing = ["jinja2 (>=3,<4)"] type = "directory" url = "../../core" +[[package]] +name = "langchain-standard-tests" +version = "0.1.0" +description = "Standard tests for LangChain implementations" +optional = false +python-versions = ">=3.8.1,<4.0" +files = [] +develop = true + +[package.dependencies] +langchain-core = "^0.1.40" +pytest = ">=7,<9" + +[package.source] +type = "directory" +url = "../../standard-tests" + [[package]] name = "langsmith" version = "0.1.4" @@ -867,4 +884,4 @@ watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "f76983ba51b9fc343a68563a9f2d847efe9c5856bbeff5796d9146c521221a53" +content-hash = "1692a375c2817216876453275294e5aa2500364b7e36ae2b4b0ec1fe1837402e" diff --git a/libs/partners/groq/pyproject.toml b/libs/partners/groq/pyproject.toml index cce56df85fe541..16bfa90a1ffd3a 100644 --- a/libs/partners/groq/pyproject.toml +++ b/libs/partners/groq/pyproject.toml @@ -24,6 +24,7 @@ pytest-mock = "^3.10.0" pytest-watcher = "^0.3.4" pytest-asyncio = "^0.21.1" langchain-core = { path = "../../core", develop = true } +langchain-standard-tests = {path = "../../standard-tests", develop = true} [tool.poetry.group.codespell] optional = true diff --git a/libs/partners/groq/tests/integration_tests/test_standard.py b/libs/partners/groq/tests/integration_tests/test_standard.py new file mode 100644 index 00000000000000..83ca841caa2233 --- /dev/null +++ b/libs/partners/groq/tests/integration_tests/test_standard.py @@ -0,0 +1,15 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.integration_tests import ChatModelIntegrationTests + +from langchain_groq import ChatGroq + + +class TestMistralStandard(ChatModelIntegrationTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatGroq diff --git a/libs/partners/groq/tests/unit_tests/test_standard.py b/libs/partners/groq/tests/unit_tests/test_standard.py new file mode 100644 index 00000000000000..38841230a9a985 --- /dev/null +++ b/libs/partners/groq/tests/unit_tests/test_standard.py @@ -0,0 +1,15 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +import pytest +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.unit_tests import ChatModelUnitTests + +from langchain_groq import ChatGroq + + +class TestGroqStandard(ChatModelUnitTests): + @pytest.fixture + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatGroq diff --git a/libs/standard-tests/langchain_standard_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_standard_tests/unit_tests/chat_models.py index bf041f06e1d7f9..c69463fa19b2c5 100644 --- a/libs/standard-tests/langchain_standard_tests/unit_tests/chat_models.py +++ b/libs/standard-tests/langchain_standard_tests/unit_tests/chat_models.py @@ -38,7 +38,10 @@ def chat_model_has_tool_calling( def chat_model_has_structured_output( self, chat_model_class: Type[BaseChatModel] ) -> bool: - return hasattr(chat_model_class, "with_structured_output") + return ( + chat_model_class.with_structured_output + is not BaseChatModel.with_structured_output + ) def test_chat_model_init( self, chat_model_class: Type[BaseChatModel], chat_model_params: dict @@ -49,7 +52,8 @@ def test_chat_model_init( def test_chat_model_init_api_key( self, chat_model_class: Type[BaseChatModel], chat_model_params: dict ) -> None: - model = chat_model_class(api_key="test", **chat_model_params) # type: ignore + params = {**chat_model_params, "api_key": "test"} + model = chat_model_class(**params) # type: ignore assert model is not None def test_chat_model_init_streaming(