From 40d345dd52af82e31e8fa34e5b0b1eebad006684 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 4 Mar 2025 05:28:41 +0000 Subject: [PATCH 1/4] chore(internal): remove unused http client options forwarding (#61) --- src/contextual/_base_client.py | 97 +--------------------------------- 1 file changed, 1 insertion(+), 96 deletions(-) diff --git a/src/contextual/_base_client.py b/src/contextual/_base_client.py index f4ea1713..ee4d0c95 100644 --- a/src/contextual/_base_client.py +++ b/src/contextual/_base_client.py @@ -9,7 +9,6 @@ import inspect import logging import platform -import warnings import email.utils from types import TracebackType from random import random @@ -36,7 +35,7 @@ import httpx import distro import pydantic -from httpx import URL, Limits +from httpx import URL from pydantic import PrivateAttr from . import _exceptions @@ -51,13 +50,10 @@ Timeout, NotGiven, ResponseT, - Transport, AnyMapping, PostParser, - ProxiesTypes, RequestFiles, HttpxSendArgs, - AsyncTransport, RequestOptions, HttpxRequestFiles, ModelBuilderProtocol, @@ -337,9 +333,6 @@ class BaseClient(Generic[_HttpxClientT, _DefaultStreamT]): _base_url: URL max_retries: int timeout: Union[float, Timeout, None] - _limits: httpx.Limits - _proxies: ProxiesTypes | None - _transport: Transport | AsyncTransport | None _strict_response_validation: bool _idempotency_header: str | None _default_stream_cls: type[_DefaultStreamT] | None = None @@ -352,9 +345,6 @@ def __init__( _strict_response_validation: bool, max_retries: int = DEFAULT_MAX_RETRIES, timeout: float | Timeout | None = DEFAULT_TIMEOUT, - limits: httpx.Limits, - transport: Transport | AsyncTransport | None, - proxies: ProxiesTypes | None, custom_headers: Mapping[str, str] | None = None, custom_query: Mapping[str, object] | None = None, ) -> None: @@ -362,9 +352,6 @@ def __init__( self._base_url = self._enforce_trailing_slash(URL(base_url)) self.max_retries = max_retries self.timeout = timeout - self._limits = limits - self._proxies = proxies - self._transport = transport self._custom_headers = custom_headers or {} self._custom_query = custom_query or {} self._strict_response_validation = _strict_response_validation @@ -800,46 +787,11 @@ def __init__( base_url: str | URL, max_retries: int = DEFAULT_MAX_RETRIES, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, - transport: Transport | None = None, - proxies: ProxiesTypes | None = None, - limits: Limits | None = None, http_client: httpx.Client | None = None, custom_headers: Mapping[str, str] | None = None, custom_query: Mapping[str, object] | None = None, _strict_response_validation: bool, ) -> None: - kwargs: dict[str, Any] = {} - if limits is not None: - warnings.warn( - "The `connection_pool_limits` argument is deprecated. The `http_client` argument should be passed instead", - category=DeprecationWarning, - stacklevel=3, - ) - if http_client is not None: - raise ValueError("The `http_client` argument is mutually exclusive with `connection_pool_limits`") - else: - limits = DEFAULT_CONNECTION_LIMITS - - if transport is not None: - kwargs["transport"] = transport - warnings.warn( - "The `transport` argument is deprecated. The `http_client` argument should be passed instead", - category=DeprecationWarning, - stacklevel=3, - ) - if http_client is not None: - raise ValueError("The `http_client` argument is mutually exclusive with `transport`") - - if proxies is not None: - kwargs["proxies"] = proxies - warnings.warn( - "The `proxies` argument is deprecated. The `http_client` argument should be passed instead", - category=DeprecationWarning, - stacklevel=3, - ) - if http_client is not None: - raise ValueError("The `http_client` argument is mutually exclusive with `proxies`") - if not is_given(timeout): # if the user passed in a custom http client with a non-default # timeout set then we use that timeout. @@ -860,12 +812,9 @@ def __init__( super().__init__( version=version, - limits=limits, # cast to a valid type because mypy doesn't understand our type narrowing timeout=cast(Timeout, timeout), - proxies=proxies, base_url=base_url, - transport=transport, max_retries=max_retries, custom_query=custom_query, custom_headers=custom_headers, @@ -875,9 +824,6 @@ def __init__( base_url=base_url, # cast to a valid type because mypy doesn't understand our type narrowing timeout=cast(Timeout, timeout), - limits=limits, - follow_redirects=True, - **kwargs, # type: ignore ) def is_closed(self) -> bool: @@ -1372,45 +1318,10 @@ def __init__( _strict_response_validation: bool, max_retries: int = DEFAULT_MAX_RETRIES, timeout: float | Timeout | None | NotGiven = NOT_GIVEN, - transport: AsyncTransport | None = None, - proxies: ProxiesTypes | None = None, - limits: Limits | None = None, http_client: httpx.AsyncClient | None = None, custom_headers: Mapping[str, str] | None = None, custom_query: Mapping[str, object] | None = None, ) -> None: - kwargs: dict[str, Any] = {} - if limits is not None: - warnings.warn( - "The `connection_pool_limits` argument is deprecated. The `http_client` argument should be passed instead", - category=DeprecationWarning, - stacklevel=3, - ) - if http_client is not None: - raise ValueError("The `http_client` argument is mutually exclusive with `connection_pool_limits`") - else: - limits = DEFAULT_CONNECTION_LIMITS - - if transport is not None: - kwargs["transport"] = transport - warnings.warn( - "The `transport` argument is deprecated. The `http_client` argument should be passed instead", - category=DeprecationWarning, - stacklevel=3, - ) - if http_client is not None: - raise ValueError("The `http_client` argument is mutually exclusive with `transport`") - - if proxies is not None: - kwargs["proxies"] = proxies - warnings.warn( - "The `proxies` argument is deprecated. The `http_client` argument should be passed instead", - category=DeprecationWarning, - stacklevel=3, - ) - if http_client is not None: - raise ValueError("The `http_client` argument is mutually exclusive with `proxies`") - if not is_given(timeout): # if the user passed in a custom http client with a non-default # timeout set then we use that timeout. @@ -1432,11 +1343,8 @@ def __init__( super().__init__( version=version, base_url=base_url, - limits=limits, # cast to a valid type because mypy doesn't understand our type narrowing timeout=cast(Timeout, timeout), - proxies=proxies, - transport=transport, max_retries=max_retries, custom_query=custom_query, custom_headers=custom_headers, @@ -1446,9 +1354,6 @@ def __init__( base_url=base_url, # cast to a valid type because mypy doesn't understand our type narrowing timeout=cast(Timeout, timeout), - limits=limits, - follow_redirects=True, - **kwargs, # type: ignore ) def is_closed(self) -> bool: From 59bb1ab3d790ee7e3d73b2b6a85e67a905d0ca22 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 07:21:17 +0000 Subject: [PATCH 2/4] feat(api): update via SDK Studio (#63) --- .stats.yml | 2 +- api.md | 8 ++ src/contextual/resources/agents/agents.py | 30 +++++++ src/contextual/resources/agents/query.py | 90 +++++++++++++++++++ .../resources/datastores/datastores.py | 10 +++ .../resources/datastores/documents.py | 64 +++++++++---- src/contextual/resources/generate.py | 28 +++--- src/contextual/resources/rerank.py | 60 ++++++++++--- src/contextual/types/__init__.py | 1 + src/contextual/types/agent_create_params.py | 9 ++ src/contextual/types/agent_metadata.py | 9 ++ src/contextual/types/agent_update_params.py | 9 ++ .../types/agents/query_create_params.py | 65 +++++++++++++- .../types/agents/query_metrics_params.py | 8 +- src/contextual/types/agents/query_response.py | 18 +++- .../types/agents/tune/tune_job_metadata.py | 4 +- .../types/composite_metadata_filter_param.py | 40 +++++++++ .../datastores/document_ingest_params.py | 25 ++++-- .../types/generate_create_params.py | 4 +- src/contextual/types/rerank_create_params.py | 25 +++++- tests/api_resources/agents/test_query.py | 14 +++ tests/api_resources/test_agents.py | 8 ++ 22 files changed, 470 insertions(+), 61 deletions(-) create mode 100644 src/contextual/types/composite_metadata_filter_param.py diff --git a/.stats.yml b/.stats.yml index 3bf9ffe6..50cd37d6 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ configured_endpoints: 46 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/contextual-ai%2Fsunrise-f43814080090927ee22816c5c7f517d8a7eb7f346329ada67915608e32124321.yml +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/contextual-ai%2Fsunrise-194878b194cd507d7c5418ff38cc0fc53441ef618f991990d334b4b75775cd8f.yml diff --git a/api.md b/api.md index b0dda095..71bda32c 100644 --- a/api.md +++ b/api.md @@ -1,3 +1,11 @@ +# ContextualAI + +Types: + +```python +from contextual.types import CompositeMetadataFilter +``` + # Datastores Types: diff --git a/src/contextual/resources/agents/agents.py b/src/contextual/resources/agents/agents.py index 72a291c1..c55e149c 100644 --- a/src/contextual/resources/agents/agents.py +++ b/src/contextual/resources/agents/agents.py @@ -104,6 +104,7 @@ def create( agent_configs: agent_create_params.AgentConfigs | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, description: str | NotGiven = NOT_GIVEN, + filter_prompt: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -126,6 +127,11 @@ def create( creates an empty `Datastore` and configures the `Agent` to use the newly created `Datastore`. + > Note that self-serve users are currently required to create agents through our + > UI. Otherwise, they will receive the following message: "This endpoint is + > disabled as you need to go through checkout. Please use the UI to make this + > request." + Args: name: Name of the agent @@ -135,6 +141,9 @@ def create( description: Description of the agent + filter_prompt: The prompt to an LLM which determines whether retrieved chunks are relevant to a + given query and filters out irrelevant chunks. + suggested_queries: These queries will show up as suggestions in the Contextual UI when users load the agent. We recommend including common queries that users will ask, as well as complex queries so users understand the types of complex queries the system can @@ -159,6 +168,7 @@ def create( "agent_configs": agent_configs, "datastore_ids": datastore_ids, "description": description, + "filter_prompt": filter_prompt, "suggested_queries": suggested_queries, "system_prompt": system_prompt, }, @@ -176,6 +186,7 @@ def update( *, agent_configs: agent_update_params.AgentConfigs | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, + filter_prompt: str | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, @@ -198,6 +209,9 @@ def update( datastore_ids: IDs of the datastore to associate with the agent. + filter_prompt: The prompt to an LLM which determines whether retrieved chunks are relevant to a + given query and filters out irrelevant chunks. + llm_model_id: The model ID to use for generation. Tuned models can only be used for the agents on which they were tuned. If no model is specified, the default model is used. Set to `default` to switch from a tuned model to the default model. @@ -226,6 +240,7 @@ def update( { "agent_configs": agent_configs, "datastore_ids": datastore_ids, + "filter_prompt": filter_prompt, "llm_model_id": llm_model_id, "suggested_queries": suggested_queries, "system_prompt": system_prompt, @@ -405,6 +420,7 @@ async def create( agent_configs: agent_create_params.AgentConfigs | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, description: str | NotGiven = NOT_GIVEN, + filter_prompt: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -427,6 +443,11 @@ async def create( creates an empty `Datastore` and configures the `Agent` to use the newly created `Datastore`. + > Note that self-serve users are currently required to create agents through our + > UI. Otherwise, they will receive the following message: "This endpoint is + > disabled as you need to go through checkout. Please use the UI to make this + > request." + Args: name: Name of the agent @@ -436,6 +457,9 @@ async def create( description: Description of the agent + filter_prompt: The prompt to an LLM which determines whether retrieved chunks are relevant to a + given query and filters out irrelevant chunks. + suggested_queries: These queries will show up as suggestions in the Contextual UI when users load the agent. We recommend including common queries that users will ask, as well as complex queries so users understand the types of complex queries the system can @@ -460,6 +484,7 @@ async def create( "agent_configs": agent_configs, "datastore_ids": datastore_ids, "description": description, + "filter_prompt": filter_prompt, "suggested_queries": suggested_queries, "system_prompt": system_prompt, }, @@ -477,6 +502,7 @@ async def update( *, agent_configs: agent_update_params.AgentConfigs | NotGiven = NOT_GIVEN, datastore_ids: List[str] | NotGiven = NOT_GIVEN, + filter_prompt: str | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, suggested_queries: List[str] | NotGiven = NOT_GIVEN, system_prompt: str | NotGiven = NOT_GIVEN, @@ -499,6 +525,9 @@ async def update( datastore_ids: IDs of the datastore to associate with the agent. + filter_prompt: The prompt to an LLM which determines whether retrieved chunks are relevant to a + given query and filters out irrelevant chunks. + llm_model_id: The model ID to use for generation. Tuned models can only be used for the agents on which they were tuned. If no model is specified, the default model is used. Set to `default` to switch from a tuned model to the default model. @@ -527,6 +556,7 @@ async def update( { "agent_configs": agent_configs, "datastore_ids": datastore_ids, + "filter_prompt": filter_prompt, "llm_model_id": llm_model_id, "suggested_queries": suggested_queries, "system_prompt": system_prompt, diff --git a/src/contextual/resources/agents/query.py b/src/contextual/resources/agents/query.py index 7f6d9de0..c02580a5 100644 --- a/src/contextual/resources/agents/query.py +++ b/src/contextual/resources/agents/query.py @@ -63,6 +63,7 @@ def create( include_retrieval_content_text: bool | NotGiven = NOT_GIVEN, retrievals_only: bool | NotGiven = NOT_GIVEN, conversation_id: str | NotGiven = NOT_GIVEN, + documents_filters: query_create_params.DocumentsFilters | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, stream: bool | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -97,6 +98,41 @@ def create( provided, all messages in the `messages` list prior to the latest user-sent query will be ignored. + documents_filters: Defines an Optional custom metadata filter, which can be a list of filters or + nested filters. The expected input is a nested JSON object that can represent a + single filter or a composite (logical) combination of filters. + + Unnested Example: + + ```json + { + "operator": "AND", + "filters": [{ "field": "status", "operator": "equals", "value": "active" }] + } + ``` + + Nested example: + + ```json + { + "operator": "AND", + "filters": [ + { "field": "status", "operator": "equals", "value": "active" }, + { + "operator": "OR", + "filters": [ + { + "field": "category", + "operator": "containsany", + "value": ["policy", "HR"] + }, + { "field": "tags", "operator": "exists" } + ] + } + ] + } + ``` + llm_model_id: Model ID of the specific fine-tuned or aligned LLM model to use. Defaults to base model if not specified. @@ -118,6 +154,7 @@ def create( { "messages": messages, "conversation_id": conversation_id, + "documents_filters": documents_filters, "llm_model_id": llm_model_id, "stream": stream, }, @@ -210,10 +247,12 @@ def metrics( self, agent_id: str, *, + conversation_ids: List[str] | NotGiven = NOT_GIVEN, created_after: Union[str, datetime] | NotGiven = NOT_GIVEN, created_before: Union[str, datetime] | NotGiven = NOT_GIVEN, limit: int | NotGiven = NOT_GIVEN, offset: int | NotGiven = NOT_GIVEN, + user_emails: List[str] | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -229,6 +268,8 @@ def metrics( Args: agent_id: Agent ID of the agent to get metrics for + conversation_ids: Filter messages by conversation ids. + created_after: Filters messages that are created after the specified timestamp. created_before: Filters messages that are created before specified timestamp. @@ -237,6 +278,8 @@ def metrics( offset: Offset for pagination. + user_emails: Filter messages by users. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -256,10 +299,12 @@ def metrics( timeout=timeout, query=maybe_transform( { + "conversation_ids": conversation_ids, "created_after": created_after, "created_before": created_before, "limit": limit, "offset": offset, + "user_emails": user_emails, }, query_metrics_params.QueryMetricsParams, ), @@ -346,6 +391,7 @@ async def create( include_retrieval_content_text: bool | NotGiven = NOT_GIVEN, retrievals_only: bool | NotGiven = NOT_GIVEN, conversation_id: str | NotGiven = NOT_GIVEN, + documents_filters: query_create_params.DocumentsFilters | NotGiven = NOT_GIVEN, llm_model_id: str | NotGiven = NOT_GIVEN, stream: bool | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. @@ -380,6 +426,41 @@ async def create( provided, all messages in the `messages` list prior to the latest user-sent query will be ignored. + documents_filters: Defines an Optional custom metadata filter, which can be a list of filters or + nested filters. The expected input is a nested JSON object that can represent a + single filter or a composite (logical) combination of filters. + + Unnested Example: + + ```json + { + "operator": "AND", + "filters": [{ "field": "status", "operator": "equals", "value": "active" }] + } + ``` + + Nested example: + + ```json + { + "operator": "AND", + "filters": [ + { "field": "status", "operator": "equals", "value": "active" }, + { + "operator": "OR", + "filters": [ + { + "field": "category", + "operator": "containsany", + "value": ["policy", "HR"] + }, + { "field": "tags", "operator": "exists" } + ] + } + ] + } + ``` + llm_model_id: Model ID of the specific fine-tuned or aligned LLM model to use. Defaults to base model if not specified. @@ -401,6 +482,7 @@ async def create( { "messages": messages, "conversation_id": conversation_id, + "documents_filters": documents_filters, "llm_model_id": llm_model_id, "stream": stream, }, @@ -493,10 +575,12 @@ async def metrics( self, agent_id: str, *, + conversation_ids: List[str] | NotGiven = NOT_GIVEN, created_after: Union[str, datetime] | NotGiven = NOT_GIVEN, created_before: Union[str, datetime] | NotGiven = NOT_GIVEN, limit: int | NotGiven = NOT_GIVEN, offset: int | NotGiven = NOT_GIVEN, + user_emails: List[str] | NotGiven = NOT_GIVEN, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -512,6 +596,8 @@ async def metrics( Args: agent_id: Agent ID of the agent to get metrics for + conversation_ids: Filter messages by conversation ids. + created_after: Filters messages that are created after the specified timestamp. created_before: Filters messages that are created before specified timestamp. @@ -520,6 +606,8 @@ async def metrics( offset: Offset for pagination. + user_emails: Filter messages by users. + extra_headers: Send extra headers extra_query: Add additional query parameters to the request @@ -539,10 +627,12 @@ async def metrics( timeout=timeout, query=await async_maybe_transform( { + "conversation_ids": conversation_ids, "created_after": created_after, "created_before": created_before, "limit": limit, "offset": offset, + "user_emails": user_emails, }, query_metrics_params.QueryMetricsParams, ), diff --git a/src/contextual/resources/datastores/datastores.py b/src/contextual/resources/datastores/datastores.py index a0791411..549ba733 100644 --- a/src/contextual/resources/datastores/datastores.py +++ b/src/contextual/resources/datastores/datastores.py @@ -83,6 +83,11 @@ def create( from multiple sources of information. This linkage of `Datastore` to `Agent` is done through the `Create Agent` or `Edit Agent` APIs. + > Note that self-serve users are currently required to create datastores through + > our UI. Otherwise, they will receive the following message: "This endpoint is + > disabled as you need to go through checkout. Please use the UI to make this + > request." + Args: name: Name of the datastore @@ -285,6 +290,11 @@ async def create( from multiple sources of information. This linkage of `Datastore` to `Agent` is done through the `Create Agent` or `Edit Agent` APIs. + > Note that self-serve users are currently required to create datastores through + > our UI. Otherwise, they will receive the following message: "This endpoint is + > disabled as you need to go through checkout. Please use the UI to make this + > request." + Args: name: Name of the datastore diff --git a/src/contextual/resources/datastores/documents.py b/src/contextual/resources/datastores/documents.py index 60dd9651..aab15af9 100644 --- a/src/contextual/resources/datastores/documents.py +++ b/src/contextual/resources/datastores/documents.py @@ -190,18 +190,34 @@ def ingest( This `id` can also be used to delete the document through the `DELETE /datastores/{datastore_id}/documents/{document_id}` API. - `file` must be a PDF or HTML file. + `file` must be a PDF, HTML, DOC(X) or PPT(X) file. The filename must end with + one of the following extensions: `.pdf`, `.html`, `.htm`, `.mhtml`, `.doc`, + `.docx`, `.ppt`, `.pptx`. Args: datastore_id: Datastore ID of the datastore in which to ingest the document - file: File to ingest + file: File to ingest. - metadata: Metadata in `JSON` format. Metadata should be passed in a nested dictionary - structure of `str` metadata type to `Dict` mapping `str` metadata keys to `str`, - `bool`, `float` or `int` values. Currently, `custom_metadata` is the only - supported metadata type.Example `metadata` dictionary: {"metadata": - {"custom_metadata": {"customKey1": "value3", "\\__filterKey": "filterValue3"}} + metadata: Metadata in `JSON` format. Metadata should be passed as a nested dictionary + structure where: + + - The **metadata type** `custom_metadata` is mapped to a dictionary. - The + **dictionary keys** represent metadata attributes. - The **values** can be of + type `str`, `bool`, `float`, or `int`. + + **Example Metadata JSON:** + + ```json + { + "metadata": { + "custom_metadata": { + "customKey1": "value3", + "_filterKey": "filterValue3" + } + } + } + ``` extra_headers: Send extra headers @@ -290,7 +306,7 @@ def set_metadata( ) -> DocumentMetadata: """ Post details of a given document that will enrich the chunk and be added to the - context or just for filtering. If JUst for filtering, start with "\\__" in the + context or just for filtering. If Just for filtering, start with "\\__" in the key. Args: @@ -480,18 +496,34 @@ async def ingest( This `id` can also be used to delete the document through the `DELETE /datastores/{datastore_id}/documents/{document_id}` API. - `file` must be a PDF or HTML file. + `file` must be a PDF, HTML, DOC(X) or PPT(X) file. The filename must end with + one of the following extensions: `.pdf`, `.html`, `.htm`, `.mhtml`, `.doc`, + `.docx`, `.ppt`, `.pptx`. Args: datastore_id: Datastore ID of the datastore in which to ingest the document - file: File to ingest + file: File to ingest. + + metadata: Metadata in `JSON` format. Metadata should be passed as a nested dictionary + structure where: + + - The **metadata type** `custom_metadata` is mapped to a dictionary. - The + **dictionary keys** represent metadata attributes. - The **values** can be of + type `str`, `bool`, `float`, or `int`. + + **Example Metadata JSON:** - metadata: Metadata in `JSON` format. Metadata should be passed in a nested dictionary - structure of `str` metadata type to `Dict` mapping `str` metadata keys to `str`, - `bool`, `float` or `int` values. Currently, `custom_metadata` is the only - supported metadata type.Example `metadata` dictionary: {"metadata": - {"custom_metadata": {"customKey1": "value3", "\\__filterKey": "filterValue3"}} + ```json + { + "metadata": { + "custom_metadata": { + "customKey1": "value3", + "_filterKey": "filterValue3" + } + } + } + ``` extra_headers: Send extra headers @@ -580,7 +612,7 @@ async def set_metadata( ) -> DocumentMetadata: """ Post details of a given document that will enrich the chunk and be added to the - context or just for filtering. If JUst for filtering, start with "\\__" in the + context or just for filtering. If Just for filtering, start with "\\__" in the key. Args: diff --git a/src/contextual/resources/generate.py b/src/contextual/resources/generate.py index 0e879a9c..88a6009d 100644 --- a/src/contextual/resources/generate.py +++ b/src/contextual/resources/generate.py @@ -67,10 +67,14 @@ def create( """ Generate a response using Contextual's Grounded Language Model (GLM), an LLM engineered specifically to prioritize faithfulness to in-context retrievals over - parametric knowledge to reduce hallucinations in Retrieval-Augmented Generation. + parametric knowledge to reduce hallucinations in Retrieval-Augmented Generation + and agentic use cases. - The total request cannot exceed 32,000 tokens. Email glm-feedback@contextual.ai - with any feedback or questions. + The total request cannot exceed 32,000 tokens. See more details and code + examples in our + [our blog post](https://contextual.ai/blog/introducing-grounded-language-model/). + Email [glm-feedback@contextual.ai](mailto:glm-feedback@contextual.ai) with any + feedback or questions. Args: knowledge: The knowledge sources the model can use when generating a response. @@ -92,11 +96,11 @@ def create( not guarantee that the model follows these instructions exactly. temperature: The sampling temperature, which affects the randomness in the response. Note - that higher temperature values can reduce groundedness + that higher temperature values can reduce groundedness. top_p: A parameter for nucleus sampling, an alternative to temperature which also affects the randomness of the response. Note that higher top_p values can reduce - groundedness + groundedness. extra_headers: Send extra headers @@ -169,10 +173,14 @@ async def create( """ Generate a response using Contextual's Grounded Language Model (GLM), an LLM engineered specifically to prioritize faithfulness to in-context retrievals over - parametric knowledge to reduce hallucinations in Retrieval-Augmented Generation. + parametric knowledge to reduce hallucinations in Retrieval-Augmented Generation + and agentic use cases. - The total request cannot exceed 32,000 tokens. Email glm-feedback@contextual.ai - with any feedback or questions. + The total request cannot exceed 32,000 tokens. See more details and code + examples in our + [our blog post](https://contextual.ai/blog/introducing-grounded-language-model/). + Email [glm-feedback@contextual.ai](mailto:glm-feedback@contextual.ai) with any + feedback or questions. Args: knowledge: The knowledge sources the model can use when generating a response. @@ -194,11 +202,11 @@ async def create( not guarantee that the model follows these instructions exactly. temperature: The sampling temperature, which affects the randomness in the response. Note - that higher temperature values can reduce groundedness + that higher temperature values can reduce groundedness. top_p: A parameter for nucleus sampling, an alternative to temperature which also affects the randomness of the response. Note that higher top_p values can reduce - groundedness + groundedness. extra_headers: Send extra headers diff --git a/src/contextual/resources/rerank.py b/src/contextual/resources/rerank.py index 65149c9a..c46e5e52 100644 --- a/src/contextual/resources/rerank.py +++ b/src/contextual/resources/rerank.py @@ -63,22 +63,38 @@ def create( timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, ) -> RerankCreateResponse: """ - Rank a list of documents according to their relevance to a query. + Rank a list of documents according to their relevance to a query and your custom + instructions about how to prioritize retrievals. We evaluated the model on + instructions for recency, document type, source, and metadata, and it can + generalize to other instructions as well. - The total request cannot exceed 400,000 tokens. The combined length of any - document, instruction and the query must not exceed 4,000 tokens. + The total request cannot exceed 400,000 tokens. The combined length of the + query, instruction and any document with its metadata must not exceed 8,000 + tokens. Email + [rerank-feedback@contextual.ai](mailto:rerank-feedback@contextual.ai) with any + feedback or questions. Args: - documents: The texts to be reranked according to their relevance to the query + documents: The texts to be reranked according to their relevance to the query and the + optional instruction - model: The version of the reranker to use. Currently, we just have "v1". + model: The version of the reranker to use. Currently, we just have + "ctxl-rerank-en-v1-instruct". query: The string against which documents will be ranked for relevance - instruction: The instruction to be used for the reranker + instruction: Instructions that the reranker references when ranking retrievals. We evaluated + the model on instructions for recency, document type, source, and metadata, and + it can generalize to other instructions as well. Note that we do not guarantee + that the reranker will follow these instructions exactly. Examples: "Prioritize + internal sales documents over market analysis reports. More recent documents + should be weighted higher. Enterprise portal content supersedes distributor + communications." and "Emphasize forecasts from top-tier investment banks. Recent + analysis should take precedence. Disregard aggregator sites and favor detailed + research notes over news summaries." metadata: Metadata for documents being passed to the reranker. Must be the same length as - the documents list. + the documents list. If a document does not have metadata, add an empty string. top_n: The number of top-ranked results to return @@ -147,22 +163,38 @@ async def create( timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, ) -> RerankCreateResponse: """ - Rank a list of documents according to their relevance to a query. + Rank a list of documents according to their relevance to a query and your custom + instructions about how to prioritize retrievals. We evaluated the model on + instructions for recency, document type, source, and metadata, and it can + generalize to other instructions as well. - The total request cannot exceed 400,000 tokens. The combined length of any - document, instruction and the query must not exceed 4,000 tokens. + The total request cannot exceed 400,000 tokens. The combined length of the + query, instruction and any document with its metadata must not exceed 8,000 + tokens. Email + [rerank-feedback@contextual.ai](mailto:rerank-feedback@contextual.ai) with any + feedback or questions. Args: - documents: The texts to be reranked according to their relevance to the query + documents: The texts to be reranked according to their relevance to the query and the + optional instruction - model: The version of the reranker to use. Currently, we just have "v1". + model: The version of the reranker to use. Currently, we just have + "ctxl-rerank-en-v1-instruct". query: The string against which documents will be ranked for relevance - instruction: The instruction to be used for the reranker + instruction: Instructions that the reranker references when ranking retrievals. We evaluated + the model on instructions for recency, document type, source, and metadata, and + it can generalize to other instructions as well. Note that we do not guarantee + that the reranker will follow these instructions exactly. Examples: "Prioritize + internal sales documents over market analysis reports. More recent documents + should be weighted higher. Enterprise portal content supersedes distributor + communications." and "Emphasize forecasts from top-tier investment banks. Recent + analysis should take precedence. Disregard aggregator sites and favor detailed + research notes over news summaries." metadata: Metadata for documents being passed to the reranker. Must be the same length as - the documents list. + the documents list. If a document does not have metadata, add an empty string. top_n: The number of top-ranked results to return diff --git a/src/contextual/types/__init__.py b/src/contextual/types/__init__.py index d5285a29..86ab066d 100644 --- a/src/contextual/types/__init__.py +++ b/src/contextual/types/__init__.py @@ -28,3 +28,4 @@ from .generate_create_response import GenerateCreateResponse as GenerateCreateResponse from .list_datastores_response import ListDatastoresResponse as ListDatastoresResponse from .create_datastore_response import CreateDatastoreResponse as CreateDatastoreResponse +from .composite_metadata_filter_param import CompositeMetadataFilterParam as CompositeMetadataFilterParam diff --git a/src/contextual/types/agent_create_params.py b/src/contextual/types/agent_create_params.py index ec8e9207..2da638a3 100644 --- a/src/contextual/types/agent_create_params.py +++ b/src/contextual/types/agent_create_params.py @@ -28,6 +28,12 @@ class AgentCreateParams(TypedDict, total=False): description: str """Description of the agent""" + filter_prompt: str + """ + The prompt to an LLM which determines whether retrieved chunks are relevant to a + given query and filters out irrelevant chunks. + """ + suggested_queries: List[str] """ These queries will show up as suggestions in the Contextual UI when users load @@ -50,6 +56,9 @@ class AgentConfigsFilterAndRerankConfig(TypedDict, total=False): class AgentConfigsGenerateResponseConfig(TypedDict, total=False): + calculate_groundedness: bool + """This parameter controls generation of groundedness scores.""" + frequency_penalty: float """ This parameter adjusts how the model treats repeated tokens during text diff --git a/src/contextual/types/agent_metadata.py b/src/contextual/types/agent_metadata.py index 58901b56..afd9230b 100644 --- a/src/contextual/types/agent_metadata.py +++ b/src/contextual/types/agent_metadata.py @@ -20,6 +20,9 @@ class AgentConfigsFilterAndRerankConfig(BaseModel): class AgentConfigsGenerateResponseConfig(BaseModel): + calculate_groundedness: Optional[bool] = None + """This parameter controls generation of groundedness scores.""" + frequency_penalty: Optional[float] = None """ This parameter adjusts how the model treats repeated tokens during text @@ -97,6 +100,12 @@ class AgentMetadata(BaseModel): description: Optional[str] = None """Description of the agent""" + filter_prompt: Optional[str] = None + """ + The prompt to an LLM which determines whether retrieved chunks are relevant to a + given query and filters out irrelevant chunks. This prompt is applied per chunk. + """ + llm_model_id: Optional[str] = None """The model ID to use for generation. diff --git a/src/contextual/types/agent_update_params.py b/src/contextual/types/agent_update_params.py index 34f9ad63..791caf97 100644 --- a/src/contextual/types/agent_update_params.py +++ b/src/contextual/types/agent_update_params.py @@ -22,6 +22,12 @@ class AgentUpdateParams(TypedDict, total=False): datastore_ids: List[str] """IDs of the datastore to associate with the agent.""" + filter_prompt: str + """ + The prompt to an LLM which determines whether retrieved chunks are relevant to a + given query and filters out irrelevant chunks. + """ + llm_model_id: str """The model ID to use for generation. @@ -52,6 +58,9 @@ class AgentConfigsFilterAndRerankConfig(TypedDict, total=False): class AgentConfigsGenerateResponseConfig(TypedDict, total=False): + calculate_groundedness: bool + """This parameter controls generation of groundedness scores.""" + frequency_penalty: float """ This parameter adjusts how the model treats repeated tokens during text diff --git a/src/contextual/types/agents/query_create_params.py b/src/contextual/types/agents/query_create_params.py index 73b177a3..285b6677 100644 --- a/src/contextual/types/agents/query_create_params.py +++ b/src/contextual/types/agents/query_create_params.py @@ -2,10 +2,10 @@ from __future__ import annotations -from typing import Iterable -from typing_extensions import Literal, Required, TypedDict +from typing import List, Union, Iterable +from typing_extensions import Literal, Required, TypeAlias, TypedDict -__all__ = ["QueryCreateParams", "Message"] +__all__ = ["QueryCreateParams", "Message", "DocumentsFilters", "DocumentsFiltersBaseMetadataFilter"] class QueryCreateParams(TypedDict, total=False): @@ -39,6 +39,44 @@ class QueryCreateParams(TypedDict, total=False): query will be ignored. """ + documents_filters: DocumentsFilters + """ + Defines an Optional custom metadata filter, which can be a list of filters or + nested filters. The expected input is a nested JSON object that can represent a + single filter or a composite (logical) combination of filters. + + Unnested Example: + + ```json + { + "operator": "AND", + "filters": [{ "field": "status", "operator": "equals", "value": "active" }] + } + ``` + + Nested example: + + ```json + { + "operator": "AND", + "filters": [ + { "field": "status", "operator": "equals", "value": "active" }, + { + "operator": "OR", + "filters": [ + { + "field": "category", + "operator": "containsany", + "value": ["policy", "HR"] + }, + { "field": "tags", "operator": "exists" } + ] + } + ] + } + ``` + """ + llm_model_id: str """Model ID of the specific fine-tuned or aligned LLM model to use. @@ -55,3 +93,24 @@ class Message(TypedDict, total=False): role: Required[Literal["user", "system", "assistant", "knowledge"]] """Role of the sender""" + + +class DocumentsFiltersBaseMetadataFilter(TypedDict, total=False): + field: Required[str] + """Field name to search for in the metadata""" + + operator: Required[ + Literal["equals", "containsany", "exists", "startswith", "gt", "gte", "lt", "lte", "notequals", "between"] + ] + """Operator to be used for the filter.""" + + value: Union[str, float, bool, List[Union[str, float, bool]], None] + """The value to be searched for in the field. + + In case of exists operator, it is not needed. + """ + + +DocumentsFilters: TypeAlias = Union[DocumentsFiltersBaseMetadataFilter, "CompositeMetadataFilterParam"] + +from ..composite_metadata_filter_param import CompositeMetadataFilterParam diff --git a/src/contextual/types/agents/query_metrics_params.py b/src/contextual/types/agents/query_metrics_params.py index b3483cb6..a24c2c90 100644 --- a/src/contextual/types/agents/query_metrics_params.py +++ b/src/contextual/types/agents/query_metrics_params.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Union +from typing import List, Union from datetime import datetime from typing_extensions import Annotated, TypedDict @@ -12,6 +12,9 @@ class QueryMetricsParams(TypedDict, total=False): + conversation_ids: List[str] + """Filter messages by conversation ids.""" + created_after: Annotated[Union[str, datetime], PropertyInfo(format="iso8601")] """Filters messages that are created after the specified timestamp.""" @@ -23,3 +26,6 @@ class QueryMetricsParams(TypedDict, total=False): offset: int """Offset for pagination.""" + + user_emails: List[str] + """Filter messages by users.""" diff --git a/src/contextual/types/agents/query_response.py b/src/contextual/types/agents/query_response.py index 56ae9e02..45aa5cde 100644 --- a/src/contextual/types/agents/query_response.py +++ b/src/contextual/types/agents/query_response.py @@ -5,7 +5,7 @@ from ..._models import BaseModel -__all__ = ["QueryResponse", "RetrievalContent", "Attribution", "Message"] +__all__ = ["QueryResponse", "RetrievalContent", "Attribution", "GroundednessScore", "Message"] class RetrievalContent(BaseModel): @@ -18,7 +18,7 @@ class RetrievalContent(BaseModel): doc_name: str """Name of the document""" - format: Literal["pdf", "html", "htm"] + format: Literal["pdf", "html", "htm", "mhtml", "doc", "docx", "ppt", "pptx"] """Format of the content, such as `pdf` or `html`""" type: str @@ -54,6 +54,17 @@ class Attribution(BaseModel): """Start index of the attributed text in the generated message""" +class GroundednessScore(BaseModel): + end_idx: int + """End index of the span in the generated message""" + + score: int + """Groundedness score for the span""" + + start_idx: int + """Start index of the span in the generated message""" + + class Message(BaseModel): content: str """Content of the message""" @@ -76,6 +87,9 @@ class QueryResponse(BaseModel): attributions: Optional[List[Attribution]] = None """Attributions for the response""" + groundedness_scores: Optional[List[GroundednessScore]] = None + """Groundedness scores for the response""" + message: Optional[Message] = None """Response to the query request""" diff --git a/src/contextual/types/agents/tune/tune_job_metadata.py b/src/contextual/types/agents/tune/tune_job_metadata.py index e71d9307..ddf93940 100644 --- a/src/contextual/types/agents/tune/tune_job_metadata.py +++ b/src/contextual/types/agents/tune/tune_job_metadata.py @@ -1,6 +1,6 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import Optional +from typing import List, Optional from ...._compat import PYDANTIC_V2, ConfigDict from ...._models import BaseModel @@ -15,7 +15,7 @@ class TuneJobMetadata(BaseModel): job_status: str """Status of the tune job""" - evaluation_metadata: Optional[object] = None + evaluation_metadata: Optional[List[object]] = None """Metadata about the model evaluation, including status and results if completed.""" model_id: Optional[str] = None diff --git a/src/contextual/types/composite_metadata_filter_param.py b/src/contextual/types/composite_metadata_filter_param.py new file mode 100644 index 00000000..fc786ef4 --- /dev/null +++ b/src/contextual/types/composite_metadata_filter_param.py @@ -0,0 +1,40 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Union, Iterable, Optional +from typing_extensions import Literal, Required, TypeAlias, TypedDict, TypeAliasType + +from .._compat import PYDANTIC_V2 + +__all__ = ["CompositeMetadataFilterParam", "Filter", "FilterBaseMetadataFilter"] + + +class FilterBaseMetadataFilter(TypedDict, total=False): + field: Required[str] + """Field name to search for in the metadata""" + + operator: Required[ + Literal["equals", "containsany", "exists", "startswith", "gt", "gte", "lt", "lte", "notequals", "between"] + ] + """Operator to be used for the filter.""" + + value: Union[str, float, bool, List[Union[str, float, bool]], None] + """The value to be searched for in the field. + + In case of exists operator, it is not needed. + """ + + +if TYPE_CHECKING or PYDANTIC_V2: + Filter = TypeAliasType("Filter", Union[FilterBaseMetadataFilter, "CompositeMetadataFilterParam"]) +else: + Filter: TypeAlias = Union[FilterBaseMetadataFilter, "CompositeMetadataFilterParam"] + + +class CompositeMetadataFilterParam(TypedDict, total=False): + filters: Required[Iterable[Filter]] + """Filters added to the query for filtering docs""" + + operator: Optional[Literal["AND", "OR", "AND_NOT"]] + """Composite operator to be used to combine filters""" diff --git a/src/contextual/types/datastores/document_ingest_params.py b/src/contextual/types/datastores/document_ingest_params.py index 295c0dc4..8e98cc60 100644 --- a/src/contextual/types/datastores/document_ingest_params.py +++ b/src/contextual/types/datastores/document_ingest_params.py @@ -11,14 +11,27 @@ class DocumentIngestParams(TypedDict, total=False): file: Required[FileTypes] - """File to ingest""" + """File to ingest.""" metadata: str """Metadata in `JSON` format. - Metadata should be passed in a nested dictionary structure of `str` metadata - type to `Dict` mapping `str` metadata keys to `str`, `bool`, `float` or `int` - values. Currently, `custom_metadata` is the only supported metadata type.Example - `metadata` dictionary: {"metadata": {"custom_metadata": {"customKey1": "value3", - "\\__filterKey": "filterValue3"}} + Metadata should be passed as a nested dictionary structure where: + + - The **metadata type** `custom_metadata` is mapped to a dictionary. - The + **dictionary keys** represent metadata attributes. - The **values** can be of + type `str`, `bool`, `float`, or `int`. + + **Example Metadata JSON:** + + ```json + { + "metadata": { + "custom_metadata": { + "customKey1": "value3", + "_filterKey": "filterValue3" + } + } + } + ``` """ diff --git a/src/contextual/types/generate_create_params.py b/src/contextual/types/generate_create_params.py index cea58b2f..c75e86ca 100644 --- a/src/contextual/types/generate_create_params.py +++ b/src/contextual/types/generate_create_params.py @@ -42,14 +42,14 @@ class GenerateCreateParams(TypedDict, total=False): temperature: float """The sampling temperature, which affects the randomness in the response. - Note that higher temperature values can reduce groundedness + Note that higher temperature values can reduce groundedness. """ top_p: float """ A parameter for nucleus sampling, an alternative to temperature which also affects the randomness of the response. Note that higher top_p values can reduce - groundedness + groundedness. """ diff --git a/src/contextual/types/rerank_create_params.py b/src/contextual/types/rerank_create_params.py index 5b4a9160..20dbe10c 100644 --- a/src/contextual/types/rerank_create_params.py +++ b/src/contextual/types/rerank_create_params.py @@ -10,21 +10,38 @@ class RerankCreateParams(TypedDict, total=False): documents: Required[List[str]] - """The texts to be reranked according to their relevance to the query""" + """ + The texts to be reranked according to their relevance to the query and the + optional instruction + """ model: Required[str] - """The version of the reranker to use. Currently, we just have "v1".""" + """The version of the reranker to use. + + Currently, we just have "ctxl-rerank-en-v1-instruct". + """ query: Required[str] """The string against which documents will be ranked for relevance""" instruction: str - """The instruction to be used for the reranker""" + """Instructions that the reranker references when ranking retrievals. + + We evaluated the model on instructions for recency, document type, source, and + metadata, and it can generalize to other instructions as well. Note that we do + not guarantee that the reranker will follow these instructions exactly. + Examples: "Prioritize internal sales documents over market analysis reports. + More recent documents should be weighted higher. Enterprise portal content + supersedes distributor communications." and "Emphasize forecasts from top-tier + investment banks. Recent analysis should take precedence. Disregard aggregator + sites and favor detailed research notes over news summaries." + """ metadata: List[str] """Metadata for documents being passed to the reranker. - Must be the same length as the documents list. + Must be the same length as the documents list. If a document does not have + metadata, add an empty string. """ top_n: int diff --git a/tests/api_resources/agents/test_query.py b/tests/api_resources/agents/test_query.py index 2e7da653..cf5ea0ca 100644 --- a/tests/api_resources/agents/test_query.py +++ b/tests/api_resources/agents/test_query.py @@ -48,6 +48,11 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: include_retrieval_content_text=True, retrievals_only=True, conversation_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + documents_filters={ + "field": "field", + "operator": "equals", + "value": "string", + }, llm_model_id="llm_model_id", stream=True, ) @@ -170,10 +175,12 @@ def test_method_metrics(self, client: ContextualAI) -> None: def test_method_metrics_with_all_params(self, client: ContextualAI) -> None: query = client.agents.query.metrics( agent_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + conversation_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], created_after=parse_datetime("2019-12-27T18:11:19.117Z"), created_before=parse_datetime("2019-12-27T18:11:19.117Z"), limit=1000, offset=0, + user_emails=["string"], ) assert_matches_type(QueryMetricsResponse, query, path=["response"]) @@ -291,6 +298,11 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual include_retrieval_content_text=True, retrievals_only=True, conversation_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + documents_filters={ + "field": "field", + "operator": "equals", + "value": "string", + }, llm_model_id="llm_model_id", stream=True, ) @@ -413,10 +425,12 @@ async def test_method_metrics(self, async_client: AsyncContextualAI) -> None: async def test_method_metrics_with_all_params(self, async_client: AsyncContextualAI) -> None: query = await async_client.agents.query.metrics( agent_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e", + conversation_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], created_after=parse_datetime("2019-12-27T18:11:19.117Z"), created_before=parse_datetime("2019-12-27T18:11:19.117Z"), limit=1000, offset=0, + user_emails=["string"], ) assert_matches_type(QueryMetricsResponse, query, path=["response"]) diff --git a/tests/api_resources/test_agents.py b/tests/api_resources/test_agents.py index 7116967f..efcdd219 100644 --- a/tests/api_resources/test_agents.py +++ b/tests/api_resources/test_agents.py @@ -36,6 +36,7 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: agent_configs={ "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, "generate_response_config": { + "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, "seed": 0, @@ -55,6 +56,7 @@ def test_method_create_with_all_params(self, client: ContextualAI) -> None: }, datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], description="xxx", + filter_prompt="filter_prompt", suggested_queries=["string"], system_prompt="system_prompt", ) @@ -98,6 +100,7 @@ def test_method_update_with_all_params(self, client: ContextualAI) -> None: agent_configs={ "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, "generate_response_config": { + "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, "seed": 0, @@ -116,6 +119,7 @@ def test_method_update_with_all_params(self, client: ContextualAI) -> None: }, }, datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], + filter_prompt="filter_prompt", llm_model_id="llm_model_id", suggested_queries=["string"], system_prompt="system_prompt", @@ -280,6 +284,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual agent_configs={ "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, "generate_response_config": { + "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, "seed": 0, @@ -299,6 +304,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncContextual }, datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], description="xxx", + filter_prompt="filter_prompt", suggested_queries=["string"], system_prompt="system_prompt", ) @@ -342,6 +348,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncContextual agent_configs={ "filter_and_rerank_config": {"top_k_reranked_chunks": 0}, "generate_response_config": { + "calculate_groundedness": True, "frequency_penalty": 0, "max_new_tokens": 0, "seed": 0, @@ -360,6 +367,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncContextual }, }, datastore_ids=["182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e"], + filter_prompt="filter_prompt", llm_model_id="llm_model_id", suggested_queries=["string"], system_prompt="system_prompt", From 39b862eca8d7443c2c86063123d8dfdc484a3c53 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Tue, 11 Mar 2025 07:25:49 +0000 Subject: [PATCH 3/4] feat: Add to_dataframe method to BinaryAPIReponse (#56) Signed-off-by: Sean Smith --- pyproject.toml | 4 +++ requirements-dev.lock | 4 +++ requirements.lock | 4 +++ src/contextual/_response.py | 58 +++++++++++++++++++++++++++++++++++++ tests/test_response.py | 18 ++++++++++++ 5 files changed, 88 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cd4ac761..c0aa3fa2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,8 @@ dependencies = [ "anyio>=3.5.0, <5", "distro>=1.7.0, <2", "sniffio", + "pandas==2.2.3", + "numpy==2.0.2", ] requires-python = ">= 3.8" classifiers = [ @@ -55,6 +57,8 @@ dev-dependencies = [ "importlib-metadata>=6.7.0", "rich>=13.7.1", "nest_asyncio==1.6.0", + "pandas==2.2.3", + "numpy==2.0.2", ] [tool.rye.scripts] diff --git a/requirements-dev.lock b/requirements-dev.lock index 83d02e00..19dcb392 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -102,3 +102,7 @@ virtualenv==20.24.5 # via nox zipp==3.17.0 # via importlib-metadata +pandas==2.2.3 + # via contextual-client +numpy==2.0.2 + # via contextual-client \ No newline at end of file diff --git a/requirements.lock b/requirements.lock index bc4698e1..3b833e41 100644 --- a/requirements.lock +++ b/requirements.lock @@ -43,3 +43,7 @@ typing-extensions==4.12.2 # via contextual-client # via pydantic # via pydantic-core +pandas==2.2.3 + # via contextual-client +numpy==2.0.2 + # via contextual-client \ No newline at end of file diff --git a/src/contextual/_response.py b/src/contextual/_response.py index 51fc249d..6d4a7bb1 100644 --- a/src/contextual/_response.py +++ b/src/contextual/_response.py @@ -1,6 +1,8 @@ from __future__ import annotations import os +import ast +import json import inspect import logging import datetime @@ -23,6 +25,7 @@ import anyio import httpx import pydantic +from pandas import DataFrame # type: ignore[import] from ._types import NoneType from ._utils import is_given, extract_type_arg, is_annotated_type, is_type_alias_type, extract_type_var_from_base @@ -479,6 +482,61 @@ class BinaryAPIResponse(APIResponse[bytes]): the API request, e.g. `.with_streaming_response.get_binary_response()` """ + def to_dataframe(self) -> DataFrame: + """Convert the response data to a pandas DataFrame. + + Note: This method requires the `pandas` library to be installed. + + Returns: + DataFrame: Processed evaluation data + """ + # Read the binary content + content = self.read() + + # Now decode the content + lines = content.decode("utf-8").strip().split("\n") + + # Parse each line and flatten the results + data = [] + for line in lines: + try: + entry = json.loads(line) + # Parse the results field directly from JSON + if 'results' in entry: + if isinstance(entry['results'], str): + # Try to handle string representations that are valid JSON + try: + results = json.loads(entry['results']) + except Exception as e: + # If not valid JSON, fall back to safer processing + results = ast.literal_eval(entry['results']) + else: + # Already a dictionary + results = entry['results'] + + # Remove the original results field + del entry['results'] + # Flatten the nested dictionary structure + if isinstance(results, dict): + for key, value in results.items(): # type: ignore + if isinstance(value, dict): + for subkey, subvalue in value.items(): # type: ignore + if isinstance(subvalue, dict): + # Handle one more level of nesting + for subsubkey, subsubvalue in subvalue.items(): # type: ignore + entry[f'{key}_{subkey}_{subsubkey}'] = subsubvalue + else: + entry[f'{key}_{subkey}'] = subvalue + else: + entry[key] = value + + data.append(entry) # type: ignore + except Exception as e: + log.error(f"Error processing line: {e}") + log.error(f"Problematic line: {line[:200]}...") # Print first 200 chars of the line + continue + return DataFrame(data) + def write_to_file( self, file: str | os.PathLike[str], diff --git a/tests/test_response.py b/tests/test_response.py index cedd75ba..d4cb409f 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -73,6 +73,24 @@ def test_response_parse_mismatched_basemodel(client: ContextualAI) -> None: response.parse(to=PydanticModel) +def test_response_binary_response_to_dataframe(client: ContextualAI) -> None: + response = BinaryAPIResponse( + raw=httpx.Response( + 200, + content=b'{"prompt": "What was Apple\'s total net sales for 2022?", "reference": "...", "response": "...", "guideline": "", "knowledge": "[]", "results": "{\'equivalence_score\': {\'score\': 0.0, \'metadata\': \\"The generated response does not provide any information about Apple\'s total net sales for 2022, whereas the reference response provides the specific figure.\\"}, \'factuality_v4.5_score\': {\'score\': 0.0, \'metadata\': {\'description\': \'There are claims but no knowledge so response is ungrounded.\'}}}", "status": "completed"}\r\n', + ), + client=client, + stream=False, + stream_cls=None, + cast_to=bytes, + options=FinalRequestOptions.construct(method="get", url="/foo"), + ) + df = response.to_dataframe() + assert df.shape == (1, 10) + assert df["prompt"].astype(str).iloc[0] == "What was Apple's total net sales for 2022?" # type: ignore + assert df["equivalence_score_score"].astype(float).iloc[0] == 0.0 # type: ignore + + @pytest.mark.asyncio async def test_async_response_parse_mismatched_basemodel(async_client: AsyncContextualAI) -> None: response = AsyncAPIResponse( From fb533a9aeac7e58926c5c13a74dc60b0c7d6ef7b Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 07:26:05 +0000 Subject: [PATCH 4/4] release: 0.5.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 14 ++++++++++++++ pyproject.toml | 2 +- src/contextual/_version.py | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index da59f99e..2aca35ae 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.4.0" + ".": "0.5.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 3273f3a0..9c5a79d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.5.0 (2025-03-11) + +Full Changelog: [v0.4.0...v0.5.0](https://github.com/ContextualAI/contextual-client-python/compare/v0.4.0...v0.5.0) + +### Features + +* Add to_dataframe method to BinaryAPIReponse ([#56](https://github.com/ContextualAI/contextual-client-python/issues/56)) ([39b862e](https://github.com/ContextualAI/contextual-client-python/commit/39b862eca8d7443c2c86063123d8dfdc484a3c53)) +* **api:** update via SDK Studio ([#63](https://github.com/ContextualAI/contextual-client-python/issues/63)) ([59bb1ab](https://github.com/ContextualAI/contextual-client-python/commit/59bb1ab3d790ee7e3d73b2b6a85e67a905d0ca22)) + + +### Chores + +* **internal:** remove unused http client options forwarding ([#61](https://github.com/ContextualAI/contextual-client-python/issues/61)) ([40d345d](https://github.com/ContextualAI/contextual-client-python/commit/40d345dd52af82e31e8fa34e5b0b1eebad006684)) + ## 0.4.0 (2025-03-03) Full Changelog: [v0.3.0...v0.4.0](https://github.com/ContextualAI/contextual-client-python/compare/v0.3.0...v0.4.0) diff --git a/pyproject.toml b/pyproject.toml index c0aa3fa2..da6a733c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "contextual-client" -version = "0.4.0" +version = "0.5.0" description = "The official Python library for the Contextual AI API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/contextual/_version.py b/src/contextual/_version.py index da82a52b..85259477 100644 --- a/src/contextual/_version.py +++ b/src/contextual/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "contextual" -__version__ = "0.4.0" # x-release-please-version +__version__ = "0.5.0" # x-release-please-version