From bc912c11744a6f4c60e87185342ba963a0783369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pstr=C4=85g?= Date: Mon, 27 May 2024 10:06:00 +0000 Subject: [PATCH] docs: update guides and API refs (#38) --- docs/how-to/llms/custom.md | 118 ++++++++++++++++++ docs/how-to/llms/litellm.md | 98 +++++++++++++++ docs/quickstart/index.md | 2 +- docs/quickstart/quickstart2.md | 4 +- docs/quickstart/quickstart3.md | 4 +- docs/reference/embeddings/index.md | 2 +- docs/reference/embeddings/litellm.md | 3 + docs/reference/embeddings/openai.md | 5 - .../event_handlers/langsmith_handler.md | 2 +- docs/reference/index.md | 2 +- docs/reference/iql/iql_generator.md | 5 +- docs/reference/llm/index.md | 15 --- docs/reference/llm/llm_options.md | 3 - docs/reference/llm/openai.md | 3 - docs/reference/llm/prompt_builder.md | 17 --- docs/reference/llms/index.md | 7 ++ docs/reference/llms/litellm.md | 7 ++ docs/reference/nl_responder.md | 4 +- docs/reference/similarity/index.md | 2 +- .../view_selection/llm_view_selector.md | 2 +- docs/{about => }/roadmap.md | 22 ++-- docs/tutorials.md | 5 + mkdocs.yml | 28 +++-- src/dbally/llms/base.py | 8 +- src/dbally/llms/litellm.py | 2 +- tests/integration/test_llm_options.py | 6 +- tests/unit/mocks.py | 2 +- tests/unit/test_iql_generator.py | 2 +- tests/unit/test_nl_responder.py | 2 +- tests/unit/test_prompt_builder.py | 6 +- tests/unit/test_view_selector.py | 2 +- tests/unit/views/text2sql/test_view.py | 2 +- 32 files changed, 300 insertions(+), 92 deletions(-) create mode 100644 docs/how-to/llms/custom.md create mode 100644 docs/how-to/llms/litellm.md create mode 100644 docs/reference/embeddings/litellm.md delete mode 100644 docs/reference/embeddings/openai.md delete mode 100644 docs/reference/llm/index.md delete mode 100644 docs/reference/llm/llm_options.md delete mode 100644 docs/reference/llm/openai.md delete mode 100644 docs/reference/llm/prompt_builder.md create mode 100644 docs/reference/llms/index.md create mode 100644 docs/reference/llms/litellm.md rename docs/{about => }/roadmap.md (82%) diff --git a/docs/how-to/llms/custom.md b/docs/how-to/llms/custom.md new file mode 100644 index 00000000..c262351d --- /dev/null +++ b/docs/how-to/llms/custom.md @@ -0,0 +1,118 @@ +# How-To: Create Custom LLM + +LLM is one of the main components of the db-ally ecosystem. It handles all interactions with the selected Large Language Model. It is used for operations like view selection, IQL generation and natural language response generation, therefore it is essential to be able to integrate with any LLM API you may encounter. + +## Implementing a Custom LLM + +The [`LLM`](../../reference/llms/index.md#dbally.llms.base.LLM) class is an abstract base class that provides a framework for interacting with a Large Language Model. To create a custom LLM, you need to create a subclass of [`LLM`](../../reference/llms/index.md#dbally.llms.base.LLM) and implement the required methods and properties. + +Here's a step-by-step guide: + +### Step 1: Define the subclass + +First, define your subclass and specify the type of options it will use. + +```python +from dbally.llms.base import LLM +from dbally.llms.litellm import LiteLLMOptions + +class MyLLM(LLM[LiteLLMOptions]): + _options_cls = LiteLLMOptions +``` + +In this example we will be using [`LiteLLMOptions`](../../reference/llms/litellm.md#dbally.llms.clients.litellm.LiteLLMOptions), which contain all options supported by most popular LLM APIs. If you need a different interface, see [Customising LLM Options](#customising-llm-options) to learn how to implement it. + +### Step 2: Create the custom LLM client + +The [`client`](../../reference/llms/index.md#dbally.llms.base.LLM.client) property is an abstract method that must be implemented in your subclass. This property should return an instance of [`LLMClient`](../../reference/llms/index.md#dbally.llms.clients.base.LLMClient) that your LLM will use to interact with the model. + +```python +class MyLLM(LLM[LiteLLMOptions]): + _options_cls = LiteLLMOptions + + @cached_property + def client(self) -> MyLLMClient: + return MyLLMClient() +``` + +`MyLLMClient` should be a class that implements the [`LLMClient`](../../reference/llms/index.md#dbally.llms.clients.base.LLMClient) interface. + +```python +from dbally.llms.clients.base import LLMClient + +class MyLLMClient(LLMClient[LiteLLMOptions]): + + async def call( + self, + prompt: ChatFormat, + response_format: Optional[Dict[str, str]], + options: LiteLLMOptions, + event: LLMEvent, + ) -> str: + # Your LLM API call +``` + +The [`call`](../../reference/llms/index.md#dbally.llms.clients.base.LLMClient.call) method is an abstract method that must be implemented in your subclass. This method should call the LLM inference API and return the response. + +### Step 3: Use tokenizer to count tokens + +The [`count_tokens`](../../reference/llms/index.md#dbally.llms.base.LLM.count_tokens) method is used to count the number of tokens in the messages. You can override this method in your custom class to use the tokenizer and count tokens specifically for your model. + +```python +class MyLLM(LLM[LiteLLMOptions]): + + def count_tokens(self, messages: ChatFormat, fmt: Dict[str, str]) -> int: + # Count tokens in the messages in a custom way +``` +!!!warning + Incorrect token counting can cause problems in the [`NLResponder`](../../reference/nl_responder.md#dbally.nl_responder.nl_responder.NLResponder) and force the use of an explanation prompt template that is more generic and does not include specific rows from the IQL response. + +### Step 4: Define custom prompt formatting + +The [`format_prompt`](../../reference/llms/index.md#dbally.llms.base.LLM.format_prompt) method is used to apply formatting to the prompt template. You can override this method in your custom class to change how the formatting is performed. + +```python +class MyLLM(LLM[LiteLLMOptions]): + + def format_prompt(self, template: PromptTemplate, fmt: Dict[str, str]) -> ChatFormat: + # Apply custom formatting to the prompt template +``` +!!!note + In general, implementation of this method is not required unless the LLM API does not support [OpenAI conversation formatting](https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages){:target="_blank"}. If your model API expects a different format, override this method to avoid issues with inference call. + +## Customising LLM Options + +[`LLMOptions`](../../reference/llms/index.md#dbally.llms.clients.base.LLMOptions) is a class that defines the options your LLM will use. To create a custom options, you need to create a subclass of [`LLMOptions`](../../reference/llms/index.md#dbally.llms.clients.base.LLMOptions) and define the required properties that will be passed to the [`LLMClient`](../../reference/llms/index.md#dbally.llms.clients.base.LLMClient). + +```python +from dbally.llms.base import LLMOptions + +@dataclass +class MyLLMOptions(LLMOptions): + temperature: float + max_tokens: int = 4096 +``` + +Each property should be annotated with its type. You can also provide default values if necessary. Don't forget to update the custom LLM class signatures. + +```python +class MyLLM(LLM[MyLLMOptions]): + _options_cls = MyLLMOptions + +class MyLLMClient(LLMClient[MyLLMOptions]): + ... +``` + +## Using the Custom LLM + +Once your subclass is defined, you can instantiate and use it with your collection like this. + +```python +import dbally + +llm = MyLLM("my_model", MyLLMOptions(temperature=0.5)) +my_collection = dbally.create_collection("my_collection", llm) +response = await my_collection.ask("Which LLM should I use?") +``` + +Now your custom model powers the db-ally engine for querying structured data. Have fun! diff --git a/docs/how-to/llms/litellm.md b/docs/how-to/llms/litellm.md new file mode 100644 index 00000000..03f41208 --- /dev/null +++ b/docs/how-to/llms/litellm.md @@ -0,0 +1,98 @@ +# How-To: Use LiteLLM models + +db-ally comes with ready-to-use LLM implementation called [`LiteLLM`](../../reference/llms/litellm.md#dbally.llms.litellm.LiteLLM) that uses the litellm package under the hood, providing access to all major LLM APIs such as OpenAI, Anthropic, VertexAI, Hugging Face and more. + +## Basic Usage + +Install litellm extension. + +```bash +pip install dbally[litellm] +``` + +Integrate db-ally with your LLM vendor. + +=== "OpenAI" + + ```python + import os + from dbally.llms.litellm import LiteLLM + + ## set ENV variables + os.environ["OPENAI_API_KEY"] = "your-api-key" + + llm=LiteLLM(model_name="gpt-4o") + ``` + +=== "Anthropic" + + ```python + import os + from dbally.llms.litellm import LiteLLM + + ## set ENV variables + os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + + llm=LiteLLM(model_name="claude-3-opus-20240229") + ``` + +=== "Anyscale" + + ```python + import os + from dbally.llms.litellm import LiteLLM + + ## set ENV variables + os.environ["ANYSCALE_API_KEY"] = "your-api-key" + + llm=LiteLLM(model_name="anyscale/meta-llama/Llama-2-70b-chat-hf") + ``` + +Use LLM in your collection. + +```python +my_collection = dbally.create_collection("my_collection", llm) +response = await my_collection.ask("Which LLM should I use?") +``` + +## Advanced Usage + +For more advanced users, you may also want to parametrize your LLM using [`LiteLLMOptions`](../../reference/llms/litellm.md#dbally.llms.clients.litellm.LiteLLMOptions). Here is the list of availabe parameters: + +- `frequency_penalty`: *number or null (optional)* - It is used to penalize new tokens based on their frequency in the text so far. + +- `max_tokens`: *integer (optional)* - The maximum number of tokens to generate in the chat completion. + +- `n`: *integer or null (optional)* - The number of chat completion choices to generate for each input message. + +- `presence_penalty`: *number or null (optional)* - It is used to penalize new tokens based on their existence in the text so far. + +- `seed`: *integer or null (optional)* - This feature is in Beta. If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend. + +- `stop`: *string/ array/ null (optional)* - Up to 4 sequences where the API will stop generating further tokens. + +- `temperature`: *number or null (optional)* - The sampling temperature to be used, between 0 and 2. Higher values like 0.8 produce more random outputs, while lower values like 0.2 make outputs more focused and deterministic. + +- `top_p`: *number or null (optional)* - An alternative to sampling with temperature. It instructs the model to consider the results of the tokens with top_p probability. For example, 0.1 means only the tokens comprising the top 10% probability mass are considered. + +```python +import dbally + +llm = MyLLM("my_model", LiteLLMOptions(temperature=0.5)) +my_collection = dbally.create_collection("my_collection", llm) +``` + +You can also override any default parameter on [`ask`](../../reference/collection.md#dbally.Collection.ask) call. + +```python +response = await my_collection.ask( + question="Which LLM should I use?", + llm_options=LiteLLMOptions( + temperature=0.65, + max_tokens=1024, + ), +) +``` + +!!!warning + Some parameters are not compatible with some models and may cause exceptions, check [LiteLLM documentation](https://docs.litellm.ai/docs/completion/input#translated-openai-params){:target="_blank"} for supported options. diff --git a/docs/quickstart/index.md b/docs/quickstart/index.md index 405f651c..2852e966 100644 --- a/docs/quickstart/index.md +++ b/docs/quickstart/index.md @@ -1,4 +1,4 @@ -# Quickstart Guide 1 +# Quickstart: Intro This guide will help you get started with db-ally. We will use a simple example to demonstrate how to use db-ally to query a database using an AI model. We will use OpenAI's GPT to generate SQL queries based on natural language questions and SqlAlchemy to interact with the database. diff --git a/docs/quickstart/quickstart2.md b/docs/quickstart/quickstart2.md index cb839bff..d3bf968a 100644 --- a/docs/quickstart/quickstart2.md +++ b/docs/quickstart/quickstart2.md @@ -1,6 +1,6 @@ -# Quickstart Guide 2: Semantic Similarity +# Quickstart: Semantic Similarity -This guide is a continuation of the [Quickstart](./index.md) guide. It assumes that you have already set up the views and the collection. If not, please refer to the complete Part 1 code here: [quickstart_code.py](quickstart_code.py). +This guide is a continuation of the [Intro](./index.md) guide. It assumes that you have already set up the views and the collection. If not, please refer to the complete Part 1 code here: [quickstart_code.py](quickstart_code.py). This guide will demonstrate how to use semantic similarity to handle queries in which the filter values are similar to those in the database, without requiring an exact match. We will use filtering by country as an example. diff --git a/docs/quickstart/quickstart3.md b/docs/quickstart/quickstart3.md index 8147017f..4fd9e86a 100644 --- a/docs/quickstart/quickstart3.md +++ b/docs/quickstart/quickstart3.md @@ -1,6 +1,6 @@ -# Quickstart Guide 3: Multiple Views +# Quickstart: Multiple Views -This guide continues from [Quickstart Guide 2](./quickstart2.md). It assumes that you have already set up the views and the collection. If not, please refer to the complete Part 2 code here: [quickstart2_code.py](quickstart2_code.py). +This guide continues from [Semantic Similarity](./quickstart2.md) guide. It assumes that you have already set up the views and the collection. If not, please refer to the complete Part 2 code here: [quickstart2_code.py](quickstart2_code.py). The guide illustrates how to use multiple views to handle queries requiring different types of data. `CandidateView` and `JobView` are used as examples. diff --git a/docs/reference/embeddings/index.md b/docs/reference/embeddings/index.md index 82a4ffa4..8fab248a 100644 --- a/docs/reference/embeddings/index.md +++ b/docs/reference/embeddings/index.md @@ -1,3 +1,3 @@ # EmbeddingClient -::: dbally.embedding_client.EmbeddingClient +::: dbally.embeddings.EmbeddingClient diff --git a/docs/reference/embeddings/litellm.md b/docs/reference/embeddings/litellm.md new file mode 100644 index 00000000..9231c51f --- /dev/null +++ b/docs/reference/embeddings/litellm.md @@ -0,0 +1,3 @@ +# LiteLLMEmbeddingClient + +::: dbally.embeddings.LiteLLMEmbeddingClient diff --git a/docs/reference/embeddings/openai.md b/docs/reference/embeddings/openai.md deleted file mode 100644 index f127d58c..00000000 --- a/docs/reference/embeddings/openai.md +++ /dev/null @@ -1,5 +0,0 @@ -# OpenAiEmbeddingClient - -db-ally provides an `OpenAIEmbeddingClient` class that can be used to create text embeddings using the OpenAI API. - -::: dbally.embedding_client.OpenAiEmbeddingClient diff --git a/docs/reference/event_handlers/langsmith_handler.md b/docs/reference/event_handlers/langsmith_handler.md index 333de56e..e3ac6c69 100644 --- a/docs/reference/event_handlers/langsmith_handler.md +++ b/docs/reference/event_handlers/langsmith_handler.md @@ -1,4 +1,4 @@ -# LangmisthEventHandler +# LangSmithEventHandler ::: dbally.audit.LangSmithEventHandler diff --git a/docs/reference/index.md b/docs/reference/index.md index 504e5122..0deb591a 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -1,4 +1,4 @@ -# db-ally root API +# dbally ::: dbally.create_collection diff --git a/docs/reference/iql/iql_generator.md b/docs/reference/iql/iql_generator.md index 087deff4..15edcb56 100644 --- a/docs/reference/iql/iql_generator.md +++ b/docs/reference/iql/iql_generator.md @@ -2,7 +2,6 @@ ::: dbally.iql_generator.iql_generator.IQLGenerator -::: dbally.data_models.prompts.iql_prompt_template.default_iql_template - -::: dbally.data_models.prompts.IQLPromptTemplate +::: dbally.iql_generator.iql_prompt_template.IQLPromptTemplate +::: dbally.iql_generator.iql_prompt_template.default_iql_template diff --git a/docs/reference/llm/index.md b/docs/reference/llm/index.md deleted file mode 100644 index 437fc577..00000000 --- a/docs/reference/llm/index.md +++ /dev/null @@ -1,15 +0,0 @@ -# LLMClient - - -`LLMClient` is an abstract class designed to interact with LLMs. - -Concrete implementations for specific LLMs, like OpenAILLMClient, can be found in this section of our documentation. - -[`LLMClient` configuration options][dbally.data_models.llm_options.LLMOptions] include: template, format, event tracker, and optional generation parameters like -frequency_penalty, max_tokens, and temperature. - -It constructs prompts using the [`PromptBuilder`](./prompt_builder.md) instance. - - -::: dbally.llm_client.base.LLMClient - diff --git a/docs/reference/llm/llm_options.md b/docs/reference/llm/llm_options.md deleted file mode 100644 index d360f00c..00000000 --- a/docs/reference/llm/llm_options.md +++ /dev/null @@ -1,3 +0,0 @@ -# LLMOptions - -::: dbally.data_models.llm_options.LLMOptions diff --git a/docs/reference/llm/openai.md b/docs/reference/llm/openai.md deleted file mode 100644 index a45ef7e4..00000000 --- a/docs/reference/llm/openai.md +++ /dev/null @@ -1,3 +0,0 @@ -# OpenAILLMClient - -::: dbally.llm_client.openai_client.OpenAIClient diff --git a/docs/reference/llm/prompt_builder.md b/docs/reference/llm/prompt_builder.md deleted file mode 100644 index 2fa1fa36..00000000 --- a/docs/reference/llm/prompt_builder.md +++ /dev/null @@ -1,17 +0,0 @@ -# PromptBuilder - -`PromptBuilder` is an abstract class designed to facilitate the construction of prompts for interaction with LLMs. - -## Lifecycle - -The class is initialized with an optional argument `model_name`, representing the name of the model for which a tokenizer should be loaded. - -!!! tip - To use OpenAIs' models, you do not need to provide the `model_name` argument because tokenization is performed inside the OpenAI API. - -The main method used to build prompts is `self.build`. -It takes a `prompt_template` (a [`ChatFormat`][dbally.data_models.prompts.common_validation_utils.ChatFormat] object) and a formatting dictionary `fmt` as input. It formats the prompt using the `format_prompt` method. If a tokenizer is available (i.e., if `model_name` was provided during initialization), it applies the tokenizer to the formatted prompt. The method returns the formatted prompt, either as a string (if it was formatted for a Hugging Face model) or as a [`ChatFormat`][dbally.data_models.prompts.common_validation_utils.ChatFormat]. - - -::: dbally.data_models.prompts.common_validation_utils.ChatFormat -::: dbally.prompts.prompt_builder.PromptBuilder diff --git a/docs/reference/llms/index.md b/docs/reference/llms/index.md new file mode 100644 index 00000000..424206f3 --- /dev/null +++ b/docs/reference/llms/index.md @@ -0,0 +1,7 @@ +# LLM + +::: dbally.llms.base.LLM + +::: dbally.llms.clients.base.LLMClient + +::: dbally.llms.clients.base.LLMOptions diff --git a/docs/reference/llms/litellm.md b/docs/reference/llms/litellm.md new file mode 100644 index 00000000..a704558f --- /dev/null +++ b/docs/reference/llms/litellm.md @@ -0,0 +1,7 @@ +# LiteLLM + +::: dbally.llms.litellm.LiteLLM + +::: dbally.llms.clients.litellm.LiteLLMClient + +::: dbally.llms.clients.litellm.LiteLLMOptions diff --git a/docs/reference/nl_responder.md b/docs/reference/nl_responder.md index dd5f1445..fb80741c 100644 --- a/docs/reference/nl_responder.md +++ b/docs/reference/nl_responder.md @@ -27,6 +27,6 @@ Otherwise, a response is generated using a `nl_responder_prompt_template`. ::: dbally.nl_responder.nl_responder.NLResponder -::: dbally.data_models.prompts.query_explainer_prompt_template +::: dbally.nl_responder.query_explainer_prompt_template -::: dbally.data_models.prompts.nl_responder_prompt_template.default_nl_responder_template +::: dbally.nl_responder.nl_responder_prompt_template.default_nl_responder_template diff --git a/docs/reference/similarity/index.md b/docs/reference/similarity/index.md index 0963cb84..aca1f119 100644 --- a/docs/reference/similarity/index.md +++ b/docs/reference/similarity/index.md @@ -10,6 +10,6 @@ Explore [Similarity Stores](./similarity_store/index.md) and [Similarity Fetcher * [How-To: Use Similarity Indexes with Data from Custom Sources](../../how-to/use_custom_similarity_fetcher.md) * [How-To: Store Similarity Index in a Custom Store](../../how-to/use_custom_similarity_store.md) * [How-To: Update Similarity Indexes](../../how-to/update_similarity_indexes.md) - * [Quickstart Guide 2: Semantic Similarity](../../quickstart/quickstart2.md) + * [Quickstart: Semantic Similarity](../../quickstart/quickstart2.md) ::: dbally.similarity.SimilarityIndex diff --git a/docs/reference/view_selection/llm_view_selector.md b/docs/reference/view_selection/llm_view_selector.md index 5077b842..774aa4b9 100644 --- a/docs/reference/view_selection/llm_view_selector.md +++ b/docs/reference/view_selection/llm_view_selector.md @@ -2,4 +2,4 @@ ::: dbally.view_selection.LLMViewSelector -::: dbally.data_models.prompts.default_view_selector_template +::: dbally.view_selection.view_selector_prompt_template.default_view_selector_template diff --git a/docs/about/roadmap.md b/docs/roadmap.md similarity index 82% rename from docs/about/roadmap.md rename to docs/roadmap.md index 6585facb..f590f35a 100644 --- a/docs/about/roadmap.md +++ b/docs/roadmap.md @@ -1,3 +1,8 @@ +--- +hide: + - navigation +--- + # Roadmap db-ally is actively developed and maintained by a core team at [deepsense.ai](https://deepsense.ai) and a community of contributors. @@ -30,13 +35,16 @@ Below you can find a list of planned integrations. - [ ] HTTP REST Endpoints - [ ] GraphQL Endpoints -### LLMs +### LLM Providers + +- [x] OpenAI +- [x] Anthropic +- [x] VertexAI +- [x] Hugging Face +- [x] Bedrock +- [x] Azure -- [x] OpenAI models -- [ ] Claude 3 -- [ ] LLama-2 -- [ ] Mistral / Mixtral -- [ ] VertexAI Gemini +And many more, the full list can be found in the [LiteLLM documentation](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs){:target="_blank"} ### Vector stores @@ -50,5 +58,5 @@ Below you can find a list of planned integrations. ### Query tracking -- [x] Langsmith +- [x] LangSmith - [ ] OpenTelemetry diff --git a/docs/tutorials.md b/docs/tutorials.md index d354c9f4..6d327503 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -1,3 +1,8 @@ +--- +hide: + - navigation +--- + # Tutorials All tutorials are opened in Google Colab so that you can easily follow them without any additional setup. diff --git a/mkdocs.yml b/mkdocs.yml index 8701dd09..5d42aa5e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,13 +1,12 @@ site_name: db-ally repo_url: https://github.com/deepsense-ai/db-ally nav: - - db-ally: index.md + - Home: index.md - Quickstart: - quickstart/index.md - quickstart/quickstart2.md - quickstart/quickstart3.md - - Tutorials: - - tutorials.md + - Tutorials: tutorials.md - Concepts: - concepts/views.md - concepts/structured_views.md @@ -21,6 +20,9 @@ nav: - how-to/sql_views.md - how-to/pandas_views.md - how-to/custom_views.md + - Using LLMs: + - how-to/llms/litellm.md + - how-to/llms/custom.md - Using similarity indexes: - how-to/use_custom_similarity_fetcher.md - how-to/use_chromadb_store.md @@ -49,10 +51,8 @@ nav: - reference/view_selection/llm_view_selector.md - reference/nl_responder.md - LLMs: - - reference/llm/index.md - - reference/llm/llm_options.md - - reference/llm/openai.md - - reference/llm/prompt_builder.md + - reference/llms/index.md + - reference/llms/litellm.md - Similarity: - reference/similarity/index.md - Store: @@ -66,12 +66,13 @@ nav: - reference/similarity/detector.md - Embeddings: - reference/embeddings/index.md - - reference/embeddings/openai.md - - About: - - about/roadmap.md + - reference/embeddings/litellm.md + - Roadmap: roadmap.md theme: name: material + icon: + repo: fontawesome/brands/github palette: - media: "(prefers-color-scheme: light)" scheme: default @@ -86,7 +87,9 @@ theme: icon: material/brightness-4 name: Switch to light mode features: + - navigation.footer - navigation.tabs + - navigation.top - content.code.annotate - content.code.copy extra_css: @@ -100,7 +103,6 @@ markdown_extensions: custom_fences: - name: mermaid class: mermaid - - admonition - pymdownx.superfences - pymdownx.snippets @@ -110,6 +112,10 @@ markdown_extensions: - def_list - pymdownx.tasklist: custom_checkbox: true + - pymdownx.tabbed: + alternate_style: true + - toc: + permalink: "#" plugins: - search - autorefs: diff --git a/src/dbally/llms/base.py b/src/dbally/llms/base.py index 3d6c90e0..e570547c 100644 --- a/src/dbally/llms/base.py +++ b/src/dbally/llms/base.py @@ -36,12 +36,12 @@ def __init_subclass__(cls) -> None: @cached_property @abstractmethod - def _client(self) -> LLMClient: + def client(self) -> LLMClient: """ Client for the LLM. """ - def _format_prompt(self, template: PromptTemplate, fmt: Dict[str, str]) -> ChatFormat: + def format_prompt(self, template: PromptTemplate, fmt: Dict[str, str]) -> ChatFormat: """ Applies formatting to the prompt template. @@ -88,12 +88,12 @@ async def generate_text( Text response from LLM. """ options = (self.default_options | options) if options else self.default_options - prompt = self._format_prompt(template, fmt) + prompt = self.format_prompt(template, fmt) event = LLMEvent(prompt=prompt, type=type(template).__name__) event_tracker = event_tracker or EventTracker() async with event_tracker.track_event(event) as span: - event.response = await self._client.call( + event.response = await self.client.call( prompt=prompt, response_format=template.response_format, options=options, diff --git a/src/dbally/llms/litellm.py b/src/dbally/llms/litellm.py index f6b6727d..c5699a1e 100644 --- a/src/dbally/llms/litellm.py +++ b/src/dbally/llms/litellm.py @@ -54,7 +54,7 @@ def __init__( self.api_version = api_version @cached_property - def _client(self) -> LiteLLMClient: + def client(self) -> LiteLLMClient: """ Client for the LLM. """ diff --git a/tests/integration/test_llm_options.py b/tests/integration/test_llm_options.py index 79dde194..e8c53435 100644 --- a/tests/integration/test_llm_options.py +++ b/tests/integration/test_llm_options.py @@ -14,7 +14,7 @@ async def test_llm_options_propagation(): expected_options = MockLLMOptions(mock_property1=2, mock_property2="default mock") llm = MockLLM(default_options=default_options) - llm._client.call = AsyncMock(return_value="MockView1") + llm.client.call = AsyncMock(return_value="MockView1") collection = create_collection( name="test_collection", @@ -30,9 +30,9 @@ async def test_llm_options_propagation(): llm_options=custom_options, ) - assert llm._client.call.call_count == 3 + assert llm.client.call.call_count == 3 - llm._client.call.assert_has_calls( + llm.client.call.assert_has_calls( [ call( prompt=ANY, diff --git a/tests/unit/mocks.py b/tests/unit/mocks.py index f7de3faf..9858e45f 100644 --- a/tests/unit/mocks.py +++ b/tests/unit/mocks.py @@ -84,5 +84,5 @@ def __init__(self, default_options: Optional[MockLLMOptions] = None) -> None: super().__init__("mock-llm", default_options) @cached_property - def _client(self) -> MockLLMClient: + def client(self) -> MockLLMClient: return MockLLMClient(model_name=self.model_name) diff --git a/tests/unit/test_iql_generator.py b/tests/unit/test_iql_generator.py index af991c31..c330f747 100644 --- a/tests/unit/test_iql_generator.py +++ b/tests/unit/test_iql_generator.py @@ -41,7 +41,7 @@ def view() -> MockView: @pytest.fixture def llm() -> MockLLM: llm = MockLLM() - llm._client.call = AsyncMock(return_value="LLM IQL mock answer") + llm.client.call = AsyncMock(return_value="LLM IQL mock answer") return llm diff --git a/tests/unit/test_nl_responder.py b/tests/unit/test_nl_responder.py index b4237d01..d7af9409 100644 --- a/tests/unit/test_nl_responder.py +++ b/tests/unit/test_nl_responder.py @@ -11,7 +11,7 @@ @pytest.fixture def llm() -> MockLLM: llm = MockLLM() - llm._client.call = AsyncMock(return_value="db-ally is the best") + llm.client.call = AsyncMock(return_value="db-ally is the best") return llm diff --git a/tests/unit/test_prompt_builder.py b/tests/unit/test_prompt_builder.py index 9c3d0695..f8a886fe 100644 --- a/tests/unit/test_prompt_builder.py +++ b/tests/unit/test_prompt_builder.py @@ -22,7 +22,7 @@ def llm(): def test_default_llm_format_prompt(llm, simple_template): - prompt = llm._format_prompt( + prompt = llm.format_prompt( template=simple_template, fmt={"question": "Example user question?"}, ) @@ -34,7 +34,7 @@ def test_default_llm_format_prompt(llm, simple_template): def test_missing_format_dict(llm, simple_template): with pytest.raises(KeyError): - _ = llm._format_prompt(simple_template, fmt={}) + _ = llm.format_prompt(simple_template, fmt={}) @pytest.mark.parametrize( @@ -66,7 +66,7 @@ def test_chat_order_validation(invalid_chat): def test_dynamic_few_shot(llm, simple_template): assert ( len( - llm._format_prompt( + llm.format_prompt( simple_template.add_assistant_message("assistant message").add_user_message("user message"), fmt={"question": "user question"}, ) diff --git a/tests/unit/test_view_selector.py b/tests/unit/test_view_selector.py index a135b34e..2d3b1d9c 100644 --- a/tests/unit/test_view_selector.py +++ b/tests/unit/test_view_selector.py @@ -17,7 +17,7 @@ def llm() -> LLM: """Return a mock LLM client.""" llm = MockLLM() - llm._client.call = AsyncMock(return_value="MockView1") + llm.client.call = AsyncMock(return_value="MockView1") return llm diff --git a/tests/unit/views/text2sql/test_view.py b/tests/unit/views/text2sql/test_view.py index 02dad2fe..a33ebba4 100644 --- a/tests/unit/views/text2sql/test_view.py +++ b/tests/unit/views/text2sql/test_view.py @@ -33,7 +33,7 @@ def sample_db() -> Engine: async def test_text2sql_view(sample_db: Engine): llm = MockLLM() - llm._client.call = AsyncMock(return_value="SELECT * FROM customers WHERE city = 'New York'") + llm.client.call = AsyncMock(return_value="SELECT * FROM customers WHERE city = 'New York'") config = Text2SQLConfig( tables={