diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d16317658..369d2fa09f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,8 +59,9 @@ jobs: enable-cache: true - run: uv sync --python 3.12 --frozen --group docs - - run: docs - if: github.repository_owner != 'pydantic' + + # always build docs to check it works without insiders packages + - run: make docs - run: make docs-insiders if: github.repository_owner == 'pydantic' diff --git a/Makefile b/Makefile index 35ccea20cb..5887557c3f 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,9 @@ docs-serve: .docs-insiders-install: ifeq ($(shell uv pip show mkdocs-material | grep -q insiders && echo 'installed'), installed) @echo 'insiders packages already installed' +else ifeq ($(PPPR_TOKEN),) + @echo "Error: PPPR_TOKEN is not set, can't install insiders packages" + @exit 1 else @echo 'installing insiders packages...' @uv pip install -U mkdocs-material mkdocstrings-python \ @@ -79,11 +82,11 @@ endif .PHONY: docs-insiders # Build the documentation using insiders packages docs-insiders: .docs-insiders-install - uv run --no-sync mkdocs build + uv run --no-sync mkdocs build -f mkdocs.insiders.yml .PHONY: docs-serve-insiders # Build and serve the documentation using insiders packages docs-serve-insiders: .docs-insiders-install - uv run --no-sync mkdocs serve + uv run --no-sync mkdocs serve -f mkdocs.insiders.yml .PHONY: cf-pages-build # Install uv, install dependencies and build the docs, used on CloudFlare Pages cf-pages-build: diff --git a/docs/agents.md b/docs/agents.md index 5c9c0d3a11..6f22f6617a 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -11,7 +11,7 @@ The [`Agent`][pydantic_ai.Agent] class is well documented, but in essence you ca * One or more [retrievers](#retrievers) — functions that the LLM may call to get information while generating a response * An optional structured [result type](results.md) — the structured datatype the LLM must return at the end of a run * A [dependency](dependencies.md) type constraint — system prompt functions, retrievers and result validators may all use dependencies when they're run -* Agents may optionally also have a default [model](models/index.md) associated with them, the model to use can also be defined when running the agent +* Agents may optionally also have a default [model](api/models/base.md) associated with them, the model to use can also be defined when running the agent In typing terms, agents are generic in their dependency and result types, e.g. an agent which required `#!python Foobar` dependencies and returned data of type `#!python list[str]` results would have type `#!python Agent[Foobar, list[str]]`. @@ -350,7 +350,9 @@ agent.run_sync('hello', model=FunctionModel(print_schema)) _(This example is complete, it can be run "as is")_ -The return type of retriever can any valid JSON object ([`JsonData`][pydantic_ai.dependencies.JsonData]) as some models (e.g. Gemini) support semi-structured return values, some expect text (OpenAI) but seem to be just as good at extracting meaning from the data, if a Python is returned and the model expects a string, the value will be serialized to JSON +The return type of retriever can any valid JSON object ([`JsonData`][pydantic_ai.dependencies.JsonData]) as some models (e.g. Gemini) support semi-structured return values, some expect text (OpenAI) but seem to be just as good at extracting meaning from the data, if a Python is returned and the model expects a string, the value will be serialized to JSON. + +If a retriever has a single parameter that can be represented as an object in JSON schema (e.g. dataclass, TypedDict, pydantic model), the schema for the retriever is simplified to be just that object. (TODO example) ## Reflection and self-correction @@ -478,23 +480,3 @@ else: 1. Define a retriever that will raise `ModelRetry` repeatedly in this case. _(This example is complete, it can be run "as is")_ - -## API Reference - -::: pydantic_ai.Agent - options: - members: - - __init__ - - run - - run_sync - - run_stream - - model - - override_deps - - override_model - - last_run_messages - - system_prompt - - retriever_plain - - retriever_context - - result_validator - -::: pydantic_ai.exceptions diff --git a/docs/api/agent.md b/docs/api/agent.md new file mode 100644 index 0000000000..06de0fcc4d --- /dev/null +++ b/docs/api/agent.md @@ -0,0 +1,17 @@ +# `pydantic_ai.Agent` + +::: pydantic_ai.Agent + options: + members: + - __init__ + - run + - run_sync + - run_stream + - model + - override_deps + - override_model + - last_run_messages + - system_prompt + - retriever_plain + - retriever_context + - result_validator diff --git a/docs/api/dependencies.md b/docs/api/dependencies.md new file mode 100644 index 0000000000..9f49436a0a --- /dev/null +++ b/docs/api/dependencies.md @@ -0,0 +1,3 @@ +# `pydantic_ai.dependencies` + +::: pydantic_ai.dependencies diff --git a/docs/api/exceptions.md b/docs/api/exceptions.md new file mode 100644 index 0000000000..277b8fd350 --- /dev/null +++ b/docs/api/exceptions.md @@ -0,0 +1,3 @@ +# `pydantic_ai.exceptions` + +::: pydantic_ai.exceptions diff --git a/docs/api/messages.md b/docs/api/messages.md new file mode 100644 index 0000000000..9986b8504e --- /dev/null +++ b/docs/api/messages.md @@ -0,0 +1,17 @@ +# `pydantic_ai.messages` + +::: pydantic_ai.messages + options: + members: + - Message + - SystemPrompt + - UserPrompt + - ToolReturn + - RetryPrompt + - ModelAnyResponse + - ModelTextResponse + - ModelStructuredResponse + - ToolCall + - ArgsJson + - ArgsObject + - MessagesTypeAdapter diff --git a/docs/models/index.md b/docs/api/models/base.md similarity index 100% rename from docs/models/index.md rename to docs/api/models/base.md diff --git a/docs/models/function.md b/docs/api/models/function.md similarity index 50% rename from docs/models/function.md rename to docs/api/models/function.md index 831fc0ff11..f83fcf4a3f 100644 --- a/docs/models/function.md +++ b/docs/api/models/function.md @@ -1,3 +1,3 @@ -# FunctionModel +# `pydantic_ai.models.function` ::: pydantic_ai.models.function diff --git a/docs/models/gemini.md b/docs/api/models/gemini.md similarity index 50% rename from docs/models/gemini.md rename to docs/api/models/gemini.md index e37f9af7ca..5cf3315be0 100644 --- a/docs/models/gemini.md +++ b/docs/api/models/gemini.md @@ -1,3 +1,3 @@ -# Gemini +# `pydantic_ai.models.gemini` ::: pydantic_ai.models.gemini diff --git a/docs/models/openai.md b/docs/api/models/openai.md similarity index 50% rename from docs/models/openai.md rename to docs/api/models/openai.md index 1f072e755a..ab3cedb646 100644 --- a/docs/models/openai.md +++ b/docs/api/models/openai.md @@ -1,3 +1,3 @@ -# OpenAI +# `pydantic_ai.models.openai` ::: pydantic_ai.models.openai diff --git a/docs/models/test.md b/docs/api/models/test.md similarity index 50% rename from docs/models/test.md rename to docs/api/models/test.md index 9f3d7d09d7..35ffc19dd7 100644 --- a/docs/models/test.md +++ b/docs/api/models/test.md @@ -1,3 +1,3 @@ -# TestModel +# `pydantic_ai.models.test` ::: pydantic_ai.models.test diff --git a/docs/api/result.md b/docs/api/result.md new file mode 100644 index 0000000000..83d61af813 --- /dev/null +++ b/docs/api/result.md @@ -0,0 +1,10 @@ +# `pydantic_ai.result` + +::: pydantic_ai.result + options: + inherited_members: true + members: + - ResultData + - RunResult + - StreamedRunResult + - Cost diff --git a/docs/dependencies.md b/docs/dependencies.md index d97c66cd38..eff5f086b0 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -342,7 +342,3 @@ The following examples demonstrate how to use dependencies in PydanticAI: - [Weather Agent](examples/weather-agent.md) - [SQL Generation](examples/sql-gen.md) - [RAG](examples/rag.md) - -## API Reference - -::: pydantic_ai.dependencies diff --git a/docs/img/logfire-weather-agent.png b/docs/img/logfire-weather-agent.png new file mode 100644 index 0000000000..c121864d79 Binary files /dev/null and b/docs/img/logfire-weather-agent.png differ diff --git a/docs/logfire.md b/docs/logfire.md new file mode 100644 index 0000000000..2781f7282d --- /dev/null +++ b/docs/logfire.md @@ -0,0 +1,26 @@ +# Monitoring and Performance + +Applications that use LLMs have some challenges that are well known and understood: LLMs are **slow**, **unreliable** and **expensive**. +These applications also have some challenges that most developers have encountered much less often: they're **fickle** and **non-deterministic**. Subtle changes in a prompt can completely change a model's performance, and there's no `EXPLAIN` query you can run to understand why. + +From a software engineers point of view, you can think of LLMs as the worst database you've ever heard of, but worse. + +To build successful applications with LLMs, we need new tools to understand both model performance, and the behavior of applications that rely on them. + +LLM Observability tools that just let you understand how your model is performing are useless: making API calls to an LLM is easy, it's building that into an application that's hard. + +## Pydantic Logfire + +[Pydantic Logfire](https://pydantic.dev/logfire) is an observability platform from the developers of Pydantic and PydanticAI, that aims to let you understand your entire application: Gen AI, classic predictive AI, HTTP traffic, database queries and everything else a modern application needs. + +!!! note "Pydantic Logfire is a commercial product" + Logfire is a commercially supported, hosted platform with an extremely generous and perpetual free tier. + You can sign up and start using Logfire in a couple of minutes. + +PydanticAI has built-in (but optional) support for Logfire via the [`logfire-api`](https://github.com/pydantic/logfire/tree/main/logfire-api) no-op package. + +That means if the `logfire` package is installed, detailed information about agent runs is sent to Logfire. But if the `logfire` package is not installed, there's no overhead and nothing is sent. + +Here's an example showing details of running the [Weather Agent](examples/weather-agent.md) in Logfire: + +![Weather Agent Logfire](img/logfire-weather-agent.png) diff --git a/docs/message-history.md b/docs/message-history.md index 528b17e6d3..46483da0fb 100644 --- a/docs/message-history.md +++ b/docs/message-history.md @@ -25,7 +25,7 @@ and [`StreamedRunResult`][pydantic_ai.result.StreamedRunResult] (returned by [`A Example of accessing methods on a [`RunResult`][pydantic_ai.result.RunResult] : -```python title="Accessing messages from a RunResult" hl_lines="9 12" +```python title="run_result_messages.py" hl_lines="10 28" from pydantic_ai import Agent agent = Agent('openai:gpt-4o', system_prompt='Be a helpful assistant.') @@ -73,7 +73,7 @@ _(This example is complete, it can be run "as is")_ Example of accessing methods on a [`StreamedRunResult`][pydantic_ai.result.StreamedRunResult] : -```python title="Accessing messages from a StreamedRunResult" hl_lines="7 13" +```python title="streamed_run_result_messages.py" hl_lines="9 31" from pydantic_ai import Agent agent = Agent('openai:gpt-4o', system_prompt='Be a helpful assistant.') @@ -142,7 +142,7 @@ To use existing messages in a run, pass them to the `message_history` parameter [`all_messages()`][pydantic_ai.result.RunResult.all_messages] or [`new_messages()`][pydantic_ai.result.RunResult.new_messages]. -```py title="Reusing messages in a conversation" hl_lines="8 11" +```py title="Reusing messages in a conversation" hl_lines="9 13" from pydantic_ai import Agent agent = Agent('openai:gpt-4o', system_prompt='Be a helpful assistant.') @@ -236,21 +236,3 @@ print(result2.all_messages()) ## Examples For a more complete example of using messages in conversations, see the [chat app](examples/chat-app.md) example. - -## API Reference - -::: pydantic_ai.messages - options: - members: - - Message - - SystemPrompt - - UserPrompt - - ToolReturn - - RetryPrompt - - ModelAnyResponse - - ModelTextResponse - - ModelStructuredResponse - - ToolCall - - ArgsJson - - ArgsObject - - MessagesTypeAdapter diff --git a/docs/results.md b/docs/results.md index ca7f1ce1fe..d4b1610d3e 100644 --- a/docs/results.md +++ b/docs/results.md @@ -1,28 +1,41 @@ ## Ending runs -TODO +**TODO** + +* runs end when either a plain text response is received or the model calls a tool associated with one of the structured result types +* example +* we should add `message_limit` (number of model messages) and `cost_limit` to `run()` etc. ## Structured result validation +**TODO** + +* structured results (like retrievers) use Pydantic, Pydantic builds the JSON schema and does the validation +* PydanticAI tries hard to simplify the schema, this means: + * if the return type is `str` or a union including `str`, plain text responses are enabled + * if the schema is a union (after remove `str` from the members), each member is registered as its own tool call + * if the schema is not an object, the result type is wrapped in a single element object + ## Result validators functions -TODO +**TODO** + +* Some validation is inconvenient or impossible to do in Pydantic validators, in particular when the validation requires IO and is asynchronous. PydanticAI provides a way to add validation functions via the [`agent.result_validator`][pydantic_ai.Agent.result_validator] decorator. +* example ## Streamed Results -TODO +**TODO** -## Cost +Streamed responses provide a unique challenge: +* validating the partial result is both practically and semantically complex, but pydantic can do this +* we don't know if a result will be the final result of a run until we start streaming it, so PydanticAI has to start streaming just enough of the response to sniff out if it's the final response, then either stream the rest of the response to call a retriever, or return an object that lets the rest of the response be streamed by the user +* examples including: streaming text, streaming validated data, streaming the raw data to do validation inside a try/except block when necessary +* explanation of how streamed responses are "debounced" -TODO +## Cost -## API Reference +**TODO** -::: pydantic_ai.result - options: - inherited_members: true - members: - - ResultData - - RunResult - - StreamedRunResult - - Cost +* counts tokens, not dollars +* example diff --git a/mkdocs.insiders.yml b/mkdocs.insiders.yml new file mode 100644 index 0000000000..5e445682b9 --- /dev/null +++ b/mkdocs.insiders.yml @@ -0,0 +1,35 @@ +INHERIT: mkdocs.yml + +markdown_extensions: + - tables + - admonition + - attr_list + - md_in_html + - pymdownx.details + - pymdownx.caret + - pymdownx.critic + - pymdownx.mark + - pymdownx.superfences + - pymdownx.snippets + - pymdownx.tilde + - pymdownx.inlinehilite + - pymdownx.highlight: + pygments_lang_class: true + - pymdownx.extra: + pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - sane_lists # this means you can start a list from any number + - material.extensions.preview: + targets: + include: + - '*' diff --git a/mkdocs.yml b/mkdocs.yml index cdd6d256ab..ad1b0e0ced 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,12 +18,7 @@ nav: - results.md - message-history.md - testing-evals.md - - Models: - - models/index.md - - models/openai.md - - models/gemini.md - - models/test.md - - models/function.md + - logfire.md - Examples: - examples/index.md - examples/pydantic-model.md @@ -33,6 +28,17 @@ nav: - examples/stream-markdown.md - examples/stream-whales.md - examples/chat-app.md + - API Reference: + - api/agent.md + - api/result.md + - api/messages.md + - api/dependencies.md + - api/exceptions.md + - api/models/base.md + - api/models/openai.md + - api/models/gemini.md + - api/models/test.md + - api/models/function.md extra: # hide the "Made with Material for MkDocs" message @@ -120,10 +126,6 @@ markdown_extensions: - pymdownx.tasklist: custom_checkbox: true - sane_lists # this means you can start a list from any number - - material.extensions.preview: - targets: - include: - - '*' watch: - pydantic_ai @@ -132,6 +134,7 @@ watch: plugins: - search - social + - glightbox - mkdocstrings: handlers: python: diff --git a/pydantic_ai/agent.py b/pydantic_ai/agent.py index bd38da5878..270f1a3492 100644 --- a/pydantic_ai/agent.py +++ b/pydantic_ai/agent.py @@ -3,7 +3,7 @@ import asyncio from collections.abc import AsyncIterator, Iterator, Sequence from contextlib import asynccontextmanager, contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Callable, Generic, cast, final, overload import logfire_api @@ -54,18 +54,18 @@ class Agent(Generic[AgentDeps, ResultData]): # dataclass fields mostly for my sanity — knowing what attributes are available model: models.Model | models.KnownModelName | None """The default model configured for this agent.""" - _result_schema: _result.ResultSchema[ResultData] | None - _result_validators: list[_result.ResultValidator[AgentDeps, ResultData]] - _allow_text_result: bool - _system_prompts: tuple[str, ...] - _retrievers: dict[str, _r.Retriever[AgentDeps, Any]] - _default_retries: int - _system_prompt_functions: list[_system_prompt.SystemPromptRunner[AgentDeps]] - _deps_type: type[AgentDeps] - _max_result_retries: int - _current_result_retry: int - _override_deps: _utils.Option[AgentDeps] = None - _override_model: _utils.Option[models.Model] = None + _result_schema: _result.ResultSchema[ResultData] | None = field(repr=False) + _result_validators: list[_result.ResultValidator[AgentDeps, ResultData]] = field(repr=False) + _allow_text_result: bool = field(repr=False) + _system_prompts: tuple[str, ...] = field(repr=False) + _retrievers: dict[str, _r.Retriever[AgentDeps, Any]] = field(repr=False) + _default_retries: int = field(repr=False) + _system_prompt_functions: list[_system_prompt.SystemPromptRunner[AgentDeps]] = field(repr=False) + _deps_type: type[AgentDeps] = field(repr=False) + _max_result_retries: int = field(repr=False) + _current_result_retry: int = field(repr=False) + _override_deps: _utils.Option[AgentDeps] = field(default=None, repr=False) + _override_model: _utils.Option[models.Model] = field(default=None, repr=False) last_run_messages: list[_messages.Message] | None = None """The messages from the last run, useful when a run raised an exception. diff --git a/pydantic_ai_examples/weather_agent.py b/pydantic_ai_examples/weather_agent.py index 99001cd320..40eaceb139 100644 --- a/pydantic_ai_examples/weather_agent.py +++ b/pydantic_ai_examples/weather_agent.py @@ -132,7 +132,6 @@ async def get_weather(ctx: CallContext[Deps], lat: float, lng: float) -> dict[st async def main(): async with AsyncClient() as client: - logfire.instrument_httpx() # create a free API key at https://www.tomorrow.io/weather-api/ weather_api_key = os.getenv('WEATHER_API_KEY') # create a free API key at https://geocode.maps.co/ diff --git a/pyproject.toml b/pyproject.toml index d69f9d60a8..b8a5294ba6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ logfire = [ examples = [ "asyncpg>=0.30.0", "fastapi>=0.115.4", - "logfire[asyncpg,fastapi]>=2", + "logfire[asyncpg,fastapi]>=2.3", "python-multipart>=0.0.17", "rich>=13.9.2", "uvicorn>=0.32.0", @@ -74,6 +74,7 @@ lint = [ ] docs = [ "mkdocs", + "mkdocs-glightbox>=0.4.0", "mkdocs-material[imaging]", "mkdocstrings-python", ] diff --git a/uv.lock b/uv.lock index 755a4ad682..c9e70b32e7 100644 --- a/uv.lock +++ b/uv.lock @@ -756,7 +756,7 @@ wheels = [ [[package]] name = "logfire" -version = "2.1.2" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "executing" }, @@ -768,9 +768,9 @@ dependencies = [ { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/04/fcee652d49e49bae56effc186ce7866c31e0ac29f463d54e1aa28acfcc4e/logfire-2.1.2.tar.gz", hash = "sha256:b6a3caec6be514656eb664dff49b7b564faaf715f42f9bfc570fa68da8cba970", size = 245310 } +sdist = { url = "https://files.pythonhosted.org/packages/58/d6/0df3e4e4d5d98674cd058b9b48c2860157f50bce46f4d9648192bf2f31c6/logfire-2.3.0.tar.gz", hash = "sha256:2a29a7a31079201cbad494b865b27bf93b75df70dd752df0f121467a23709bbb", size = 250234 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/bf/dace0cb47b1679914df8c77dc4ec88acaa40610906492acc26e96756e41b/logfire-2.1.2-py3-none-any.whl", hash = "sha256:894ea34ec1d2c87d69b61607dd66d894e93b08003e626a82b2ad167912eb038b", size = 167376 }, + { url = "https://files.pythonhosted.org/packages/e6/c7/7de2f15842845aaf5d4d6e0b00469f574bbb332c42322dbc2ff2dc5c3466/logfire-2.3.0-py3-none-any.whl", hash = "sha256:812febd7cc29fdd7551d40161b7aa877a1bc8b4f74ef5bcfb0f0ca790f5184f4", size = 170621 }, ] [package.optional-dependencies] @@ -954,6 +954,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/d4/029f984e8d3f3b6b726bd33cafc473b75e9e44c0f7e80a5b29abc466bdea/mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134", size = 9521 }, ] +[[package]] +name = "mkdocs-glightbox" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/5a/0bc456397ba0acc684b5b1daa4ca232ed717938fd37198251d8bcc4053bf/mkdocs-glightbox-0.4.0.tar.gz", hash = "sha256:392b34207bf95991071a16d5f8916d1d2f2cd5d5bb59ae2997485ccd778c70d9", size = 32010 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/72/b0c2128bb569c732c11ae8e49a777089e77d83c05946062caa19b841e6fb/mkdocs_glightbox-0.4.0-py3-none-any.whl", hash = "sha256:e0107beee75d3eb7380ac06ea2d6eac94c999eaa49f8c3cbab0e7be2ac006ccf", size = 31154 }, +] + [[package]] name = "mkdocs-material" version = "9.5.44" @@ -1454,6 +1463,7 @@ dev = [ ] docs = [ { name = "mkdocs" }, + { name = "mkdocs-glightbox" }, { name = "mkdocs-material", extra = ["imaging"] }, { name = "mkdocstrings-python" }, ] @@ -1471,7 +1481,7 @@ requires-dist = [ { name = "griffe", specifier = ">=1.3.2" }, { name = "httpx", specifier = ">=0.27.2" }, { name = "logfire", marker = "extra == 'logfire'", specifier = ">=2" }, - { name = "logfire", extras = ["asyncpg", "fastapi"], marker = "extra == 'examples'", specifier = ">=2" }, + { name = "logfire", extras = ["asyncpg", "fastapi"], marker = "extra == 'examples'", specifier = ">=2.3" }, { name = "logfire-api", specifier = ">=1.2.0" }, { name = "openai", specifier = ">=1.54.3" }, { name = "pydantic", specifier = ">=2.10.0b1" }, @@ -1494,6 +1504,7 @@ dev = [ ] docs = [ { name = "mkdocs" }, + { name = "mkdocs-glightbox", specifier = ">=0.4.0" }, { name = "mkdocs-material", extras = ["imaging"] }, { name = "mkdocstrings-python" }, ]