diff --git a/README.md b/README.md index 3e165a6..53360cf 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Restate provides the following capabilities: - 🌍 **Deploy anywhere** – Whether it's AWS Lambda, CloudRun, Fly.io, Cloudflare, Kubernetes, Deno Deploy,... - ☁️ **Easy to self-host** – Single-binary self-hosted deployments or connect to [Restate Cloud](https://restate.dev/cloud/). -OpenAI Agent SDK invocation UI +OpenAI Agent SDK invocation UI Restate can also be used for other use cases, such as: [workflows](https://docs.restate.dev/use-cases/workflows), diff --git a/advanced/insurance-claims/app/workflow.py b/advanced/insurance-claims/app/workflow.py index 021dbd3..3f22db5 100644 --- a/advanced/insurance-claims/app/workflow.py +++ b/advanced/insurance-claims/app/workflow.py @@ -46,18 +46,16 @@ def parse_claim_data(extra_info: str = "") -> ClaimData: # Repetitively check for missing fields and request additional information if needed while True: - missing_fields = await ctx.run( - "completeness check", lambda last_claim=claim: check_missing_fields(last_claim)) + missing_fields = await ctx.run("completeness check", check_missing_fields, args=(claim,)) if not missing_fields: break id, promise = ctx.awakeable() - await ctx.run( - "Request missing info", lambda last_id=id: send_message_to_customer(missing_fields, last_id) - ) + await ctx.run("Request missing info", send_message_to_customer, args=(missing_fields, id)) extra_info = await promise - claim = await ctx.run("Extracting", lambda new_info=extra_info: parse_claim_data(new_info), type_hint=ClaimData) + + claim = await ctx.run("Extracting", parse_claim_data, args=(extra_info,)) # Create the claim in the legacy system - await ctx.run("create", lambda last_claim=claim: create_claim(last_claim)) + await ctx.run("create", lambda: create_claim(claim)) return claim diff --git a/agents/openai-agents-python/README.md b/agents/openai-agents-python/README.md index 0cc09e4..46810b3 100644 --- a/agents/openai-agents-python/README.md +++ b/agents/openai-agents-python/README.md @@ -4,7 +4,7 @@ **The agent composes the workflow on the fly, and Restate persists the execution as it takes place.** -Combining Restate and an Agent SDK is ideal for turning brittle agent implementations into resilient ones. +Combining Restate and the OpenAI Agent SDK is ideal for turning the default brittle agents into resilient ones. Restate powers your agents with the following features: - 🛡️ **Automatic retries**: Built-in retry mechanisms for failed operations @@ -16,48 +16,30 @@ Restate powers your agents with the following features: - 🙂 **Resilient human-in-the-loop**: Both approaches support human intervention in workflows - 👬 **Idempotency/deduplication**: Prevents duplicate agent requests -## Plugging Restate into existing Agent SDKs -Combine Restate's durability with existing Agent SDKs for rapid development. +As opposed to the [OpenAI Agents + Restate template](../../get-started/openai-agents-python/README.md), this example shows how to do handoffs and stateful sessions. -To make the agent resilient, we need to: -- persist the results of LLM calls in Restate's journal by wrapping them in `ctx.run()` -- have the context available to us in the tools so that we can use it to persist the intermediate tool execution steps. - -The details of how to do this depend on the Agent SDK you are using. - -⚠ **LIMITATIONS**: You cannot do parallel tool calls or any type of parallel execution if you integrate Restate with an Agent SDK. -If you execute actions on the context in different tools in parallel, Restate will not be able to deterministically replay them because the order might be different during recovery and will crash. -We are working on a solution to this, but for now, you can only use Restate with Agent SDKs for sequential tool calls. - -### Restate + OpenAI Agent SDK -[](openai_sdk/agent.py) +## Plugging Restate into the OpenAI Agents Python SDK Use the OpenAI Agent SDK to implement the agent loop, while Restate handles the persistence and resiliency of the agent's decisions and tool executions. -The OpenAI Agent SDK lets you wrap the LLM calls into durable actions by implementing a Restate Model Provider ([code](openai_sdk/middleware.py)). -In order to have access to the Restate context in the tools, we can pass it along in the context that we pass to the tools. - -The example is a customer service agent for an airline that can send invoices and update seat bookings. -This is [an OpenAI SDK example](https://github.com/openai/openai-agents-python/blob/main/examples/customer_service/main.py) that has been adapted to use Restate for resiliency and workflow guarantees: +To make the agent resilient, we need to: +- persist the results of LLM calls in Restate's journal by wrapping them in `ctx.run()`. This is handled by the [`DurableModelCalls` model provider](utils/middleware.py). +- persist intermediate tool execution steps by wrapping steps in Restate SDK actions. To do this, we pass the Restate context along to the tools. Using Agent SDK Using Agent SDK - journal Using Agent SDK - state -### Other Agent SDKs -Are you using another Agent SDK? We can help you evaluate whether it can be integrated with Restate. -Join our [Discord](https://discord.gg/skW3AZ6uGd) or [Slack](https://join.slack.com/t/restatecommunity/shared_invite/zt-2v9gl005c-WBpr167o5XJZI1l7HWKImA) to discuss. -## Running the examples +⚠ **LIMITATIONS**: You cannot do parallel tool calls or any type of parallel execution if you integrate Restate with an Agent SDK. +If you execute actions on the context in different tools in parallel, Restate will not be able to deterministically replay them because the order might be different during recovery and will crash. +We are working on a solution to this, but for now, you can only use Restate with Agent SDKs for sequential tool calls. -### Restate + OpenAI Agent SDK -[](openai_sdk/agent.py) +## Running the example This example implements an airline customer service agent that can answer questions about your flights, and change your seat. -The example uses the OpenAI Agent SDK to implement the agent. Although this could be adapted to other agent SDKs. - 1. Export your OpenAI or Anthrophic API key as an environment variable: ```shell export OPENAI_API_KEY=your_openai_api_key @@ -68,7 +50,6 @@ The example uses the OpenAI Agent SDK to implement the agent. Although this coul ``` 3. Start the services: ```shell - cd openai_sdk uv run . ``` 4. Register the services: @@ -85,22 +66,24 @@ Or with the [client](client/__main__.py): - **Request**: ```shell - uv run client "can you send me an invoice for booking AB4568?" + uv run client peter "can you send me an invoice for booking AB4568?" ``` + + With `peter` as the conversation ID. Example response: `I've sent the invoice to your email associated with confirmation number AB4568. If there's anything else you need, feel free to ask!.` - **Or have longer conversations**: ```shell - uv run client "can you change my seat to 10b?" + uv run client peter "can you change my seat to 10b?" ``` Example response: `To change your seat to 10B, I'll need your confirmation number. Could you please provide that?` Respond to the question by sending a new message to the same stateful session: ```shell - uv run client "5666" + uv run client peter "5666" ``` Example response: `Your seat has been successfully changed to 5B. If there's anything else you need, feel free to ask!` diff --git a/agents/openai-agents-python/agent.py b/agents/openai-agents-python/agent.py index 72948fa..07ba0ce 100644 --- a/agents/openai-agents-python/agent.py +++ b/agents/openai-agents-python/agent.py @@ -1,10 +1,9 @@ import agents import restate -from pydantic import BaseModel, ConfigDict from agents import Agent, function_tool, handoff, RunContextWrapper, RunConfig from agents.extensions.handoff_prompt import RECOMMENDED_PROMPT_PREFIX -from utils.middleware import RestateModelProvider +from utils.middleware import DurableModelCalls from utils.utils import ( retrieve_flight_info, send_invoice, @@ -19,21 +18,11 @@ https://github.com/openai/openai-agents-python/blob/main/examples/customer_service/main.py """ - -# To have access to the Restate context in the tools, we can pass it along in the context that we pass to the tools -class ToolContext(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - - restate_context: restate.ObjectContext - customer_id: str | None = None - - # TOOLS - @function_tool async def update_seat( - context: RunContextWrapper[ToolContext], + wrapper: RunContextWrapper[restate.ObjectContext], confirmation_number: str, new_seat: str, ) -> str: @@ -46,15 +35,15 @@ async def update_seat( """ # Do durable steps in your tools by using the Restate context - ctx = context.context.restate_context + restate_context = wrapper.context # 1. Look up the flight using the confirmation number - flight = await ctx.run( + flight = await restate_context.run( "Info lookup", retrieve_flight_info, args=(confirmation_number,) ) # 2. Update the seat in the booking system - success = await ctx.run( + success = await restate_context.run( "Update seat", update_seat_in_booking_system, args=(confirmation_number, new_seat, flight), @@ -70,24 +59,24 @@ async def update_seat( description_override="Sends invoices to customers for booked flights.", ) async def invoice_sending( - context: RunContextWrapper[ToolContext], confirmation_number: str + wrapper: RunContextWrapper[restate.ObjectContext], confirmation_number: str ): # Do durable steps in your tools by using the Restate context - ctx = context.context.restate_context + restate_context = wrapper.context # 1. Look up the flight using the confirmation number - flight = await ctx.run( + flight = await restate_context.run( "Info lookup", retrieve_flight_info, args=(confirmation_number,) ) # 2. Send the invoice to the customer - await ctx.run("Send invoice", send_invoice, args=(confirmation_number, flight)) + await restate_context.run("Send invoice", send_invoice, args=(confirmation_number, flight)) ### AGENTS # This is identical to the examples of the OpenAI SDK -faq_agent = Agent[ToolContext]( +faq_agent = Agent[restate.ObjectContext]( name="Invoice Sending Agent", handoff_description="A helpful agent that can send invoices.", instructions=f"""{RECOMMENDED_PROMPT_PREFIX} @@ -100,7 +89,7 @@ async def invoice_sending( tools=[invoice_sending], ) -seat_booking_agent = Agent[ToolContext]( +seat_booking_agent = Agent[restate.ObjectContext]( name="Seat Booking Agent", handoff_description="A helpful agent that can update a seat on a flight.", instructions=f"""{RECOMMENDED_PROMPT_PREFIX} @@ -114,8 +103,7 @@ async def invoice_sending( tools=[update_seat], ) -triage_agent = Agent[ToolContext]( - model="o3-mini", +triage_agent = Agent[restate.ObjectContext]( name="Triage Agent", handoff_description="A triage agent that can delegate a customer's request to the appropriate agent.", instructions=( @@ -140,43 +128,27 @@ async def invoice_sending( # Keyed by conversation id agent = restate.VirtualObject("Agent") -# Keys of the K/V state stored in Restate per chat -INPUT_ITEMS = "input-items" - @agent.handler() -async def run(ctx: restate.ObjectContext, req: str) -> str: - """ - Send a message to the agent. +async def run(restate_context: restate.ObjectContext, message: str) -> str: + """Send a message to the agent.""" - Args: - req (str): The message to send to the agent. - - Returns: - str: The response from the agent. - """ # Use Restate's K/V store to keep track of the conversation history and last agent - input_items = await ctx.get(INPUT_ITEMS) or [] - input_items.append({"role": "user", "content": req}) - ctx.set(INPUT_ITEMS, input_items) - - last_agent_name = await ctx.get("agent") or triage_agent.name - last_agent = agent_dict[last_agent_name] - - # Pass the Restate context to the tools - tool_context = ToolContext(restate_context=ctx, customer_id=ctx.key()) + memory = await restate_context.get("memory") or [] + memory.append({"role": "user", "content": message}) + restate_context.set("memory", memory) + last_agent_name = await restate_context.get("agent") or triage_agent.name result = await agents.Runner.run( - last_agent, - input=input_items, - context=tool_context, - # Use the RestateModelProvider to persist the LLM calls in Restate - run_config=RunConfig(model_provider=RestateModelProvider(ctx)), + agent_dict[last_agent_name], + input=memory, + # Pass the Restate context to the tools to make tool execution steps durable + context=restate_context, + # Choose any model and let Restate persist your calls + run_config=RunConfig(model="gpt-4o", model_provider=DurableModelCalls(restate_context)), ) - ctx.set("agent", result.last_agent.name) - - output = str(result.final_output) - input_items.append({"role": "system", "content": output}) - ctx.set(INPUT_ITEMS, input_items) - return output + restate_context.set("agent", result.last_agent.name) + memory.append({"role": "assistant", "content": result.final_output}) + restate_context.set("memory", memory) + return result.final_output diff --git a/agents/openai-agents-python/client/__main__.py b/agents/openai-agents-python/client/__main__.py index df8453c..9ca76e3 100644 --- a/agents/openai-agents-python/client/__main__.py +++ b/agents/openai-agents-python/client/__main__.py @@ -6,10 +6,11 @@ def main(): if len(sys.argv) == 0: raise ValueError("No input provided") - data = sys.argv[1] + key = sys.argv[1] + data = sys.argv[2] r = httpx.post( - "http://localhost:8080/Agent/my-user/run", + f"http://localhost:8080/Agent/{key}/run", json=data, timeout=60, ) diff --git a/agents/openai-agents-python/utils/middleware.py b/agents/openai-agents-python/utils/middleware.py index 64fac7e..8fea88d 100644 --- a/agents/openai-agents-python/utils/middleware.py +++ b/agents/openai-agents-python/utils/middleware.py @@ -37,11 +37,9 @@ def to_input_items(self) -> list[TResponseInputItem]: return [it.model_dump(exclude_unset=True) for it in self.output] # type: ignore -class RestateModelProvider(MultiProvider): +class DurableModelCalls(MultiProvider): """ A Restate model provider that wraps the OpenAI SDK's default MultiProvider. - It let - to return a Restate persist LLM calls in the Restate journal. """ def __init__(self, ctx: restate.Context): @@ -90,7 +88,7 @@ async def call_llm() -> RestateModelResponse: response_id=resp.response_id, ) - return await self.ctx.run("call LLM", call_llm) + return await self.ctx.run("call LLM", call_llm, max_attempts=3) def stream_response( self, diff --git a/diy-patterns/README.md b/diy-patterns/README.md index 76be75f..959cbf0 100644 --- a/diy-patterns/README.md +++ b/diy-patterns/README.md @@ -71,6 +71,9 @@ Send an HTTP request to the service by running the [client](chaining/client.py): uv run chaining_client ``` +You see in the UI how the LLM is called multiple times, and how the results are refined step by step: + +Chaining LLM calls - UI
View output @@ -137,6 +140,12 @@ Send an HTTP request to the service by running the [client](parallelization/clie uv run parallelization_client ``` +You see in the UI how the different tasks are executed in parallel: + +Chaining LLM calls - UI + +Once all tasks are done, the results are aggregated and returned to the client. +
View output @@ -376,6 +385,10 @@ Send an HTTP request to the service by running the [client](routing/client.py): uv run routing_client ``` +In the UI, you can see how the LLM decides to forward the request to the technical support team, and how the response is processed: + +Dynamic routing based on LLM output - UI +
View Output @@ -514,6 +527,8 @@ Send an HTTP request to the service by running the [client](orchestrator_workers uv run orchestrator_client ``` +Orchestrator-worker pattern - UI +
View output @@ -646,6 +661,8 @@ Send an HTTP request to the service by running the [client](evaluator_optimizer/ uv run evaluator_client ``` +Evaluator-optimizer pattern - UI +
View Output @@ -757,6 +774,12 @@ curl localhost:8080/HumanInTheLoopService/giselle/run_with_promise \ --json '"Write a poem about Durable Execution"' ``` +Then use the printed curl command to incorporate external feedback. And supply `PASS` as feedback to accept the solution. + +You can see how the feedback gets incorporated in the Invocations tab in the Restate UI (`http://localhost:9070`): + +Human-in-the-loop pattern - UI +
View Output @@ -807,11 +830,6 @@ Answer 'PASS' to accept the solution.
-Then use the printed curl command to incorporate external feedback. And supply `PASS` as feedback to accept the solution. - -You can see how the feedback gets incorporated in the Invocations tab in the Restate UI (`http://localhost:9070`): - -Human-in-the-loop #### Option 2: `run` handler [](human_in_the_loop/service.py) diff --git a/diy-patterns/chaining/service.py b/diy-patterns/chaining/service.py index 5ae38db..b559c2f 100644 --- a/diy-patterns/chaining/service.py +++ b/diy-patterns/chaining/service.py @@ -26,9 +26,8 @@ class ChainRequest(BaseModel): async def chain_call(ctx: restate.Context, req: ChainRequest) -> str: result = req.input for i, prompt in enumerate(req.prompts, 1): - print(f"\nStep {i}:") result = await ctx.run( - f"LLM call ${i}", lambda: llm_call(f"{prompt}\nInput: {result}") + f"LLM call {i}", lambda: llm_call(f"{prompt}\nInput: {result}") ) print(result) return result diff --git a/diy-patterns/img/chaining.png b/diy-patterns/img/chaining.png new file mode 100644 index 0000000..0947ff3 Binary files /dev/null and b/diy-patterns/img/chaining.png differ diff --git a/diy-patterns/img/evaluator.png b/diy-patterns/img/evaluator.png new file mode 100644 index 0000000..bbe799b Binary files /dev/null and b/diy-patterns/img/evaluator.png differ diff --git a/diy-patterns/img/human_in_the_loop.png b/diy-patterns/img/human_in_the_loop.png new file mode 100644 index 0000000..c3e4cc3 Binary files /dev/null and b/diy-patterns/img/human_in_the_loop.png differ diff --git a/diy-patterns/img/orchestrator.png b/diy-patterns/img/orchestrator.png new file mode 100644 index 0000000..c3f523e Binary files /dev/null and b/diy-patterns/img/orchestrator.png differ diff --git a/diy-patterns/img/parallel.png b/diy-patterns/img/parallel.png new file mode 100644 index 0000000..405ad77 Binary files /dev/null and b/diy-patterns/img/parallel.png differ diff --git a/diy-patterns/img/routing.png b/diy-patterns/img/routing.png new file mode 100644 index 0000000..b6bb1f9 Binary files /dev/null and b/diy-patterns/img/routing.png differ diff --git a/diy-patterns/pyproject.toml b/diy-patterns/pyproject.toml index 32e0da1..677eebf 100644 --- a/diy-patterns/pyproject.toml +++ b/diy-patterns/pyproject.toml @@ -23,7 +23,7 @@ orchestrator_client = "orchestrator_workers.client:main" evaluator_client = "evaluator_optimizer.client:main" [tool.hatch.build.targets.wheel] -packages = ["chaining", "evaluator_optimizer", "human_evaluator_optimizer", "orchestrator_workers", "parallelization", "routing"] +packages = ["chaining", "evaluator_optimizer", "human_evaluator_optimizer", "orchestrator_workers", "parallelization", "routing", "util"] [build-system] requires = ["hatchling"] diff --git a/get-started/openai-agents-python/agent.py b/get-started/openai-agents-python/agent.py index 0c5b470..63ca634 100644 --- a/get-started/openai-agents-python/agent.py +++ b/get-started/openai-agents-python/agent.py @@ -1,17 +1,10 @@ import restate from agents import Agent, RunConfig, Runner, function_tool, RunContextWrapper -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel -from utils.middleware import RestateModelProvider -from utils.utils import fetch_weather, parse_weather_data - - -# Pass the Restate context to the tools to journal tool execution steps -class ToolContext(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - restate_context: restate.Context - # you can add more fields here to pass to your tools, e.g. customer_id, ... +from utils.middleware import DurableModelCalls +from utils.utils import fetch_weather, parse_weather_data, WeatherResponse class WeatherRequest(BaseModel): @@ -21,23 +14,17 @@ class WeatherRequest(BaseModel): @function_tool async def get_weather( - context: RunContextWrapper[ToolContext], req: WeatherRequest -) -> str: + wrapper: RunContextWrapper[restate.Context], req: WeatherRequest +) -> WeatherResponse: """Get the current weather for a given city.""" # Do durable steps using the Restate context - restate_ctx = context.context.restate_context - - response = await restate_ctx.run("Get weather", fetch_weather, args=(req.city,)) - if response.startswith("Unknown location"): - return f"Unknown location: {req.city}. Please provide a valid city name." - - weather = await parse_weather_data(response) - return ( - f"Weather in {req.city}: {weather["temperature"]}°C, {weather['description']}" - ) + restate_context = wrapper.context + resp = await restate_context.run( + "Get weather", fetch_weather, args=(req.city,)) + return await parse_weather_data(resp) -my_agent = Agent[ToolContext]( +my_agent = Agent[restate.Context]( name="Helpful Agent", handoff_description="A helpful agent.", instructions="You are a helpful agent.", @@ -50,14 +37,18 @@ async def get_weather( @agent.handler() -async def run(ctx: restate.Context, message: str) -> str: +async def run(restate_context: restate.Context, message: str) -> str: result = await Runner.run( my_agent, input=message, # Pass the Restate context to tools to make tool execution steps durable - context=ToolContext(restate_context=ctx), - # Use the RestateModelProvider to persist the LLM calls in Restate - run_config=RunConfig(model_provider=RestateModelProvider(ctx)), + context=restate_context, + # Choose any model and let Restate persist your calls + run_config=RunConfig( + model="gpt-4o", + model_provider=DurableModelCalls(restate_context) + ), ) + return result.final_output diff --git a/get-started/openai-agents-python/utils/middleware.py b/get-started/openai-agents-python/utils/middleware.py index 64fac7e..8fea88d 100644 --- a/get-started/openai-agents-python/utils/middleware.py +++ b/get-started/openai-agents-python/utils/middleware.py @@ -37,11 +37,9 @@ def to_input_items(self) -> list[TResponseInputItem]: return [it.model_dump(exclude_unset=True) for it in self.output] # type: ignore -class RestateModelProvider(MultiProvider): +class DurableModelCalls(MultiProvider): """ A Restate model provider that wraps the OpenAI SDK's default MultiProvider. - It let - to return a Restate persist LLM calls in the Restate journal. """ def __init__(self, ctx: restate.Context): @@ -90,7 +88,7 @@ async def call_llm() -> RestateModelResponse: response_id=resp.response_id, ) - return await self.ctx.run("call LLM", call_llm) + return await self.ctx.run("call LLM", call_llm, max_attempts=3) def stream_response( self, diff --git a/get-started/openai-agents-python/utils/utils.py b/get-started/openai-agents-python/utils/utils.py index 2e7e00a..b8ccfce 100644 --- a/get-started/openai-agents-python/utils/utils.py +++ b/get-started/openai-agents-python/utils/utils.py @@ -1,18 +1,47 @@ +import os + import httpx +import restate + +from pydantic import BaseModel + + +class WeatherResponse(BaseModel): + """Request to get the weather for a city.""" + + temperature: float + description: str + + +async def fetch_weather(city: str) -> dict: + # This is a simulated failure to demo Durable Execution retries. + if os.getenv("WEATHER_API_FAIL") == "true": + print(f"[👻 SIMULATED] Weather API down...") + raise Exception(f"[👻 SIMULATED] Weather API down") + try: + resp = httpx.get(f"https://wttr.in/{httpx.URL(city)}?format=j1", timeout=10.0) + resp.raise_for_status() -async def fetch_weather(city: str) -> str: - resp = httpx.get(f"https://wttr.in/{city}?format=j1", timeout=10.0) - resp.raise_for_status() - return resp.text + if resp.text.startswith("Unknown location"): + raise restate.TerminalError( + f"Unknown location: {city}. Please provide a valid city name." + ) + return resp.json() + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + raise restate.TerminalError( + f"City not found: {city}. Please provide a valid city name." + ) from e + else: + raise Exception(f"HTTP error occurred: {e}") from e -async def parse_weather_data(weather_data: str) -> dict: - import json - weather_json = json.loads(weather_data) - current = weather_json["current_condition"][0] - return { - "temperature": current["temp_C"], - "description": current["weatherDesc"][0]["value"], - } +async def parse_weather_data(weather_data: dict) -> WeatherResponse: + # weather_json = json.loads(weather_data) + current = weather_data["current_condition"][0] + return WeatherResponse( + temperature=float(current["temp_C"]), + description=current["weatherDesc"][0]["value"], + ) diff --git a/get-started/vercel-ai/src/app.ts b/get-started/vercel-ai/src/app.ts index 82b4957..429ad03 100644 --- a/get-started/vercel-ai/src/app.ts +++ b/get-started/vercel-ai/src/app.ts @@ -8,16 +8,18 @@ import { durableCalls } from "./utils/ai_infra"; import { fetchWeather, parseWeatherResponse } from "./utils/utils"; // Durable tool workflow -const getWeather = async (ctx: restate.Context, city: string) => { - // implement durable tool steps using the Restate context - const result = await ctx.run("get weather", async () => fetchWeather(city)); - if (result.startsWith("Unknown location")) { - return `Unknown location: ${city}. Please provide a valid city name.`; - } - - const { temperature, description } = await parseWeatherResponse(result); - return `Weather in ${city}: ${temperature}°C, ${description}`; -}; +const getWeatherTool = (restate_context: restate.Context) => + tool({ + description: "Get the current weather for a given city.", + parameters: z.object({ city: z.string() }), + execute: async ({ city }) => { + // implement durable tool steps using the Restate context + const result = await restate_context.run("get weather", async () => + fetchWeather(city), + ); + return await parseWeatherResponse(result); + }, + }); const agent = restate.service({ name: "Agent", @@ -25,7 +27,6 @@ const agent = restate.service({ run: restate.handlers.handler( { input: serde.zod(z.string()) }, async (ctx: restate.Context, prompt) => { - // Persist the results of LLM calls via the durableCalls middleware const model = wrapLanguageModel({ model: openai("gpt-4o-2024-08-06", { structuredOutputs: true }), @@ -36,13 +37,7 @@ const agent = restate.service({ model, system: "You are a helpful agent.", messages: [{ role: "user", content: prompt }], - tools: { - getWeatherTool: tool({ - description: "Get the current weather for a given city.", - parameters: z.object({ city: z.string() }), - execute: async ({ city }) => getWeather(ctx, city), - }), - }, + tools: { getWeather: getWeatherTool(ctx) }, maxRetries: 0, maxSteps: 10, }); diff --git a/get-started/vercel-ai/src/utils/utils.ts b/get-started/vercel-ai/src/utils/utils.ts index 6f2feba..4130e59 100644 --- a/get-started/vercel-ai/src/utils/utils.ts +++ b/get-started/vercel-ai/src/utils/utils.ts @@ -1,3 +1,7 @@ +import { TerminalError } from "@restatedev/restate-sdk"; +import * as os from "node:os"; +import { parseEnv } from "node:util"; + type WeatherResponse = { current_condition: { temp_C: string; @@ -6,10 +10,22 @@ type WeatherResponse = { }; export async function fetchWeather(city: string): Promise { + // This is a simulated failure to demo Durable Execution retries. + if (process.env.WEATHER_API_FAIL === "true") { + console.error(`[👻 SIMULATED] Weather API down...`); + throw new Error(`[👻 SIMULATED] Weather API down...`); + } + const url = `https://wttr.in/${encodeURIComponent(city)}?format=j1`; const res = await fetch(url); if (!res.ok) throw new Error(`Failed calling weather API: ${res.status}`); - return res.text(); + const output = await res.text(); + if (output.startsWith("Unknown location")) { + throw new TerminalError( + `Unknown location: ${city}. Please provide a valid city name.`, + ); + } + return output; } export async function parseWeatherResponse( @@ -17,8 +33,9 @@ export async function parseWeatherResponse( ): Promise<{ temperature: string; description: string }> { const data = JSON.parse(response) as WeatherResponse; const current = data.current_condition[0]; + return { temperature: current.temp_C, description: current.weatherDesc[0].value, }; -} +} \ No newline at end of file