diff --git a/docs/docs/modules/agents/agent_types/openai_functions_agent.mdx b/docs/docs/modules/agents/agent_types/openai_functions_agent.mdx index 69f6eec9917..6d59911d0c5 100644 --- a/docs/docs/modules/agents/agent_types/openai_functions_agent.mdx +++ b/docs/docs/modules/agents/agent_types/openai_functions_agent.mdx @@ -5,7 +5,7 @@ sidebar_position: 0 # OpenAI functions -Certain OpenAI models (like `gpt-3.5-turbo` and `gpt-4`) have been fine-tuned to detect when a function should to be called and respond with the inputs that should be passed to the function. +Certain OpenAI models (like `gpt-3.5-turbo` and `gpt-4`) have been fine-tuned to detect when a function should be called and respond with the inputs that should be passed to the function. In an API call, you can describe functions and have the model intelligently choose to output a JSON object containing arguments to call those functions. The goal of the OpenAI Function APIs is to more reliably return valid and useful function calls than a generic text completion or chat API. @@ -22,7 +22,7 @@ Must be used with an [OpenAI Functions](https://platform.openai.com/docs/guides/ # With LCEL -In this example we'll use LCEL to construct a highly customizable agent that is given two tools: search and calculator. +In this example we'll use LCEL to construct a customizable agent that is given two tools: search and calculator. We'll then pull in a prompt template from the [LangChainHub](https://smith.langchain.com/hub) and pass that to our runnable agent. Lastly we'll use the default OpenAI functions output parser `OpenAIFunctionsAgentOutputParser`. This output parser contains a method `parseAIMessage` which when provided with a message, either returns an instance of `FunctionsAgentAction` if there is another action to be taken my the agent, or `AgentFinish` if the agent has completed its objective. diff --git a/docs/docs/modules/agents/agent_types/openai_tools_agent.mdx b/docs/docs/modules/agents/agent_types/openai_tools_agent.mdx new file mode 100644 index 00000000000..f7a9cff7186 --- /dev/null +++ b/docs/docs/modules/agents/agent_types/openai_tools_agent.mdx @@ -0,0 +1,131 @@ +--- +hide_table_of_contents: true +sidebar_position: 1 +--- + +# OpenAI tool calling + +:::tip Compatibility +Tool calling is new and only available on [OpenAI's latest models](https://platform.openai.com/docs/guides/function-calling). +::: + +OpenAI's latest `gpt-3.5-turbo-1106` and `gpt-4-1106-preview` models have been fine-tuned to detect when one or more tools should be called to gather sufficient information +to answer the initial query, and respond with the inputs that should be passed to those tools. + +While the goal of more reliably returning valid and useful function calls is the same as the functions agent, the ability to return multiple tools at once results in +both fewer roundtrips for complex questions. + +The OpenAI Tools Agent is designed to work with these models. + +import CodeBlock from "@theme/CodeBlock"; +import RunnableExample from "@examples/agents/openai_tools_runnable.ts"; + +# Usage + +In this example we'll use LCEL to construct a customizable agent with a mocked weather tool and a calculator. + +The basic flow is this: + +1. Define the tools the agent will be able to call. You can use [OpenAI's tool syntax](https://platform.openai.com/docs/guides/function-calling), or LangChain tool instances as shown below. +2. Initialize our model and bind those tools as arguments. +3. Define a function that formats any previous agent steps as messages. The agent will pass those back to OpenAI for the next agent iteration. +4. Create a `RunnableSequence` that will act as the agent. We use a specialized output parser to extract any tool calls from the model's output. +5. Initialize an `AgentExecutor` with the agent and the tools to execute the agent on a loop. +6. Run the `AgentExecutor` and see the output. + +Here's how it looks: + +{RunnableExample} + +You can check out this example trace for an inspectable view of the steps taken to answer the question: https://smith.langchain.com/public/2bbffb7d-4f9d-47ad-90be-09910e5b4b34/r + +## Adding memory + +We can also use memory to save our previous agent input/outputs, and pass it through to each agent iteration. +Using memory can help give the agent better context on past interactions, which can lead to more accurate responses beyond what the `agent_scratchpad` can do. + +Adding memory only requires a few changes to the above example. + +First, import and instantiate your memory class, in this example we'll use `BufferMemory`. + +```typescript +import { BufferMemory } from "langchain/memory"; +``` + +```typescript +const memory = new BufferMemory({ + memoryKey: "history", // The object key to store the memory under + inputKey: "question", // The object key for the input + outputKey: "answer", // The object key for the output + returnMessages: true, +}); +``` + +Then, update your prompt to include another `MessagesPlaceholder`. This time we'll be passing in the `chat_history` variable from memory. + +```typescript +const prompt = ChatPromptTemplate.fromMessages([ + ["ai", "You are a helpful assistant"], + ["human", "{input}"], + new MessagesPlaceholder("agent_scratchpad"), + new MessagesPlaceholder("chat_history"), +]); +``` + +Next, inside your `RunnableSequence` add a field for loading the `chat_history` from memory. + +```typescript +const runnableAgent = RunnableSequence.from([ + { + input: (i: { input: string; steps: AgentStep[] }) => i.input, + agent_scratchpad: (i: { input: string; steps: AgentStep[] }) => + formatAgentSteps(i.steps), + // Load memory here + chat_history: async (_: { input: string; steps: AgentStep[] }) => { + const { history } = await memory.loadMemoryVariables({}); + return history; + }, + }, + prompt, + modelWithTools, + new OpenAIFunctionsAgentOutputParser(), +]); +``` + +Finally we can call the agent, and save the output after the response is returned. + +```typescript +const query = "What is the weather in New York?"; +console.log(`Calling agent executor with query: ${query}`); +const result = await executor.call({ + input: query, +}); +console.log(result); +/* +Calling agent executor with query: What is the weather in New York? +{ + output: 'The current weather in New York is sunny with a temperature of 66 degrees Fahrenheit. The humidity is at 54% and the wind is blowing at 6 mph. There is 0% chance of precipitation.' +} +*/ + +// Save the result and initial input to memory +await memory.saveContext( + { + question: query, + }, + { + answer: result.output, + } +); + +const query2 = "Do I need a jacket?"; +const result2 = await executor.call({ + input: query2, +}); +console.log(result2); +/* +{ + output: 'Based on the current weather in New York, you may not need a jacket. However, if you feel cold easily or will be outside for a long time, you might want to bring a light jacket just in case.' +} + */ +``` diff --git a/environment_tests/test-exports-bun/src/entrypoints.js b/environment_tests/test-exports-bun/src/entrypoints.js index 5404725a933..f376fb57b6b 100644 --- a/environment_tests/test-exports-bun/src/entrypoints.js +++ b/environment_tests/test-exports-bun/src/entrypoints.js @@ -3,6 +3,7 @@ export * from "langchain/load/serializable"; export * from "langchain/agents"; export * from "langchain/agents/toolkits"; export * from "langchain/agents/format_scratchpad"; +export * from "langchain/agents/format_scratchpad/openai_tools"; export * from "langchain/agents/format_scratchpad/log"; export * from "langchain/agents/format_scratchpad/xml"; export * from "langchain/agents/format_scratchpad/log_to_message"; diff --git a/environment_tests/test-exports-cf/src/entrypoints.js b/environment_tests/test-exports-cf/src/entrypoints.js index 5404725a933..f376fb57b6b 100644 --- a/environment_tests/test-exports-cf/src/entrypoints.js +++ b/environment_tests/test-exports-cf/src/entrypoints.js @@ -3,6 +3,7 @@ export * from "langchain/load/serializable"; export * from "langchain/agents"; export * from "langchain/agents/toolkits"; export * from "langchain/agents/format_scratchpad"; +export * from "langchain/agents/format_scratchpad/openai_tools"; export * from "langchain/agents/format_scratchpad/log"; export * from "langchain/agents/format_scratchpad/xml"; export * from "langchain/agents/format_scratchpad/log_to_message"; diff --git a/environment_tests/test-exports-cjs/src/entrypoints.js b/environment_tests/test-exports-cjs/src/entrypoints.js index 447aca01fe7..26d669bfaae 100644 --- a/environment_tests/test-exports-cjs/src/entrypoints.js +++ b/environment_tests/test-exports-cjs/src/entrypoints.js @@ -3,6 +3,7 @@ const load_serializable = require("langchain/load/serializable"); const agents = require("langchain/agents"); const agents_toolkits = require("langchain/agents/toolkits"); const agents_format_scratchpad = require("langchain/agents/format_scratchpad"); +const agents_format_scratchpad_openai_tools = require("langchain/agents/format_scratchpad/openai_tools"); const agents_format_scratchpad_log = require("langchain/agents/format_scratchpad/log"); const agents_format_scratchpad_xml = require("langchain/agents/format_scratchpad/xml"); const agents_format_scratchpad_log_to_message = require("langchain/agents/format_scratchpad/log_to_message"); diff --git a/environment_tests/test-exports-esbuild/src/entrypoints.js b/environment_tests/test-exports-esbuild/src/entrypoints.js index b6503d4a974..5fc692c45ab 100644 --- a/environment_tests/test-exports-esbuild/src/entrypoints.js +++ b/environment_tests/test-exports-esbuild/src/entrypoints.js @@ -3,6 +3,7 @@ import * as load_serializable from "langchain/load/serializable"; import * as agents from "langchain/agents"; import * as agents_toolkits from "langchain/agents/toolkits"; import * as agents_format_scratchpad from "langchain/agents/format_scratchpad"; +import * as agents_format_scratchpad_openai_tools from "langchain/agents/format_scratchpad/openai_tools"; import * as agents_format_scratchpad_log from "langchain/agents/format_scratchpad/log"; import * as agents_format_scratchpad_xml from "langchain/agents/format_scratchpad/xml"; import * as agents_format_scratchpad_log_to_message from "langchain/agents/format_scratchpad/log_to_message"; diff --git a/environment_tests/test-exports-esm/src/entrypoints.js b/environment_tests/test-exports-esm/src/entrypoints.js index b6503d4a974..5fc692c45ab 100644 --- a/environment_tests/test-exports-esm/src/entrypoints.js +++ b/environment_tests/test-exports-esm/src/entrypoints.js @@ -3,6 +3,7 @@ import * as load_serializable from "langchain/load/serializable"; import * as agents from "langchain/agents"; import * as agents_toolkits from "langchain/agents/toolkits"; import * as agents_format_scratchpad from "langchain/agents/format_scratchpad"; +import * as agents_format_scratchpad_openai_tools from "langchain/agents/format_scratchpad/openai_tools"; import * as agents_format_scratchpad_log from "langchain/agents/format_scratchpad/log"; import * as agents_format_scratchpad_xml from "langchain/agents/format_scratchpad/xml"; import * as agents_format_scratchpad_log_to_message from "langchain/agents/format_scratchpad/log_to_message"; diff --git a/environment_tests/test-exports-vercel/src/entrypoints.js b/environment_tests/test-exports-vercel/src/entrypoints.js index 5404725a933..f376fb57b6b 100644 --- a/environment_tests/test-exports-vercel/src/entrypoints.js +++ b/environment_tests/test-exports-vercel/src/entrypoints.js @@ -3,6 +3,7 @@ export * from "langchain/load/serializable"; export * from "langchain/agents"; export * from "langchain/agents/toolkits"; export * from "langchain/agents/format_scratchpad"; +export * from "langchain/agents/format_scratchpad/openai_tools"; export * from "langchain/agents/format_scratchpad/log"; export * from "langchain/agents/format_scratchpad/xml"; export * from "langchain/agents/format_scratchpad/log_to_message"; diff --git a/environment_tests/test-exports-vite/src/entrypoints.js b/environment_tests/test-exports-vite/src/entrypoints.js index 5404725a933..f376fb57b6b 100644 --- a/environment_tests/test-exports-vite/src/entrypoints.js +++ b/environment_tests/test-exports-vite/src/entrypoints.js @@ -3,6 +3,7 @@ export * from "langchain/load/serializable"; export * from "langchain/agents"; export * from "langchain/agents/toolkits"; export * from "langchain/agents/format_scratchpad"; +export * from "langchain/agents/format_scratchpad/openai_tools"; export * from "langchain/agents/format_scratchpad/log"; export * from "langchain/agents/format_scratchpad/xml"; export * from "langchain/agents/format_scratchpad/log_to_message"; diff --git a/examples/src/agents/openai_runnable.ts b/examples/src/agents/openai_runnable.ts index 0a54c892d09..8cc0f22a619 100644 --- a/examples/src/agents/openai_runnable.ts +++ b/examples/src/agents/openai_runnable.ts @@ -36,7 +36,7 @@ const prompt = ChatPromptTemplate.fromMessages([ * Here we're using the `formatToOpenAIFunction` util function * to format our tools into the proper schema for OpenAI functions. */ -const modelWithTools = model.bind({ +const modelWithFunctions = model.bind({ functions: [...tools.map((tool) => formatToOpenAIFunction(tool))], }); /** @@ -68,7 +68,7 @@ const runnableAgent = RunnableSequence.from([ formatAgentSteps(i.steps), }, prompt, - modelWithTools, + modelWithFunctions, new OpenAIFunctionsAgentOutputParser(), ]); /** Pass the runnable along with the tools to create the Agent Executor */ diff --git a/examples/src/agents/openai_tools_runnable.ts b/examples/src/agents/openai_tools_runnable.ts new file mode 100644 index 00000000000..368d9f9722a --- /dev/null +++ b/examples/src/agents/openai_tools_runnable.ts @@ -0,0 +1,73 @@ +import { z } from "zod"; +import { ChatOpenAI } from "langchain/chat_models/openai"; +import { DynamicStructuredTool, formatToOpenAITool } from "langchain/tools"; +import { Calculator } from "langchain/tools/calculator"; +import { ChatPromptTemplate, MessagesPlaceholder } from "langchain/prompts"; +import { RunnableSequence } from "langchain/schema/runnable"; +import { AgentExecutor } from "langchain/agents"; +import { formatToOpenAIToolMessages } from "langchain/agents/format_scratchpad/openai_tools"; +import { + OpenAIToolsAgentOutputParser, + type ToolsAgentStep, +} from "langchain/agents/openai/output_parser"; + +const model = new ChatOpenAI({ + modelName: "gpt-3.5-turbo-1106", + temperature: 0, +}); + +const weatherTool = new DynamicStructuredTool({ + name: "get_current_weather", + description: "Get the current weather in a given location", + func: async ({ location }) => { + if (location.toLowerCase().includes("tokyo")) { + return JSON.stringify({ location, temperature: "10", unit: "celsius" }); + } else if (location.toLowerCase().includes("san francisco")) { + return JSON.stringify({ + location, + temperature: "72", + unit: "fahrenheit", + }); + } else { + return JSON.stringify({ location, temperature: "22", unit: "celsius" }); + } + }, + schema: z.object({ + location: z.string().describe("The city and state, e.g. San Francisco, CA"), + unit: z.enum(["celsius", "fahrenheit"]), + }), +}); + +const tools = [new Calculator(), weatherTool]; + +// Convert to OpenAI tool format +const modelWithTools = model.bind({ tools: tools.map(formatToOpenAITool) }); + +const prompt = ChatPromptTemplate.fromMessages([ + ["ai", "You are a helpful assistant"], + ["human", "{input}"], + new MessagesPlaceholder("agent_scratchpad"), +]); + +const runnableAgent = RunnableSequence.from([ + { + input: (i: { input: string; steps: ToolsAgentStep[] }) => i.input, + agent_scratchpad: (i: { input: string; steps: ToolsAgentStep[] }) => + formatToOpenAIToolMessages(i.steps), + }, + prompt, + modelWithTools, + new OpenAIToolsAgentOutputParser(), +]).withConfig({ runName: "OpenAIToolsAgent" }); + +const executor = AgentExecutor.fromAgentAndTools({ + agent: runnableAgent, + tools, +}); + +const res = await executor.invoke({ + input: + "What is the sum of the current temperature in San Francisco, New York, and Tokyo?", +}); + +console.log(res); diff --git a/langchain/.gitignore b/langchain/.gitignore index d304224d985..2e99f4a7913 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -22,6 +22,9 @@ agents/toolkits/sql.d.ts agents/format_scratchpad.cjs agents/format_scratchpad.js agents/format_scratchpad.d.ts +agents/format_scratchpad/openai_tools.cjs +agents/format_scratchpad/openai_tools.js +agents/format_scratchpad/openai_tools.d.ts agents/format_scratchpad/log.cjs agents/format_scratchpad/log.js agents/format_scratchpad/log.d.ts diff --git a/langchain/package.json b/langchain/package.json index 1afddcdf452..b8c417e81f1 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -34,6 +34,9 @@ "agents/format_scratchpad.cjs", "agents/format_scratchpad.js", "agents/format_scratchpad.d.ts", + "agents/format_scratchpad/openai_tools.cjs", + "agents/format_scratchpad/openai_tools.js", + "agents/format_scratchpad/openai_tools.d.ts", "agents/format_scratchpad/log.cjs", "agents/format_scratchpad/log.js", "agents/format_scratchpad/log.d.ts", @@ -1364,7 +1367,7 @@ "langchainhub": "~0.0.6", "langsmith": "~0.0.48", "ml-distance": "^4.0.0", - "openai": "^4.16.1", + "openai": "^4.17.0", "openapi-types": "^12.1.3", "p-queue": "^6.6.2", "p-retry": "4", @@ -1431,6 +1434,11 @@ "import": "./agents/format_scratchpad.js", "require": "./agents/format_scratchpad.cjs" }, + "./agents/format_scratchpad/openai_tools": { + "types": "./agents/format_scratchpad/openai_tools.d.ts", + "import": "./agents/format_scratchpad/openai_tools.js", + "require": "./agents/format_scratchpad/openai_tools.cjs" + }, "./agents/format_scratchpad/log": { "types": "./agents/format_scratchpad/log.d.ts", "import": "./agents/format_scratchpad/log.js", diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js index 810ce2c63a4..0776e592f0d 100644 --- a/langchain/scripts/create-entrypoints.js +++ b/langchain/scripts/create-entrypoints.js @@ -17,6 +17,7 @@ const entrypoints = { "agents/toolkits/aws_sfn": "agents/toolkits/aws_sfn", "agents/toolkits/sql": "agents/toolkits/sql/index", "agents/format_scratchpad": "agents/format_scratchpad/openai_functions", + "agents/format_scratchpad/openai_tools": "agents/format_scratchpad/openai_tools", "agents/format_scratchpad/log": "agents/format_scratchpad/log", "agents/format_scratchpad/xml": "agents/format_scratchpad/xml", "agents/format_scratchpad/log_to_message": diff --git a/langchain/src/agents/format_scratchpad/openai_tools.ts b/langchain/src/agents/format_scratchpad/openai_tools.ts new file mode 100644 index 00000000000..036af8750ef --- /dev/null +++ b/langchain/src/agents/format_scratchpad/openai_tools.ts @@ -0,0 +1,24 @@ +import type { ToolsAgentStep } from "../openai/output_parser.js"; +import { + type BaseMessage, + ToolMessage, + AIMessage, +} from "../../schema/index.js"; + +export function formatToOpenAIToolMessages( + steps: ToolsAgentStep[] +): BaseMessage[] { + return steps.flatMap(({ action, observation }) => { + if ("messageLog" in action && action.messageLog !== undefined) { + const log = action.messageLog as BaseMessage[]; + return log.concat( + new ToolMessage({ + content: observation, + tool_call_id: action.toolCallId, + }) + ); + } else { + return [new AIMessage(action.log)]; + } + }); +} diff --git a/langchain/src/agents/openai/output_parser.ts b/langchain/src/agents/openai/output_parser.ts index d42c0a3063f..534c046dd49 100644 --- a/langchain/src/agents/openai/output_parser.ts +++ b/langchain/src/agents/openai/output_parser.ts @@ -2,11 +2,15 @@ import type { OpenAI as OpenAIClient } from "openai"; import { AgentAction, AgentFinish, + AgentStep, BaseMessage, ChatGeneration, isBaseMessage, } from "../../schema/index.js"; -import { AgentActionOutputParser } from "../types.js"; +import { + AgentActionOutputParser, + AgentMultiActionOutputParser, +} from "../types.js"; import { OutputParserException } from "../../schema/output_parser.js"; /** @@ -83,3 +87,88 @@ export class OpenAIFunctionsAgentOutputParser extends AgentActionOutputParser { ); } } + +/** + * Type that represents an agent action with an optional message log. + */ +export type ToolsAgentAction = AgentAction & { + toolCallId: string; + messageLog?: BaseMessage[]; +}; + +export type ToolsAgentStep = AgentStep & { + action: ToolsAgentAction; +}; + +export class OpenAIToolsAgentOutputParser extends AgentMultiActionOutputParser { + lc_namespace = ["langchain", "agents", "openai"]; + + static lc_name() { + return "OpenAIToolsAgentOutputParser"; + } + + async parse(text: string): Promise { + throw new Error( + `OpenAIFunctionsAgentOutputParser can only parse messages.\nPassed input: ${text}` + ); + } + + async parseResult(generations: ChatGeneration[]) { + if ("message" in generations[0] && isBaseMessage(generations[0].message)) { + return this.parseAIMessage(generations[0].message); + } + throw new Error( + "parseResult on OpenAIFunctionsAgentOutputParser only works on ChatGeneration output" + ); + } + + /** + * Parses the output message into a ToolsAgentAction[] or AgentFinish + * object. + * @param message The BaseMessage to parse. + * @returns A ToolsAgentAction[] or AgentFinish object. + */ + parseAIMessage(message: BaseMessage): ToolsAgentAction[] | AgentFinish { + if (message.content && typeof message.content !== "string") { + throw new Error("This agent cannot parse non-string model responses."); + } + if (message.additional_kwargs.tool_calls) { + const toolCalls: OpenAIClient.Chat.ChatCompletionMessageToolCall[] = + message.additional_kwargs.tool_calls; + try { + return toolCalls.map((toolCall, i) => { + const toolInput = toolCall.function.arguments + ? JSON.parse(toolCall.function.arguments) + : {}; + const messageLog = i === 0 ? [message] : []; + return { + tool: toolCall.function.name as string, + toolInput, + toolCallId: toolCall.id, + log: `Invoking "${toolCall.function.name}" with ${ + toolCall.function.arguments ?? "{}" + }\n${message.content}`, + messageLog, + }; + }); + } catch (error) { + throw new OutputParserException( + `Failed to parse tool arguments from chat model response. Text: "${JSON.stringify( + toolCalls + )}". ${error}` + ); + } + } else { + return { + returnValues: { output: message.content }, + log: message.content, + }; + } + } + + getFormatInstructions(): string { + throw new Error( + "getFormatInstructions not implemented inside OpenAIToolsAgentOutputParser." + ); + } +} diff --git a/langchain/src/agents/tests/openai_tools.int.test.ts b/langchain/src/agents/tests/openai_tools.int.test.ts new file mode 100644 index 00000000000..85cf827840f --- /dev/null +++ b/langchain/src/agents/tests/openai_tools.int.test.ts @@ -0,0 +1,80 @@ +/* eslint-disable no-process-env */ +import { test } from "@jest/globals"; +import { z } from "zod"; +import { Calculator } from "../../tools/calculator.js"; +import { ChatOpenAI } from "../../chat_models/openai.js"; +import { AgentExecutor } from "../executor.js"; +import { formatToOpenAIToolMessages } from "../format_scratchpad/openai_tools.js"; +import { + OpenAIToolsAgentOutputParser, + ToolsAgentStep, +} from "../openai/output_parser.js"; +import { ChatPromptTemplate, MessagesPlaceholder } from "../../prompts/chat.js"; +import { RunnableSequence } from "../../schema/runnable/base.js"; +import { DynamicStructuredTool } from "../../tools/dynamic.js"; +import { formatToOpenAITool } from "../../tools/convert_to_openai.js"; + +test("OpenAIToolsAgent", async () => { + const model = new ChatOpenAI({ + modelName: "gpt-3.5-turbo-1106", + temperature: 0, + }); + + const weatherTool = new DynamicStructuredTool({ + name: "get_current_weather", + description: "Get the current weather in a given location", + func: async ({ location }) => { + if (location.toLowerCase().includes("tokyo")) { + return JSON.stringify({ location, temperature: "10", unit: "celsius" }); + } else if (location.toLowerCase().includes("san francisco")) { + return JSON.stringify({ + location, + temperature: "72", + unit: "fahrenheit", + }); + } else { + return JSON.stringify({ location, temperature: "22", unit: "celsius" }); + } + }, + schema: z.object({ + location: z + .string() + .describe("The city and state, e.g. San Francisco, CA"), + unit: z.enum(["celsius", "fahrenheit"]), + }), + }); + + const tools = [new Calculator(), weatherTool]; + + const modelWithTools = model.bind({ tools: tools.map(formatToOpenAITool) }); + + const prompt = ChatPromptTemplate.fromMessages([ + ["ai", "You are a helpful assistant"], + ["human", "{input}"], + new MessagesPlaceholder("agent_scratchpad"), + ]); + + const runnableAgent = RunnableSequence.from([ + { + input: (i: { input: string; steps: ToolsAgentStep[] }) => i.input, + agent_scratchpad: (i: { input: string; steps: ToolsAgentStep[] }) => + formatToOpenAIToolMessages(i.steps), + }, + prompt, + modelWithTools, + new OpenAIToolsAgentOutputParser(), + ]).withConfig({ runName: "OpenAIToolsAgent" }); + + const executor = AgentExecutor.fromAgentAndTools({ + agent: runnableAgent, + tools, + verbose: true, + }); + + const res = await executor.invoke({ + input: + "What is the sum of the current temperature in San Francisco, New York, and Tokyo?", + }); + + console.log(res); +}); diff --git a/langchain/src/agents/types.ts b/langchain/src/agents/types.ts index a0fe0343be4..b89274effbe 100644 --- a/langchain/src/agents/types.ts +++ b/langchain/src/agents/types.ts @@ -44,6 +44,14 @@ export abstract class AgentActionOutputParser extends BaseOutputParser< AgentAction | AgentFinish > {} +/** + * Abstract class representing an output parser specifically for agents + * that return multiple actions. + */ +export abstract class AgentMultiActionOutputParser extends BaseOutputParser< + AgentAction[] | AgentFinish +> {} + /** * Type representing the stopping method for an agent. It can be either * 'force' or 'generate'. diff --git a/langchain/src/load/import_map.ts b/langchain/src/load/import_map.ts index 01f53c8d235..7dd272491d3 100644 --- a/langchain/src/load/import_map.ts +++ b/langchain/src/load/import_map.ts @@ -4,6 +4,7 @@ export * as load__serializable from "../load/serializable.js"; export * as agents from "../agents/index.js"; export * as agents__toolkits from "../agents/toolkits/index.js"; export * as agents__format_scratchpad from "../agents/format_scratchpad/openai_functions.js"; +export * as agents__format_scratchpad__openai_tools from "../agents/format_scratchpad/openai_tools.js"; export * as agents__format_scratchpad__log from "../agents/format_scratchpad/log.js"; export * as agents__format_scratchpad__xml from "../agents/format_scratchpad/xml.js"; export * as agents__format_scratchpad__log_to_message from "../agents/format_scratchpad/log_to_message.js"; diff --git a/langchain/src/tools/convert_to_openai.ts b/langchain/src/tools/convert_to_openai.ts index d2bf7c12388..0c599a14e3b 100644 --- a/langchain/src/tools/convert_to_openai.ts +++ b/langchain/src/tools/convert_to_openai.ts @@ -22,12 +22,13 @@ export function formatToOpenAIFunction( export function formatToOpenAITool( tool: StructuredTool ): OpenAIClient.Chat.ChatCompletionTool { + const schema = zodToJsonSchema(tool.schema); return { type: "function", function: { name: tool.name, description: tool.description, - parameters: zodToJsonSchema(tool.schema), + parameters: schema, }, }; } diff --git a/langchain/src/tools/index.ts b/langchain/src/tools/index.ts index b13f10dc3f5..f2995501634 100644 --- a/langchain/src/tools/index.ts +++ b/langchain/src/tools/index.ts @@ -43,4 +43,7 @@ export { } from "./dataforseo_api_search.js"; export { SearxngSearch } from "./searxng_search.js"; export { SearchApi, type SearchApiParameters } from "./searchapi.js"; -export { formatToOpenAIFunction } from "./convert_to_openai.js"; +export { + formatToOpenAIFunction, + formatToOpenAITool, +} from "./convert_to_openai.js"; diff --git a/langchain/tsconfig.json b/langchain/tsconfig.json index b5dc042a417..307df661c54 100644 --- a/langchain/tsconfig.json +++ b/langchain/tsconfig.json @@ -40,6 +40,7 @@ "src/agents/toolkits/aws_sfn.ts", "src/agents/toolkits/sql/index.ts", "src/agents/format_scratchpad/openai_functions.ts", + "src/agents/format_scratchpad/openai_tools.ts", "src/agents/format_scratchpad/log.ts", "src/agents/format_scratchpad/xml.ts", "src/agents/format_scratchpad/log_to_message.ts", diff --git a/yarn.lock b/yarn.lock index ffd64a0e9f7..99cd5c968ed 100644 --- a/yarn.lock +++ b/yarn.lock @@ -21691,7 +21691,7 @@ __metadata: neo4j-driver: ^5.12.0 node-llama-cpp: 2.7.3 notion-to-md: ^3.1.0 - openai: ^4.16.1 + openai: ^4.17.0 openapi-types: ^12.1.3 p-queue: ^6.6.2 p-retry: 4 @@ -24234,9 +24234,9 @@ __metadata: languageName: node linkType: hard -"openai@npm:^4.16.1": - version: 4.16.1 - resolution: "openai@npm:4.16.1" +"openai@npm:^4.17.0": + version: 4.17.0 + resolution: "openai@npm:4.17.0" dependencies: "@types/node": ^18.11.18 "@types/node-fetch": ^2.6.4 @@ -24249,7 +24249,7 @@ __metadata: web-streams-polyfill: ^3.2.1 bin: openai: bin/cli - checksum: 1d917df7c77c488e8dc2bb685acd3dea2ea373431f28e31c9831de7d4f9b14a440b6a187ca92a37512c3d00d9371036b308c27ad18e9b242c4743fb510c1a491 + checksum: 22b0a7704ba2dd4f139c1085e02aa45ca57c0df53b6f8d097a897da9cb17b9e1083e4dd65ec7ad2ca9a6c4110811bcb080e7fa01f61316b4e3a949c072d53666 languageName: node linkType: hard