Update to Use a Factory Method

examples/src/client/tracing_datasets.ts

@@ -89,8 +89,18 @@ what is 1213 divided by 4345?,approximately 0.2791714614499425
     );
   }
 
+  // Many chains incorporate memory. For independent trials over the dataset, we
+  // pass in a factory function that creates a new executor for each trial.
+  // If you know that your chain does not use memory, you can return the same
+  // executor for each trial.
+  const executorFactory = async () =>
+    initializeAgentExecutorWithOptions(tools, model, {
+      agentType: "chat-conversational-react-description",
+      verbose: true,
+    });
+
   // If using the traced dataset, you can update the datasetName to be
   // "calculator-example-dataset" or the custom name you chose.
-  const results = await client.runOnDataset(datasetName, executor);
+  const results = await client.runOnDataset(datasetName, executorFactory);
   console.log(results);
 };

langchain/src/client/langchainplus.ts

@@ -108,28 +108,57 @@ const stringifyError = (err: Error | unknown): string => {
   return result;
 };
 
-export function isLLM(llm: BaseLanguageModel | BaseChain): llm is BaseLLM {
+export function isLLM(
+  llm: BaseLanguageModel | (() => Promise<BaseChain>)
+): llm is BaseLLM {
   const blm = llm as BaseLanguageModel;
   return (
     typeof blm?._modelType === "function" && blm?._modelType() === "base_llm"
   );
 }
 
-export function isChatModel(llm: BaseLanguageModel): llm is BaseChatModel {
+export function isChatModel(
+  llm: BaseLanguageModel | (() => Promise<BaseChain>)
+): llm is BaseChatModel {
   const blm = llm as BaseLanguageModel;
   return (
     typeof blm?._modelType === "function" &&
     blm?._modelType() === "base_chat_model"
   );
 }
 
-export function isChain(llm: BaseLanguageModel | BaseChain): llm is BaseChain {
-  const bch = llm as BaseChain;
+export async function isChain(
+  llm: BaseLanguageModel | (() => Promise<BaseChain>)
+): Promise<boolean> {
+  if (isLLM(llm)) {
+    return false;
+  }
+  const bchFactory = llm as () => Promise<BaseChain>;
+  const bch = await bchFactory();
   return (
     typeof bch?._chainType === "function" && bch?._chainType() !== undefined
   );
 }
 
+type _ModelType = "llm" | "chatModel" | "chainFactory";
+
+async function getModelOrFactoryType(
+  llm: BaseLanguageModel | (() => Promise<BaseChain>)
+): Promise<_ModelType> {
+  if (isLLM(llm)) {
+    return "llm";
+  }
+  if (isChatModel(llm)) {
+    return "chatModel";
+  }
+  const bchFactory = llm as () => Promise<BaseChain>;
+  const bch = await bchFactory();
+  if (typeof bch?._chainType === "function") {
+    return "chainFactory";
+  }
+  throw new Error("Unknown model or factory type");
+}
+
 export class LangChainPlusClient {
   private apiKey?: string;
 
@@ -453,12 +482,13 @@ export class LangChainPlusClient {
   protected async runChain(
     example: Example,
     tracer: LangChainTracer,
-    chain: BaseChain,
+    chainFactory: () => Promise<BaseChain>,
     numRepetitions = 1
   ): Promise<(ChainValues | string)[]> {
     const results: (ChainValues | string)[] = await Promise.all(
       Array.from({ length: numRepetitions }).map(async () => {
         try {
+          const chain = await chainFactory();
           return chain.call(example.inputs, [tracer]);
         } catch (e) {
           console.error(e);
@@ -495,51 +525,55 @@ export class LangChainPlusClient {
 
   public async runOnDataset(
     datasetName: string,
-    llmOrChain: BaseLanguageModel | BaseChain,
+    llmOrChainFactory: BaseLanguageModel | (() => Promise<BaseChain>),
     numRepetitions = 1,
     sessionName: string | undefined = undefined
   ): Promise<DatasetRunResults> {
     const examples = await this.listExamples(undefined, datasetName);
     let sessionName_: string;
     if (sessionName === undefined) {
       const currentTime = new Date().toISOString();
-      sessionName_ = `${datasetName}-${llmOrChain.constructor.name}-${currentTime}`;
+      sessionName_ = `${datasetName}-${llmOrChainFactory.constructor.name}-${currentTime}`;
     } else {
       sessionName_ = sessionName;
     }
     const results: DatasetRunResults = {};
+    const modelOrFactoryType = await getModelOrFactoryType(llmOrChainFactory);
     await Promise.all(
       examples.map(async (example) => {
         const tracer = new LangChainTracer({
           exampleId: example.id,
           sessionName: sessionName_,
         });
-        if (isLLM(llmOrChain)) {
+        if (modelOrFactoryType === "llm") {
+          const llm = llmOrChainFactory as BaseLLM;
           const llmResult = await this.runLLM(
             example,
             tracer,
-            llmOrChain,
+            llm,
             numRepetitions
           );
           results[example.id] = llmResult;
-        } else if (isChain(llmOrChain)) {
+        } else if (modelOrFactoryType === "chainFactory") {
+          const chainFactory = llmOrChainFactory as () => Promise<BaseChain>;
           const chainResult = await this.runChain(
             example,
             tracer,
-            llmOrChain,
+            chainFactory,
             numRepetitions
           );
           results[example.id] = chainResult;
-        } else if (isChatModel(llmOrChain)) {
+        } else if (modelOrFactoryType === "chatModel") {
+          const chatModel = llmOrChainFactory as BaseChatModel;
           const chatModelResult = await this.runChatModel(
             example,
             tracer,
-            llmOrChain,
+            chatModel,
             numRepetitions
           );
           results[example.id] = chatModelResult;
         } else {
-          throw new Error(` llm or chain type: ${llmOrChain}`);
+          throw new Error(` llm or chain type: ${llmOrChainFactory}`);
         }
       })
     );

langchain/src/client/tests/langchainplus.int.test.ts

@@ -156,13 +156,14 @@ what is 1213 divided by 4345?,approximately 0.2791714614499425
   const model = new ChatOpenAI({ temperature: 0 });
   const tools = [new Calculator()];
 
-  const executor = await initializeAgentExecutorWithOptions(tools, model, {
-    agentType: "chat-conversational-react-description",
-    verbose: true,
-  });
+  const executorFactory = async () =>
+    await initializeAgentExecutorWithOptions(tools, model, {
+      agentType: "chat-conversational-react-description",
+      verbose: true,
+    });
   console.log("Loaded agent.");
 
-  const results = await client.runOnDataset(datasetName, executor);
+  const results = await client.runOnDataset(datasetName, executorFactory);
   console.log(results);
   expect(Object.keys(results).length).toEqual(2);
 });
@@ -205,13 +206,14 @@ what was the tjtal number of points scored in the 2023 super bowl? what is that
     new Calculator(),
   ];
 
-  const executor = await initializeAgentExecutorWithOptions(tools, model, {
-    agentType: "chat-conversational-react-description",
-    verbose: true,
-  });
+  const executorFactory = async () =>
+    await initializeAgentExecutorWithOptions(tools, model, {
+      agentType: "chat-conversational-react-description",
+      verbose: true,
+    });
   console.log("Loaded agent.");
 
-  const results = await client.runOnDataset(datasetName, executor);
+  const results = await client.runOnDataset(datasetName, executorFactory);
   console.log(results);
   expect(Object.keys(results).length).toEqual(10);
 });