Add monitor + openai support

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Program",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "program": "${workspaceFolder}/lib/index.js",
+      "outFiles": [
+        "${workspaceFolder}/**/*.js"
+      ]
+    }
+  ]
+}

README.md

@@ -32,6 +32,6 @@ Full docs are available [here](https://llmonitor.com/docs/js).
 
 ## 👷 TODO
 
-- [ ] maintain/forward types while wrapping (for `.langchain` and `.wrap` methods)
-- [ ] `openai` module wrapper
-- [ ] langchain embeddings wrapping
+- [X] `openai` module wrapper
+- [ ] langchain embeddings wrapping
+- [ ] put useless packages for publication in dev dependencies

examples/agent.ts

@@ -0,0 +1,35 @@
+import "dotenv/config"
+import { ChatOpenAI } from "langchain/chat_models/openai"
+import { HumanMessage, SystemMessage } from "langchain/schema"
+import monitor, { llmonitor } from "../src"
+
+const chat = new ChatOpenAI({
+  temperature: 0.2,
+  modelName: "gpt-3.5-turbo",
+  tags: ["test-tag"],
+})
+
+llmonitor.identify("123", {
+  email: "my-user@example.org",
+})
+monitor(chat)
+
+const TranslatorAgent = async (query) => {
+  console.log(query)
+  const res = await chat.call([
+    new SystemMessage("You are a translator agent."),
+    new HumanMessage(
+      `Translate this sentence from English to French. ${query}`
+    ),
+  ])
+
+  return res.content
+}
+
+// By wrapping the executor, we automatically track all input, outputs and errors
+// And tools and logs will be tied to the correct agent
+const translate = llmonitor.wrapAgent(TranslatorAgent, { name: "translate" })
+
+translate("Hello, what's up").then((res) => {
+  console.log(res) // "Bonjour, comment allez-vous?"
+})

examples/langchain.ts

@@ -0,0 +1,24 @@
+import "dotenv/config"
+import { ChatOpenAI } from "langchain/chat_models/openai"
+import { HumanMessage, SystemMessage } from "langchain/schema"
+import monitor from "../src/index"
+
+const chat = new ChatOpenAI({
+  temperature: 0.2,
+  modelName: "gpt-3.5-turbo",
+  tags: ["test-tag"],
+})
+monitor(chat)
+
+async function main(query: string) {
+  const res = await chat.call([
+    new SystemMessage("You are a translator agent."),
+    new HumanMessage(
+      `Translate this sentence from English to French. ${query}`
+    ),
+  ])
+
+  console.log(res.content)
+}
+
+main("Hello friend")

examples/openai.ts

@@ -0,0 +1,22 @@
+import "dotenv/config"
+import { Configuration, OpenAIApi } from "openai"
+import monitor from "../src/index"
+
+const configuration = new Configuration({
+  apiKey: process.env.OPENAI_API_KEY,
+})
+const openai = new OpenAIApi(configuration)
+monitor(openai)
+
+async function main() {
+  const chatCompletion = await openai.createChatCompletion({
+    model: "gpt-3.5-turbo",
+    messages: [
+      { role: "system", content: "You are an helpful assitant" },
+      { role: "user", content: "Hello world" },
+    ],
+  })
+  console.log(chatCompletion.data.choices[0].message)
+}
+
+main()

src/index.ts

@@ -1,7 +1,6 @@
 import LLMonitor from "./llmonitor"
 
-export { LLMonitor }
-
-const monitor = new LLMonitor()
+export const llmonitor = new LLMonitor()
+const monitor = llmonitor.monitor.bind(llmonitor)
 
 export default monitor

src/langchain.ts

@@ -0,0 +1,62 @@
+import { Callbacks } from "langchain/callbacks"
+import { ChatOpenAI, ChatOpenAICallOptions } from "langchain/chat_models/openai"
+import { BaseMessage, LLMResult } from "langchain/schema"
+import ctx from "src/context"
+import { cleanError, cleanExtra, parseLangchainMessages } from "src/utils"
+
+// TODO: type with other chat models (not basechat/llm, but an union of all chat models)
+export function monitorLangchainLLM(chat: ChatOpenAI, llmonitor: LLMonitor) {
+  const originalGenerate = chat.generate
+
+  chat.generate = async function (
+    messages: BaseMessage[][],
+    options?: ChatOpenAICallOptions | string[],
+    callbacks?: Callbacks
+  ) {
+    const runId = crypto.randomUUID()
+    const input = parseLangchainMessages(messages)
+    const { tags, modelName: name } = chat
+
+    const rawExtra = {
+      temperature: chat.temperature,
+      maxTokens: chat.maxTokens,
+      frequencyPenalty: chat.frequencyPenalty,
+      presencePenalty: chat.presencePenalty,
+      stop: chat.stop,
+      timeout: chat.timeout,
+      modelKwargs: Object.keys(chat.modelKwargs || {}).length
+        ? chat.modelKwargs
+        : undefined,
+    }
+    const extra = cleanExtra(rawExtra)
+
+    const event = { name, input, runId, extra, tags }
+
+    try {
+      llmonitor.trackEvent("llm", "start", event)
+
+      const rawOutput = await ctx.callAsync<LLMResult>(runId, () =>
+        originalGenerate.apply(this, [messages, options, callbacks])
+      )
+
+      const output = parseLangchainMessages(rawOutput.generations)
+
+      const tokensUsage = {
+        completion: rawOutput.llmOutput?.tokenUsage?.completionTokens,
+        prompt: rawOutput.llmOutput?.tokenUsage?.promptTokens,
+      }
+
+      llmonitor.trackEvent("llm", "end", { ...event, output, tokensUsage })
+
+      return rawOutput
+    } catch (error: unknown) {
+      llmonitor.trackEvent("llm", "error", {
+        runId,
+        name,
+        error: cleanError(error),
+      })
+
+      throw error
+    }
+  }
+}

src/llmonitor.ts

@@ -1,3 +1,5 @@
+import { BaseLanguageModel } from "langchain/base_language"
+import { Tool } from "langchain/tools"
 import {
   checkEnv,
   cleanError,
@@ -8,15 +10,21 @@ import {
 } from "./utils"
 
 import {
-  cJSON,
-  LLMonitorOptions,
+  EntityToMonitor,
   Event,
+  EventName,
   EventType,
-  RunEvent,
+  LLMonitorOptions,
   LogEvent,
+  RunEvent,
   WrapParams,
+  cJSON,
 } from "./types"
 
+import { OpenAIApi } from "openai"
+import { monitorLangchainLLM } from "src/langchain"
+import { monitorOpenAi } from "src/openai"
+import { monitorTool } from "src/tool"
 import ctx from "./context"
 
 class LLMonitor {
@@ -32,31 +40,53 @@ class LLMonitor {
   /**
    * @param {LLMonitorOptions} options
    */
-
   constructor() {
     this.load({
       appId: checkEnv("LLMONITOR_APP_ID"),
-      log: false,
+      log: true,
       apiUrl: checkEnv("LLMONITOR_API_URL") || "https://app.llmonitor.com",
     })
   }
 
-  load(options?: Partial<LLMonitorOptions>) {
-    if (options.appId) this.appId = options.appId
-    if (options.log) this.logConsole = options.log
-    if (options.apiUrl) this.apiUrl = options.apiUrl
-    if (options.userId) this.userId = options.userId
-    if (options.userProps) this.userProps = options.userProps
+  load({ appId, log, apiUrl, userId, userProps }: LLMonitorOptions = {}) {
+    if (appId) this.appId = appId
+    if (log) this.logConsole = log
+    if (apiUrl) this.apiUrl = apiUrl
+    if (userId) this.userId = userId
+    if (userProps) this.userProps = userProps
   }
 
   identify(userId: string, userProps?: cJSON) {
     this.userId = userId
     this.userProps = userProps
   }
 
+  monitor(entities: EntityToMonitor | [EntityToMonitor]) {
+    const llmonitor = this
+
+    entities = Array.isArray(entities) ? entities : [entities]
+
+    entities.forEach((entity) => {
+      if (entity instanceof OpenAIApi) {
+        monitorOpenAi(entity, llmonitor)
+        return
+      }
+
+      if (entity instanceof BaseLanguageModel) {
+        monitorLangchainLLM(entity as any, llmonitor)
+        return
+      }
+
+      if (entities instanceof Tool) {
+        monitorTool(entity, llmonitor)
+        return
+      }
+    })
+  }
+
   async trackEvent(
     type: EventType,
-    event: string,
+    event: EventName,
     data: Partial<RunEvent | LogEvent>
   ) {
     if (!this.appId)
@@ -138,7 +168,8 @@ class LLMonitor {
       const runId = crypto.randomUUID()
 
       // Get agent name from function name or params
-      const name = params?.name ?? func.name
+      // const name = params?.name ?? func.name
+      const name = func.name
 
       const { inputParser, outputParser, tokensUsageParser, extra, tags } =
         params || {}

src/openai.ts

@@ -0,0 +1,64 @@
+import { AxiosRequestConfig } from "axios"
+import { CreateChatCompletionRequest, OpenAIApi } from "openai"
+import ctx from "src/context"
+import { cleanError, parseOpenaiMessage } from "src/utils"
+import LLMonitor from "./llmonitor"
+
+export function monitorOpenAi(openai: OpenAIApi, llmonitor: LLMonitor) {
+  const originalCreateChatCompletion = openai.createChatCompletion
+
+  openai.createChatCompletion = async function (
+    request: CreateChatCompletionRequest,
+    options?: AxiosRequestConfig
+  ) {
+    const runId = crypto.randomUUID()
+    const input = request.messages.map(parseOpenaiMessage)
+    const { model: name } = request
+
+    const rawExtra = {
+      temperature: request.temperature,
+      maxTokens: request.max_tokens,
+      frequencyPenalty: request.frequency_penalty,
+      presencePenalty: request.presence_penalty,
+      stop: request.stop,
+    }
+    // const extra = cleanExtra(rawExtra)
+    const extra = {
+      temperature: 0.2,
+      frequencyPenalty: 0,
+      presencePenalty: 0,
+    }
+    const tags = ["test-tag"]
+
+    const event = { name, input, runId, extra, tags }
+
+    try {
+      llmonitor.trackEvent("llm", "start", event)
+      const rawOutput = await ctx.callAsync<
+        ReturnType<typeof originalCreateChatCompletion>
+      >(runId, () =>
+        originalCreateChatCompletion.apply(this, [request, options])
+      )
+
+      const { data } = rawOutput
+      const output = parseOpenaiMessage(data.choices[0].message)
+
+      const tokensUsage = {
+        completion: data.usage?.completion_tokens,
+        prompt: data.usage?.prompt_tokens,
+      }
+
+      llmonitor.trackEvent("llm", "end", { ...event, output, tokensUsage })
+
+      return rawOutput
+    } catch (error: unknown) {
+      llmonitor.trackEvent("llm", "error", {
+        runId,
+        name,
+        error: cleanError(error),
+      })
+
+      throw error
+    }
+  }
+}

src/test.ts

@@ -0,0 +1,32 @@
+import "dotenv/config"
+// Basic agent monitoring example
+import { ChatOpenAI } from "langchain/chat_models/openai"
+import { HumanMessage, SystemMessage } from "langchain/schema"
+import { llmonitor } from "src"
+
+const Chat = llmonitor.langchain(ChatOpenAI)
+
+const chat = new Chat({
+  temperature: 0.2,
+  modelName: "gpt-3.5-turbo",
+  tags: ["test-tag"],
+})
+
+const TranslatorAgent = async (query) => {
+  const res = await chat.call([
+    new SystemMessage("You are a translator agent."),
+    new HumanMessage(
+      `Translate this sentence from English to French. ${query}`
+    ),
+  ])
+
+  return res.content
+}
+
+// By wrapping the executor, we automatically track all input, outputs and errors
+// And tools and logs will be tied to the correct agent
+const translate = llmonitor.wrapAgent(TranslatorAgent)
+
+translate("Hello, how are you?").then((res) => {
+  console.log(res) // "Bonjour, comment allez-vous?"
+})