From 3c50606a57a8dc18c91fe36e3b180bafd15a4cbc Mon Sep 17 00:00:00 2001 From: Svetlana Brennan <50715937+svetlanabrennan@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:48:47 -0600 Subject: [PATCH] feat: Added instrumentation for VectorStore.similaritySearch for langchain.js (#2049) Co-authored-by: Bob Evans --- docker-compose.yml | 2 +- lib/instrumentation/langchain/nr-hooks.js | 6 + lib/instrumentation/langchain/vectorstore.js | 108 ++++++++ lib/llm-events/error-message.js | 5 +- .../langchain/vector-search-result.js | 5 +- lib/metrics/names.js | 6 +- .../langchain/vectorstore.test.js | 68 +++++ .../langchain/vector-search-result.test.js | 19 +- test/unit/llm-events/openai/error.test.js | 3 +- test/versioned/langchain/common.js | 55 +++- test/versioned/langchain/package.json | 7 +- .../langchain/runnables-streaming.tap.js | 20 +- test/versioned/langchain/runnables.tap.js | 20 +- test/versioned/langchain/vectorstore.tap.js | 257 ++++++++++++++++++ test/versioned/openai/mock-responses.js | 4 +- test/versioned/openai/mock-server.js | 4 + 16 files changed, 551 insertions(+), 38 deletions(-) create mode 100644 lib/instrumentation/langchain/vectorstore.js create mode 100644 test/unit/instrumentation/langchain/vectorstore.test.js create mode 100644 test/versioned/langchain/vectorstore.tap.js diff --git a/docker-compose.yml b/docker-compose.yml index 5ab17035f1..ac6fb6a20a 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: elasticsearch: container_name: nr_node_elastic - image: docker.elastic.co/elasticsearch/elasticsearch:8.7.1 + image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0 environment: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # Set cluster to single node diff --git a/lib/instrumentation/langchain/nr-hooks.js b/lib/instrumentation/langchain/nr-hooks.js index 8a47a554ef..961257c917 100644 --- a/lib/instrumentation/langchain/nr-hooks.js +++ b/lib/instrumentation/langchain/nr-hooks.js @@ -7,6 +7,7 @@ const toolsInstrumentation = require('./tools') const cbManagerInstrumentation = require('./callback-manager') const runnableInstrumentation = require('./runnable') +const vectorstoreInstrumentation = require('./vectorstore') module.exports = [ { @@ -23,5 +24,10 @@ module.exports = [ type: 'generic', moduleName: '@langchain/core/dist/runnables/base', onRequire: runnableInstrumentation + }, + { + type: 'generic', + moduleName: '@langchain/core/vectorstores', + onRequire: vectorstoreInstrumentation } ] diff --git a/lib/instrumentation/langchain/vectorstore.js b/lib/instrumentation/langchain/vectorstore.js new file mode 100644 index 0000000000..d2ebba5f45 --- /dev/null +++ b/lib/instrumentation/langchain/vectorstore.js @@ -0,0 +1,108 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' +const { + AI: { LANGCHAIN } +} = require('../../metrics/names') +const { LangChainVectorSearch, LangChainVectorSearchResult } = require('../../llm-events/langchain') +const { recordEvent, shouldSkipInstrumentation } = require('./common') +const { DESTINATIONS } = require('../../config/attribute-filter') +const { RecorderSpec } = require('../../shim/specs') +const LlmErrorMessage = require('../../llm-events/error-message') + +/** + * Generates a LangChainVectorSearch for entire search request. + * Also iterates over documents in output and generates a + * LangChainVectorSearchResult for each document. + * + * @param {object} params input params + * @param {string} params.request vector search query + * @param {number} params.k vector search top k + * @param {object} params.output vector search documents + * @param {Agent} params.agent NR agent instance + * @param {TraceSegment} params.segment active segment from vector search + * @param {string} params.pkgVersion langchain version + * @param {err} params.err if it exists + */ +function recordVectorSearch({ request, k, output, agent, segment, pkgVersion, err }) { + const vectorSearch = new LangChainVectorSearch({ + agent, + segment, + query: request, + k, + documents: output, + error: err !== null + }) + + recordEvent({ agent, type: 'LlmVectorSearch', pkgVersion, msg: vectorSearch }) + + output.forEach((document, sequence) => { + const vectorSearchResult = new LangChainVectorSearchResult({ + agent, + segment, + metadata: document.metadata, + pageContent: document.pageContent, + sequence, + search_id: vectorSearch.id + }) + + recordEvent({ + agent, + type: 'LlmVectorSearchResult', + pkgVersion, + msg: vectorSearchResult + }) + }) + + if (err) { + agent.errors.add( + segment.transaction, + err, + new LlmErrorMessage({ + response: output, + cause: err, + vectorsearch: vectorSearch + }) + ) + } +} + +module.exports = function initialize(shim, vectorstores) { + const { agent, pkgVersion } = shim + + if (shouldSkipInstrumentation(agent.config)) { + shim.logger.debug( + 'langchain instrumentation is disabled. To enable set `config.ai_monitoring.enabled` to true' + ) + return + } + + shim.record( + vectorstores.VectorStore.prototype, + 'similaritySearch', + function wrapCall(shim, similaritySearch, fnName, args) { + const [request, k] = args + + return new RecorderSpec({ + name: `${LANGCHAIN.VECTORSTORE}/${fnName}`, + promise: true, + // eslint-disable-next-line max-params + after(_shim, _fn, _name, err, output, segment) { + if (!output) { + // If we get an error, it is possible that `output = null`. + // In that case, we define it to be an empty array. + output = [] + } + + segment.end() + recordVectorSearch({ request, k, output, agent, segment, pkgVersion, err }) + + segment.transaction.trace.attributes.addAttribute(DESTINATIONS.TRANS_EVENT, 'llm', true) + } + }) + } + ) +} diff --git a/lib/llm-events/error-message.js b/lib/llm-events/error-message.js index e448781411..cf28e70126 100644 --- a/lib/llm-events/error-message.js +++ b/lib/llm-events/error-message.js @@ -18,14 +18,17 @@ module.exports = class LlmErrorMessage { * conversation if it was a chat completion conversation. * @param {LlmEmbedding} [params.embedding] Details about the conversation * if it was an embedding conversation. + * @param {LlmVectorStoreSearch} [params.vectorsearch] Details about the vector + * search if it was a vector search event. */ - constructor({ response, cause, summary, embedding } = {}) { + constructor({ response, cause, summary, embedding, vectorsearch } = {}) { this['http.statusCode'] = response?.status ?? cause?.status this['error.message'] = cause?.message this['error.code'] = response?.code ?? cause?.error?.code this['error.param'] = response?.param ?? cause?.error?.param this.completion_id = summary?.id this.embedding_id = embedding?.id + this.vector_store_id = vectorsearch?.id } get [Symbol.toStringTag]() { diff --git a/lib/llm-events/langchain/vector-search-result.js b/lib/llm-events/langchain/vector-search-result.js index 2612218fc8..7aae16cd2b 100644 --- a/lib/llm-events/langchain/vector-search-result.js +++ b/lib/llm-events/langchain/vector-search-result.js @@ -6,13 +6,13 @@ 'use strict' const LangChainEvent = require('./event') -const crypto = require('crypto') /** * @typedef {object} LangChainVectorSearchResultParams * @augments LangChainEventParams * @property {string} pageContent The stringified contents of the pageContent attribute on each returned search result document. * @property {number} [sequence=0] The index of the document in the search result documents list. + * @property {string} search_id The identifier from the LangChainVectorSearch event. */ /** * @type {LangChainVectorSearchResultParams} @@ -23,13 +23,12 @@ const defaultParams = { } class LangChainVectorSearchResult extends LangChainEvent { - search_id = crypto.randomUUID() - constructor(params) { params = Object.assign({}, defaultParams, params) super(params) const { agent } = params + this.search_id = params.search_id this.sequence = params.sequence if (agent.config.ai_monitoring.record_content.enabled === true) { diff --git a/lib/metrics/names.js b/lib/metrics/names.js index 41b0103211..82d6e3a1a3 100644 --- a/lib/metrics/names.js +++ b/lib/metrics/names.js @@ -170,7 +170,8 @@ const AI = { EMBEDDING: 'Llm/embedding', COMPLETION: 'Llm/completion', TOOL: 'Llm/tool', - CHAIN: 'Llm/chain' + CHAIN: 'Llm/chain', + VECTORSTORE: 'Llm/vectorstore' } AI.OPENAI = { @@ -184,7 +185,8 @@ AI.LANGCHAIN = { EMBEDDING: `${AI.EMBEDDING}/Langchain`, COMPLETION: `${AI.COMPLETION}/Langchain`, TOOL: `${AI.TOOL}/Langchain`, - CHAIN: `${AI.CHAIN}/Langchain` + CHAIN: `${AI.CHAIN}/Langchain`, + VECTORSTORE: `${AI.VECTORSTORE}/Langchain` } const RESTIFY = { diff --git a/test/unit/instrumentation/langchain/vectorstore.test.js b/test/unit/instrumentation/langchain/vectorstore.test.js new file mode 100644 index 0000000000..3c6111ea9f --- /dev/null +++ b/test/unit/instrumentation/langchain/vectorstore.test.js @@ -0,0 +1,68 @@ +/* + * Copyright 2023 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const { test } = require('tap') +const helper = require('../../../lib/agent_helper') +const GenericShim = require('../../../../lib/shim/shim') +const sinon = require('sinon') + +test('langchain/core/vectorstore unit tests', (t) => { + t.beforeEach(function (t) { + const sandbox = sinon.createSandbox() + const agent = helper.loadMockedAgent() + agent.config.ai_monitoring = { enabled: true } + agent.config.feature_flag = { langchain_instrumentation: true } + const shim = new GenericShim(agent, 'langchain') + shim.pkgVersion = '0.1.26' + sandbox.stub(shim.logger, 'debug') + sandbox.stub(shim.logger, 'warn') + + t.context.agent = agent + t.context.shim = shim + t.context.sandbox = sandbox + t.context.initialize = require('../../../../lib/instrumentation/langchain/vectorstore') + }) + + t.afterEach(function (t) { + helper.unloadAgent(t.context.agent) + t.context.sandbox.restore() + }) + + function getMockModule() { + function VectorStore() {} + VectorStore.prototype.similaritySearch = async function call() {} + return { VectorStore } + } + + ;[ + { aiMonitoring: false, langChain: true }, + { aiMonitoring: true, langChain: false }, + { aiMonitoring: false, langChain: false } + ].forEach(({ aiMonitoring, langChain }) => { + t.test( + `should not register instrumentation if ai_monitoring is ${aiMonitoring} and langchain_instrumentation is ${langChain}`, + (t) => { + const { shim, agent, initialize } = t.context + const MockVectorstore = getMockModule() + agent.config.ai_monitoring.enabled = aiMonitoring + agent.config.feature_flag.langchain_instrumentation = langChain + + initialize(shim, MockVectorstore) + t.equal(shim.logger.debug.callCount, 1, 'should log 1 debug messages') + t.equal( + shim.logger.debug.args[0][0], + 'langchain instrumentation is disabled. To enable set `config.ai_monitoring.enabled` to true' + ) + const isWrapped = shim.isWrapped(MockVectorstore.VectorStore.prototype.similaritySearch) + t.equal(isWrapped, false, 'should not wrap vectorstore similaritySearch') + t.end() + } + ) + }) + + t.end() +}) diff --git a/test/unit/llm-events/langchain/vector-search-result.test.js b/test/unit/llm-events/langchain/vector-search-result.test.js index 7d1f3c49d4..5e12046eab 100644 --- a/test/unit/llm-events/langchain/vector-search-result.test.js +++ b/test/unit/llm-events/langchain/vector-search-result.test.js @@ -7,6 +7,7 @@ const tap = require('tap') const LangChainVectorSearchResult = require('../../../../lib/llm-events/langchain/vector-search-result') +const LangChainVectorSearch = require('../../../../lib/llm-events/langchain/vector-search') tap.beforeEach((t) => { t.context._tx = { @@ -44,6 +45,9 @@ tap.beforeEach((t) => { transaction: { id: 'tx-1', traceId: 'trace-1' + }, + getDurationInMillis() { + return 42 } } @@ -52,12 +56,19 @@ tap.beforeEach((t) => { }) tap.test('create entity', async (t) => { - const search = new LangChainVectorSearchResult({ + const search = new LangChainVectorSearch({ + ...t.context, + query: 'hello world', + k: 1 + }) + + const searchResult = new LangChainVectorSearchResult({ ...t.context, sequence: 1, - pageContent: 'hello world' + pageContent: 'hello world', + search_id: search.id }) - t.match(search, { + t.match(searchResult, { id: /[a-z0-9-]{36}/, appName: 'test-app', ['llm.conversation_id']: 'test-conversation', @@ -71,7 +82,7 @@ tap.test('create entity', async (t) => { virtual_llm: true, sequence: 1, page_content: 'hello world', - search_id: /[a-z0-9-]{36}/ + search_id: search.id }) }) diff --git a/test/unit/llm-events/openai/error.test.js b/test/unit/llm-events/openai/error.test.js index 1deff0925a..24d4a634d2 100644 --- a/test/unit/llm-events/openai/error.test.js +++ b/test/unit/llm-events/openai/error.test.js @@ -18,7 +18,8 @@ tap.test('LlmErrorMessage', (t) => { 'error.code': 'insufficient_quota', 'error.param': 'test-param', 'completion_id': undefined, - 'embedding_id': undefined + 'embedding_id': undefined, + 'vector_store_id': undefined } t.same(errorMsg, expected) t.end() diff --git a/test/versioned/langchain/common.js b/test/versioned/langchain/common.js index a6ff7e64d1..f72da9e272 100644 --- a/test/versioned/langchain/common.js +++ b/test/versioned/langchain/common.js @@ -14,13 +14,62 @@ function filterLangchainEvents(events) { }) } -function filterLangchainMessages(events, msgType) { +function filterLangchainEventsByType(events, msgType) { return events.filter((event) => { const [{ type }] = event return type === msgType }) } +function assertLangChainVectorSearch({ tx, vectorSearch, responseDocumentSize }) { + const expectedSearch = { + 'id': /[a-f0-9]{36}/, + 'appName': 'New Relic for Node.js tests', + 'span_id': tx.trace.root.children[0].id, + 'trace_id': tx.traceId, + 'transaction_id': tx.id, + 'request.k': 1, + 'request.query': 'This is an embedding test.', + 'ingest_source': 'Node', + 'vendor': 'langchain', + 'virtual_llm': true, + ['response.number_of_documents']: responseDocumentSize, + 'duration': tx.trace.root.children[0].getDurationInMillis() + } + + this.equal(vectorSearch[0].type, 'LlmVectorSearch') + this.match(vectorSearch[1], expectedSearch, 'should match vector search') +} + +function assertLangChainVectorSearchResult({ tx, vectorSearchResult, vectorSearchId }) { + const baseSearchResult = { + 'id': /[a-f0-9]{36}/, + 'search_id': vectorSearchId, + 'appName': 'New Relic for Node.js tests', + 'span_id': tx.trace.root.children[0].id, + 'trace_id': tx.traceId, + 'transaction_id': tx.id, + 'ingest_source': 'Node', + 'vendor': 'langchain', + 'metadata.id': '2', + 'virtual_llm': true + } + + vectorSearchResult.forEach((search) => { + const expectedChatMsg = { ...baseSearchResult } + if (search[1].sequence === 0) { + expectedChatMsg.sequence = 0 + expectedChatMsg.page_content = 'This is an embedding test.' + } else if (search[1].sequence === 1) { + expectedChatMsg.sequence = 1 + expectedChatMsg.page_content = '212 degrees Fahrenheit is equal to 100 degrees Celsius.' + } + + this.equal(search[0].type, 'LlmVectorSearchResult') + this.match(search[1], expectedChatMsg, 'should match vector search result') + }) +} + function assertLangChainChatCompletionSummary({ tx, chatSummary, withCallback }) { const expectedSummary = { 'id': /[a-f0-9]{36}/, @@ -93,8 +142,10 @@ function assertLangChainChatCompletionMessages({ tap.Test.prototype.addAssert('langchainMessages', 1, assertLangChainChatCompletionMessages) tap.Test.prototype.addAssert('langchainSummary', 1, assertLangChainChatCompletionSummary) +tap.Test.prototype.addAssert('langchainVectorSearch', 1, assertLangChainVectorSearch) +tap.Test.prototype.addAssert('langchainVectorSearchResult', 1, assertLangChainVectorSearchResult) module.exports = { filterLangchainEvents, - filterLangchainMessages + filterLangchainEventsByType } diff --git a/test/versioned/langchain/package.json b/test/versioned/langchain/package.json index 34ff9d2b19..669c21f129 100644 --- a/test/versioned/langchain/package.json +++ b/test/versioned/langchain/package.json @@ -12,12 +12,15 @@ }, "dependencies": { "@langchain/core": ">=0.1.17", - "@langchain/openai": "latest" + "@langchain/openai": "latest", + "@langchain/community": "latest", + "@elastic/elasticsearch": "latest" }, "files": [ "tools.tap.js", "runnables.tap.js", - "runnables-streaming.tap.js" + "runnables-streaming.tap.js", + "vectorstore.tap.js" ] } ] diff --git a/test/versioned/langchain/runnables-streaming.tap.js b/test/versioned/langchain/runnables-streaming.tap.js index d866890ad3..f94751ec2f 100644 --- a/test/versioned/langchain/runnables-streaming.tap.js +++ b/test/versioned/langchain/runnables-streaming.tap.js @@ -9,7 +9,7 @@ const tap = require('tap') const helper = require('../../lib/agent_helper') // load the assertSegments assertion require('../../lib/metrics_helper') -const { filterLangchainEvents, filterLangchainMessages } = require('./common') +const { filterLangchainEvents, filterLangchainEventsByType } = require('./common') const { version: pkgVersion } = require('@langchain/core/package.json') const createOpenAIMockServer = require('../openai/mock-server') const mockResponses = require('../openai/mock-responses') @@ -135,11 +135,11 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -180,11 +180,11 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -229,11 +229,11 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -322,11 +322,11 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -429,7 +429,7 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) diff --git a/test/versioned/langchain/runnables.tap.js b/test/versioned/langchain/runnables.tap.js index ab026fa40b..e4f6f7ec57 100644 --- a/test/versioned/langchain/runnables.tap.js +++ b/test/versioned/langchain/runnables.tap.js @@ -9,7 +9,7 @@ const tap = require('tap') const helper = require('../../lib/agent_helper') // load the assertSegments assertion require('../../lib/metrics_helper') -const { filterLangchainEvents, filterLangchainMessages } = require('./common') +const { filterLangchainEvents, filterLangchainEventsByType } = require('./common') const { version: pkgVersion } = require('@langchain/core/package.json') const createOpenAIMockServer = require('../openai/mock-server') const config = { @@ -116,11 +116,11 @@ tap.test('Langchain instrumentation - runnable sequence', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -155,11 +155,11 @@ tap.test('Langchain instrumentation - runnable sequence', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -198,11 +198,11 @@ tap.test('Langchain instrumentation - runnable sequence', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -280,11 +280,11 @@ tap.test('Langchain instrumentation - runnable sequence', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) - const langChainSummaryEvents = filterLangchainMessages( + const langChainSummaryEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionSummary' ) @@ -374,7 +374,7 @@ tap.test('Langchain instrumentation - runnable sequence', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainMessages( + const langChainMessageEvents = filterLangchainEventsByType( langchainEvents, 'LlmChatCompletionMessage' ) diff --git a/test/versioned/langchain/vectorstore.tap.js b/test/versioned/langchain/vectorstore.tap.js new file mode 100644 index 0000000000..2356e7f52b --- /dev/null +++ b/test/versioned/langchain/vectorstore.tap.js @@ -0,0 +1,257 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const tap = require('tap') +const helper = require('../../lib/agent_helper') +// load the assertSegments assertion +require('../../lib/metrics_helper') +const { version: pkgVersion } = require('@langchain/core/package.json') +const createOpenAIMockServer = require('../openai/mock-server') +const { filterLangchainEvents, filterLangchainEventsByType } = require('./common') +const { DESTINATIONS } = require('../../../lib/config/attribute-filter') +const params = require('../../lib/params') +const { Document } = require('@langchain/core/documents') + +const config = { + ai_monitoring: { + enabled: true + }, + feature_flag: { + langchain_instrumentation: true + } +} + +tap.test('Langchain instrumentation - vectorstore', (t) => { + t.autoend() + + t.beforeEach(async (t) => { + const { host, port, server } = await createOpenAIMockServer() + t.context.server = server + t.context.agent = helper.instrumentMockedAgent(config) + const { OpenAIEmbeddings } = require('@langchain/openai') + + const { Client } = require('@elastic/elasticsearch') + const clientArgs = { + client: new Client({ + node: `http://${params.elastic_host}:${params.elastic_port}` + }) + } + const { ElasticVectorSearch } = require('@langchain/community/vectorstores/elasticsearch') + + t.context.embedding = new OpenAIEmbeddings({ + openAIApiKey: 'fake-key', + configuration: { + baseURL: `http://${host}:${port}` + } + }) + const docs = [ + new Document({ + metadata: { id: '2' }, + pageContent: 'This is an embedding test.' + }) + ] + const vectorStore = new ElasticVectorSearch(t.context.embedding, clientArgs) + await vectorStore.deleteIfExists() + await vectorStore.addDocuments(docs) + t.context.vs = vectorStore + }) + + t.afterEach(async (t) => { + t.context?.server?.close() + helper.unloadAgent(t.context.agent) + // bust the require-cache so it can re-instrument + Object.keys(require.cache).forEach((key) => { + if ( + key.includes('@langchain/core') || + key.includes('openai') || + key.includes('@elastic') || + key.includes('@langchain/community') + ) { + delete require.cache[key] + } + }) + }) + + t.test('should create vectorstore events for every similarity search call', (t) => { + const { agent, vs } = t.context + + helper.runInNamedTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const events = agent.customEventAggregator.events.toArray() + t.equal(events.length, 3, 'should create 3 events') + + const langchainEvents = events.filter((event) => { + const [, chainEvent] = event + return chainEvent.vendor === 'langchain' + }) + + t.equal(langchainEvents.length, 2, 'should create 2 langchain events') + + tx.end() + t.end() + }) + }) + + t.test('should create span on successful vectorstore create', (t) => { + const { agent, vs } = t.context + helper.runInTransaction(agent, async (tx) => { + const result = await vs.similaritySearch('This is an embedding test.', 1) + t.ok(result) + t.assertSegments(tx.trace.root, ['Llm/vectorstore/Langchain/similaritySearch'], { + exact: false + }) + tx.end() + t.end() + }) + }) + + t.test('should increment tracking metric for each langchain vectorstore event', (t) => { + const { agent, vs } = t.context + + helper.runInTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const metrics = agent.metrics.getOrCreateMetric( + `Supportability/Nodejs/ML/Langchain/${pkgVersion}` + ) + t.equal(metrics.callCount > 0, true) + + tx.end() + t.end() + }) + }) + + t.test( + 'should create vectorstore events for every similarity search call with embeddings', + (t) => { + const { agent, vs } = t.context + + helper.runInNamedTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const events = agent.customEventAggregator.events.toArray() + const langchainEvents = filterLangchainEvents(events) + + const vectorSearchResultEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmVectorSearchResult' + ) + + const vectorSearchEvents = filterLangchainEventsByType(langchainEvents, 'LlmVectorSearch') + + t.langchainVectorSearch({ + tx, + vectorSearch: vectorSearchEvents[0], + responseDocumentSize: 1 + }) + t.langchainVectorSearchResult({ + tx, + vectorSearchResult: vectorSearchResultEvents, + vectorSearchId: vectorSearchEvents[0][1].id + }) + + tx.end() + t.end() + }) + } + ) + + t.test( + 'should create only vectorstore search event for similarity search call with embeddings and invalid metadata filter', + (t) => { + const { agent, vs } = t.context + + helper.runInNamedTransaction(agent, async (tx) => { + // search for documents with invalid filter + await vs.similaritySearch('This is an embedding test.', 1, { + a: 'some filter' + }) + + const events = agent.customEventAggregator.events.toArray() + const langchainEvents = filterLangchainEvents(events) + + const vectorSearchResultEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmVectorSearchResult' + ) + + const vectorSearchEvents = filterLangchainEventsByType(langchainEvents, 'LlmVectorSearch') + + // there are no documents in vector store with that filter + t.equal(vectorSearchResultEvents.length, 0, 'should have 0 events') + t.langchainVectorSearch({ + tx, + vectorSearch: vectorSearchEvents[0], + responseDocumentSize: 0 + }) + + tx.end() + t.end() + }) + } + ) + + t.test('should not create vectorstore events when not in a transaction', async (t) => { + const { agent, vs } = t.context + + await vs.similaritySearch('This is an embedding test.', 1) + + const events = agent.customEventAggregator.events.toArray() + t.equal(events.length, 0, 'should not create vectorstore events') + t.end() + }) + + t.test('should add llm attribute to transaction', (t) => { + const { agent, vs } = t.context + + helper.runInTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + t.equal(attributes.llm, true) + + tx.end() + t.end() + }) + }) + + t.test('should create error events', (t) => { + const { agent, vs } = t.context + + helper.runInNamedTransaction(agent, async (tx) => { + try { + await vs.similaritySearch('Embedding not allowed.', 1) + } catch (error) { + t.ok(error) + } + + const events = agent.customEventAggregator.events.toArray() + // Only LlmEmbedding and LlmVectorSearch events will be created + // LangChainVectorSearchResult event won't be created since there was an error + t.equal(events.length, 2, 'should create 2 events') + + const langchainEvents = events.filter((event) => { + const [, chainEvent] = event + return chainEvent.vendor === 'langchain' + }) + + t.equal(langchainEvents.length, 1, 'should create 1 langchain vectorsearch event') + t.equal(langchainEvents[0][1].error, true) + + // But, we should also get two error events: 1xLLM and 1xLangChain + const exceptions = tx.exceptions + for (const e of exceptions) { + const str = Object.prototype.toString.call(e.customAttributes) + t.equal(str, '[object LlmErrorMessage]') + } + + tx.end() + t.end() + }) + }) +}) diff --git a/test/versioned/openai/mock-responses.js b/test/versioned/openai/mock-responses.js index 5a385abba3..9692540d8c 100644 --- a/test/versioned/openai/mock-responses.js +++ b/test/versioned/openai/mock-responses.js @@ -76,8 +76,8 @@ responses.set('This is an embedding test.', { body: { data: [ { - embedding: - 'SLewvFF6iztXKj07UOCQO41IorspWOk79KHuu12FrbwjqLe8FCTnvBKqj7sz6bM8qqUEvFSfITpPrJu7uOSbPM8agzyYYqM7YJl/PBF2mryNN967uRiRO9lGcbszcuq7RZIavAnnNLwWA5s8mnb1vG+UGTyqpYS846PGO2M1X7wIxAO8HfgFvc8s8LuQXPQ5qgsKPOinEL15ndY8/MrOu1LRMTxCbQS7PEYJOyMx7rwDJj+79dVjO5P4UzmoPZq8jUgivL36UjzA/Lc8Jt6Ru4bKAL1jRiM70i5VO4neUjwneAy7mlNEPBVpoDuayo28TO2KvAmBrzzwvyy8B3/KO0ZgCry3sKa6QTmPO0a1Szz46Iw87AAcPF0O5DyJVZw8Ac+Yu1y3Pbqzesw8DUDAuq8hQbyALLy7TngmPL6lETxXxLc6TzXSvKJrYLy309c8OHa0OU3NZ7vru2K8mIXUPCxrErxLU5C5s/EVPI+wjLp7BcE74TvcO+2aFrx4A9w80j+Zu/aAojwmzU08k/hTvBpL4rvHFFQ76YftutrxL7wyxgK9BsIevLkYkTq4B028OZnlPPkcgjxhzfS79oCiuB34BbwITTq97nrzOugwRzwGS1U7CqTgvFxROLx4aWG7E/DxPA3J9jwd+AU8dVWPvGlc2jzwWae57nrzu569E72GU7e8Vn9+vFLA7TtVbZE8eOCqPG+3Sjxr5/W8s+DRPE+sm7wFKKQ8A8A5vUSBVryeIxk8hsqAPAeQjryeIxm8gU/tuxVpoDxVXM250GDlOlEDwjs0t6O8Tt6rOVrGHLvmyFy6dhI7PLPxlbv3YP88B/YTPEZgCrxqKsq8Xh+ou96wQLp5rpo8LSg+vL63/rsFjqk8E/DxPEi3MDzTcw66PjcqPNgSfLwqnaK85QuxPI7iHL2+pRE8Z+ICOxzEELvph+07jHqyu2ltnrwNQMC82BL8vAOdiDwSqo88CLM/PCKFBrzmP6a85Nc7PBaM0bvh1VY7NB2pvMkF9Tx3New87mgGPAoKZjo+nS+/Rk/GucqwMz3fwYS8yrCzPMo56jyDHV08XLe9vB4+aLwXwMY8dVUPvCFATbx2eMC8V7NzvEnrpTsIxIO7yVmNu2lc2ryGQnM8A6/1PH/VFbySO6g80i5VPOY/prv6cyi7W5QMPJVP+jsyLIi84H6wPKM50DrZNIS8UEaWPPrIaTzvrmg8rcoaPRuQm7ysH9y8OxIUO7ss4zq3Od08paG6vAPAuTjYAI88/qmCuuROhbzBMK08R4M7u67+j7uClKa6/KedOsqNArzysM08QJ8UvMD8t7v5P7M799fIvAWx2jxiEi48ja6nPL0LFzxFkpq7LAWNPA1AQLyWlLO6qrfxvOGypTxJUau8aJ8uPceLnTtS0TG9omtgPO7xPDvzbfm7FfJWu2CqwzwAASk96FN4PLPgUbwRdhq8Vn9+PLk7wjs8NUW84yx9vHJCZjzysM079hodO/NbDL2BxrY6CE26OzpEpDv7DaM8y0quO41IIr1+Kte8QdMJvKlxDzy9+lI8hfyQPA3J9jzWmKS7z6O5u4a5vLtXKj088XzYO1fEtzwY4/e7Js1NugbCnjymxOu7906SvPSPAb1ieDO8dnjAu/EW0zp/b5C8mGIjvWTPWTwIxIM8YgFqPKvrZrwKpOA7/jK5O2vViDyfaXs8DR2Pu0AFGrvTc446IIOhvDreHrxRnTw8ROdbu55Gyrsht5Y8tVmAvHK5rzzZvTo8bx1QPMglmLvigBU8oIuDvAFYz7pblIw8OZnlOsTvPbxhzfS8BxnFOpkwE72E60w7cNp7utp6ZrtvHdC4uwmyO5dRX7sAm6M7kqEtvElRK7yWg++7JHanvM6ACDvrZqG8Xh+oupQsyTwkZWO8VzuBu5xVKbzEZoc7wB9pvA796zyZlpi8YbsHvQs+W7u9cZy8gKMFOxYDGzyu7Uu71KeDPJxVqbxwyI68VpDCu9VT67xKqFG7KWmtuvNteTocs0w7aJ8uPMUSbzz6cyg8MiwIPEtlfTo+wOA75tkgu7VZgDw8WPa8mGIjPKq38bsr0Zc7Ot4evNNiyju9C5c7YCENPP6pAj3uV8I7X3bOusfxIjvpZLy655bMvL9ivbxO3iu8NKbfPNe7VTz9ZMk88RZTu5QsybxeQtk7qpTAOzGSjTxSwO27mGIjPO7OC7x7FoW8wJayvI2uJzttxqk84H4wOUtlfbxblAw8uTtCPIO3Vzxkz9k8ENwfvfQYuLvHFNQ8LvatPF65ojzPLHA8+RyCvK3Kmjx27wk8Dcn2PARatDv3tBc8hkLzPEOz5jyQSoe8gU/tPMRmhzzp2wU90shPPBv2oLsNQMA8jTdevIftMTt/Xsw7MMQdPICjBT012tS7SLewvJBtuDuevZM8LyojPa6HxjtOAd07v9mGusZXqDoPqKo8qdeUvETnW7y5occ5pOSOvPPkwjsDN4O8Mk85vKnXlDtp06O7kZDpO6GuNDtRFAY9lAkYPGHNdDx2Afc7RRtROy5/5LyUoxI9mu0+u/dOEryrYrC867vivJp29TtVbZG8SVGrO0im7LnhsqU80frfPL/IwryBT+07/+/kPLZ8sTwoNbg7ZkiIOxadlbxlnUm68RbTuxkX7Tu/cwG7aqGTPO8CAbzTYsq6AIpfvA50tbzllOc7s3rMO0SBVjzXzJm8eZ3Wu4vgtzwPDrA8W6b5uwJpEzwLtaQ81pgkPJuqarxmro288369u48WkjwREBU9JP/dPJ69kzvw4t27h3bouxhrBbwrNx29F9EKPFmSJ7v8px08Tt6rvEJthLxon648UYz4u61TUTz4lPQ7ERAVuhwqFrzfSjs8RRtRO6lxD7zHelm87lfCu10O5LrXMh886YftvL9iPTxCf/E6MZKNOmAhDb2diZ47eRSgPBfRCrznlsw5MiwIvHW7FD3tI807uG3SPE7eqzx1VY864TtcO3zTMDw7EhS8c+0kPLr47TvUDQm8domEvEi3MLruaAa7tUi8u4FgsTwbkBu6pQfAvEJthLwDnQg8S1OQO55GSrxZLCK8nkZKvFXTFr01dM+8W6Z5vO+u6Luh0eW8rofGvFsdw7x7KHK8sN5svCFAzbo/0SS8f9UVu7Qli7wr0Re95E4FvSg1ODok/907AAGpPHQhGrwtS++71pgkvCtazjsSzcC7exYFPLVZgLzZmom7W6Z5PHr0fLtn9O86oUivukvcRrzjPcE8a8REPAei+zoBNZ685aUrPNBg5bqeIxk8FJuwPPdOkrtUOZy8GRftO4KD4rz/72Q7ERCVu8WJODy5O8I5L7NZuxJECjxFkpq8Uq4AOy2fh7wY9Du8GRdtu48o/7mHdug803MOvCUQIrw2hZM8v+tzvE54pruyI6a6exYFvDXrGDwNQEA8zyxwO7c53TwUJGe8Wk9Tu6ouu7yqCwo8vi7IvNe71TxB04m8domEvKTkDrzsidK8+nOovLfT1zr11eM7SVErO3EOcbzqMqw74Tvcut4WRrz5pbi8oznQvMi/Er0aS+I87lfCvK+qdztd6zI83eJQPFy3vbyACQu9/8wzO/k/s7weG7e8906SPA3J9jw8NUU8TUQxPfEWU7wjH4E8J3gMPC72LTp6SJU8exaFOXBiibyf4MS6EXYaO3DIjjy61by7ACRaO5NvnTvMGB48Dw6wPFEUBr30j4E7niMZvIZC87s7EpS8OZnlPJZxgrxug9U7/DDUvNrxL7yV14e3E2c7PBdaQTwT8HE8oIuDPGIB6rvMB9o6cR+1OwbCHrylfgm8z6M5vIiqXbxFG1G8a9WIPItp7rpGT8Y838GEvAoK5jyAG3g7xRJvPPxBGLzJWQ28XYWtO85vRLp0IZq8cR81vc7mDb28PSe89LKyuig1uDyxEuK8GlwmPIbKgLwHGcW7/qkCvC8ZXzzSyE89F8BGOxPw8Tx+Ktc8BkvVurXiNryRkOk8jyj/OcKH0zp69Pw8apDPPFuUjLwPDrC8xuBeuD43KrxuYKQ7qXGPvF0OZDx1VQ88VVzNvD9rn7ushWE7EZlLvSL9+DrHi528dzXsu3k30bzeFka7hrm8vD3gAz1/Xsy80D20PNPZE7sorAG86WS8u2Y3xDtvHVC7PKwOO5DkAT3KOeo8c+0kvI+fyLuY61k8SKbsO4TrzLrrZqE87O9XvMkF9Tynb6q847SKvBjjdzyhSK88zTtPPNNzjjsvGV87UQPCvMD8t7stn4e7GRftPBQkZ7x4eiW7sqzcu3ufO7yAG3g8OHa0u0T4n7wcxJC7r6r3vAbCnrth3rg7BxnFumqQzzyXyCi8V8Q3vEPEqjyIu6E8Ac+YvGR6GLulkHY8um83PMqNgrv5pTi8N7kIPOhTeLy6TIY8B5COvDLGArvEzAy9IbcWvIUfQjxQ4BC7B/aTvCfwfrz15ie8ucR4PD1pursLtSS8AgMOOzIsiLv0srI7Q01hPCvRF7vySsg6O5tKunh6JTvCZCI7xuDevLc53btvLhQ8/pi+PJU9Dbugi4O8Qn/xvLpMhrth3ji8n/GIPKouu7tBS3y853MbPGAQyTt27wk7iokRO8d62bzZRnG7sN5svAG+1Lqvqve8JGXjur0Ll7tCf/E75/xRPIWFx7wgDNi8ucT4OZNvHb2nktu8qrfxuyR2J7zWh2A6juKcPDhlcLx/1RU9IAxYPGJ4szylB8C8qfrFO276HjuWcQK9QdOJvCUQIjzjo8a8SeslvBrCKztCf/E66MrBOx1eCz2Xt+Q66YdtvKg9mrrLSq47fFznO1uUjDsoNTg8QyqwuzH4Ejz/Zi67A8A5uKg9GrtFkhq862ahOzSmXzkMDEs8q+vmvNVkLzwc1n28mu0+vCbekTyCg+K7ekgVvO8CAT2yRtc8apBPu1b2R7zUp4M8VW2RvPc9zrx69Hw753ObvCcSB71sG+u8OwHQuv67b7zLSi65HrWxO0ZPRrxmwPq7t7CmPGxvAzygnfC8oIsDvKY7tbwZF+07p2+qvOnbhbv0oW47/2auuThlcDwIxIM8n/EIO6ijH7vHetk7uRiRPGUDT7pgh5I85shcPpGQabykShS7FWmgPPjojDvJ8wc8mlPEOY2uJzt7FoW7HNb9O7rVvDzKjQI80NcuuqvINbvNTBO8TgFdvEJ/cbzEZoe8SVGrvMvkqLyHdui7P2ufvBSbMDw0t6O82GaUPOLmGrxSNze8KVjpuwizPzwqjN48Xh8ovE4B3TtiAeo8azsOO8eLnbyO4py7x/GiPIvgNzzvi7c8BFq0O/dOEj1fU5282ZoJPCL9+LqyIyY8IoUGPNI/mbwKpGC7EkQKuzrN2jwVzyU7QpA1vLIjpjwi64s8HYE8u6eSW7yryLU8yK5OOzysjjwi6wu8GsIrOu7xPDwCaRO8dzVsPP/vZLwT3oQ8cQ7xvOJv0TtWBww8hlM3PBPeBDxT9OK71pgkPPSysrugiwO90GDlvHOHHz3xfNg8904SPVpglzzmP6a7Cgrmu9/BBLyH7bG85QsxvVSfIb2Xt2Q8paG6vOqYsTos9Mi8nqxPu8wHWjuYhdS7GAWAvCIOvTp/bxA8j7CMPG1P4Dxd67I7xxRUvOM9wbxMhwU9Kp0iPfF82LvQYOU6XkJZPBxNx7y0nX28B5COO8FT3rp4eiW8R/oEvSfw/jtC9rq8n/GIux3nQTw8WPY8LBf6uzSmXzzSPxm88rDNvDysDjwyPnW7tdFyPBLNwDo8WHa8bPi5vOO0CrylGAQ8YgFqvEFLfDy7LOO7TIeFPAHPmDv3YP+6/+9kPBKqjzt5rpo8VJ+hvE7eKzyc3t88P2sfvLQUR7wJ1vC6exaFvD6dr7zNO888i+A3ulwuhzuF/JC8gKMFveoyLLxqBxk7YgFquws+2zwOUYS8agcZvGJ4M71AjtC747QKvAizP73UH3a7LvatPJBtuLzEzIy8bG8DvJEHM75E59s7zbIYPObZIL2uZJW7WRveugblTzy6TIa802JKvD9rH7xlA088QAWavIFP7bwL2FW8vqWRu0ZgijyRkGm7ZGnUvIeHLD1c2m48THbBPPkcAr1NzWc8+JT0uulkvLvXMp+7lU96u7kYET1xhTo8e3wKvItGPTxb+hG87mgGPWqhk7uhrrQ73rBAPCbNTT13rDW8K8DTus8s8DsNt4k8gpQmPLES4ryyvSA8lcbDO60woDyLVwE9BFq0u+cNFj3C7Vi8UXoLPDYOyryQ0z083+S1Ox34hTzEzIw7pX4Ju6ouuzxIpmw8w5iXuylYaTy5sgu9Js3NOo+fyLyjFp+8MMSdvOROBb2n+OA7b7fKOeIJzDoNpkW8WsYct7SdfTxXxLc7TO2KO3YB9zynktu7OkSkPKnXFLvtRv47AJujuzGSDT0twjg8AgOOO4d26DvpZDy8lAkYPI5r0zcGS9W8OGXwu9xIVjyH7TG9IUDNuiqMXrwb9qA79I+BPL1xHLuVPY07MOfOO0ztCruvMoW8BuXPu4AbeLyIRNg8uG3SPO5XQjuFH0K8zm9EPEAoSz0tKL652ZqJOgABqbwsjsM8mlPEPLewpjsVWNw8OGXwOlYHjLzfwQQ81iFbOyJ0Qj3d85S7cQ7xvIqswjxKhSC7906SvAFYz72xiau8LAWNPB1eCz09jGu72ZoJPfDiXTwPDrA8CYGvvNH6XzxTa6y8+RwCvY8of7xxDnG8Ef/QvJ9p+zqh0eU8a16/OzBN1LyDLiE9PFh2u+0jTbxLUxA9ZZ3JvItXgbqL4Dc8BuXPvKnXFDzmPyY8k/hTOlum+bqAksG8OZnluPmluLxRnTy6/KcdvKAUOrzRcSm8fqEgPcTeebzeOXc8KCR0OnN2W7xRA0K8Wsacu+M9wToyLIi8mTATu21P4LuadvW8Dtq6vPmlODsjqLe88ieXPJEHszySoa08U/RiPNQNCbwb9qC8bG+DOXW7FL0OdLW7Tc3nvG8dULsAJNo7fNMwO7sJMr2O4hy85ZTnuwAkWjw+Nyq8rcoaO+8lsrvx86E8U/TivGUUkzp6SJW8lT0NvWz4uTzeFka6qguKvIKD4rt/1ZU8LBf6vD6dr7es/Ko7qWBLvIlVHDxwUUU6Jt4RvRJEijnRcSk88235PGvVCL3zbfm8DaZFO+7xvLs3qES8oznQO9XKNDxZLKK8IIMhvComWb0CAw48fDk2O+nbBb29C5e8ogVbu1EUBryYhdS7OTPgOul1AD25sgs7i1cBPBYmzLtSroA8hfyQvP3bErz9h/o82ZoJO7/ZhjxtT+A8UZ28uzaFk7wJ1nA6dd7FPGg5Kbwb9iC8psRrvBXyVjzGRuS8uAfNu0+smzvFAAK96FN4vC2fhzy65oC7tgXou/9mLjxMELw8GSgxPRBlVjxDxCq80j8ZveinkDxHgzu70j8ZvPGNnDyPn0i8Vn9+urXR8ju10fI7sRJiPDBemLt8OTa8tJ39O4ne0rsaXKa7t0ohPHQhGrdYXjI824sqvDw1RT2/2YY8E/BxPIUOfjv9dQ08PM8/PMwYHrwwXpi7nqxPPM8aA7w+wOC7ROdbO79iPTxVbRE8U45dPOOjRjxwYok8ME1Uu1SfIbyifKQ8UXqLPI85wzsITTq8R+lAPMRVQzzcv58892B/Oqg9mjw3MXu7P9EkvM6AiLyx7zA8eHolPLYWLLugFLq8AJsjvEOzZjk6RKQ8uRgRPXVVjzw0HSk9PWk6PLss47spzzK93rBAvJpTxDun+OC7OTPgvEa1yzvAH+k5fZDcOid4jLuN0di8N7kIPPe0F7wVaSC8zxoDvJVgvrvUpwO9dd7FPKUHQLxn4oI7Ng7KPIydYzzZRvE8LTkCu3bvCTy10fK7QAWaPGHeOLu6+O27omvgO8Rmh7xrXj87AzeDvORg8jnGRuS8UEYWPLPg0TvYZpQ9FJuwPLC7O7xug1U8bvoevAnW8DvxFtM8kEoHPDxYdrzcWZq8n3q/O94nCjvZI0C82yUlvayWpbyHh6y7ME1UO9b+KTzbFGG89oCiPFpgFzzhTKA84gnMPKgsVjyia+C7XNpuPHxc5zyDLqG8ukyGvKqUQLwG5U88wB/pO+B+ML2O4py8MOdOPHt8irsDnYg6rv6PumJ4szzuV0I80qWePKTkDj14A9y8fqEgu9DXLjykbUU7yEhJvLYFaLyfVw68', + // a small sample of a real embedding response + embedding: [-0.021616805, 0.004173375, 0.002796262, 0.004489489, -0.004940119], index: 0, object: 'embedding' } diff --git a/test/versioned/openai/mock-server.js b/test/versioned/openai/mock-server.js index 7f03f7ba48..47cdb0b431 100644 --- a/test/versioned/openai/mock-server.js +++ b/test/versioned/openai/mock-server.js @@ -153,5 +153,9 @@ function getShortenedPrompt(reqBody) { const prompt = reqBody.prompt || reqBody.input || reqBody.messages.map((m) => m.content).join('\n') + if (Array.isArray(prompt)) { + return prompt[0] + } + return prompt.split('\n')[0] }