Skip to content

Commit db23e7f

Browse files
sabrennersabrenner
authored
ci(llmobs): resolve some test flakiness (#6316)
1 parent a8b2284 commit db23e7f

File tree

7 files changed

+154
-18
lines changed

7 files changed

+154
-18
lines changed

‎packages/dd-trace/src/llmobs/plugins/ai/index.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ class VercelAILLMObsPlugin extends BaseLLMObsPlugin {
8484
findToolName (toolDescription) {
8585
for (const availableTool of this.#availableTools) {
8686
const description = availableTool.description
87-
if (description === toolDescription) {
87+
if (description === toolDescription && availableTool.id) {
8888
return availableTool.id
8989
}
9090
}

‎packages/dd-trace/test/llmobs/cassettes/azure-openai/azure-openai_chat_completions_post_782ff5d3.yaml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
interactions:
2+
- request:
3+
body: "{\n \"model\": \"gpt-4.1\",\n \"messages\": [\n {\n \"role\":
4+
\"system\",\n \"content\": \"You are a helpful assistant.\"\n },\n {\n
5+
\ \"role\": \"user\",\n \"content\": \"Hello, OpenAI!\"\n }\n ],\n
6+
\ \"temperature\": 0.5,\n \"stream\": false,\n \"max_tokens\": 100,\n \"n\":
7+
1,\n \"user\": \"dd-trace-test\"\n}"
8+
headers:
9+
? !!python/object/apply:multidict._multidict.istr
10+
- Accept
11+
: - application/json
12+
? !!python/object/apply:multidict._multidict.istr
13+
- Accept-Encoding
14+
: - gzip,deflate
15+
? !!python/object/apply:multidict._multidict.istr
16+
- Connection
17+
: - keep-alive
18+
Content-Length:
19+
- '297'
20+
? !!python/object/apply:multidict._multidict.istr
21+
- Content-Type
22+
: - application/json
23+
? !!python/object/apply:multidict._multidict.istr
24+
- User-Agent
25+
: - OpenAI/JS 4.0.0
26+
? !!python/object/apply:multidict._multidict.istr
27+
- X-Stainless-Arch
28+
: - arm64
29+
? !!python/object/apply:multidict._multidict.istr
30+
- X-Stainless-Lang
31+
: - js
32+
? !!python/object/apply:multidict._multidict.istr
33+
- X-Stainless-OS
34+
: - MacOS
35+
? !!python/object/apply:multidict._multidict.istr
36+
- X-Stainless-Package-Version
37+
: - 4.0.0
38+
? !!python/object/apply:multidict._multidict.istr
39+
- X-Stainless-Runtime
40+
: - node
41+
? !!python/object/apply:multidict._multidict.istr
42+
- X-Stainless-Runtime-Version
43+
: - v22.17.0
44+
method: POST
45+
uri: https://dd.openai.azure.com/chat/completions
46+
response:
47+
body:
48+
string: '{"error":{"code":"404","message": "Resource not found"}}'
49+
headers:
50+
Content-Length:
51+
- '56'
52+
Content-Type:
53+
- application/json
54+
Date:
55+
- Wed, 20 Aug 2025 13:52:23 GMT
56+
Strict-Transport-Security:
57+
- max-age=31536000; includeSubDomains; preload
58+
apim-request-id:
59+
- 49a7f7ae-f881-4784-816e-855d03e730d8
60+
x-content-type-options:
61+
- nosniff
62+
status:
63+
code: 404
64+
message: Resource Not Found
65+
version: 1

‎packages/dd-trace/test/llmobs/cassettes/azure-openai/azure-openai_openai_deployments_gpt-4.1_chat_completions_api-version_2024-05-01-preview_post_e5b3df5e.yaml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
interactions:
2+
- request:
3+
body: "{\n \"model\": \"gpt-4.1\",\n \"messages\": [\n {\n \"role\":
4+
\"system\",\n \"content\": \"You are a helpful assistant.\"\n },\n {\n
5+
\ \"role\": \"user\",\n \"content\": \"Hello, OpenAI!\"\n }\n ],\n
6+
\ \"temperature\": 0.5,\n \"stream\": false,\n \"max_tokens\": 100,\n \"n\":
7+
1,\n \"user\": \"dd-trace-test\"\n}"
8+
headers:
9+
? !!python/object/apply:multidict._multidict.istr
10+
- Accept
11+
: - application/json
12+
? !!python/object/apply:multidict._multidict.istr
13+
- Accept-Encoding
14+
: - gzip,deflate
15+
? !!python/object/apply:multidict._multidict.istr
16+
- Connection
17+
: - keep-alive
18+
Content-Length:
19+
- '297'
20+
? !!python/object/apply:multidict._multidict.istr
21+
- Content-Type
22+
: - application/json
23+
? !!python/object/apply:multidict._multidict.istr
24+
- User-Agent
25+
: - AzureOpenAI/JS 4.104.0
26+
? !!python/object/apply:multidict._multidict.istr
27+
- x-stainless-arch
28+
: - arm64
29+
? !!python/object/apply:multidict._multidict.istr
30+
- x-stainless-lang
31+
: - js
32+
? !!python/object/apply:multidict._multidict.istr
33+
- x-stainless-os
34+
: - MacOS
35+
? !!python/object/apply:multidict._multidict.istr
36+
- x-stainless-package-version
37+
: - 4.104.0
38+
? !!python/object/apply:multidict._multidict.istr
39+
- x-stainless-retry-count
40+
: - '0'
41+
? !!python/object/apply:multidict._multidict.istr
42+
- x-stainless-runtime
43+
: - node
44+
? !!python/object/apply:multidict._multidict.istr
45+
- x-stainless-runtime-version
46+
: - v22.17.0
47+
? !!python/object/apply:multidict._multidict.istr
48+
- x-stainless-timeout
49+
: - '0'
50+
method: POST
51+
uri: https://dd.openai.azure.com/openai/deployments/gpt-4.1/chat/completions?api-version=2024-05-01-preview
52+
response:
53+
body:
54+
string: '{"error":{"code":"401","message":"Access denied due to invalid subscription
55+
key or wrong API endpoint. Make sure to provide a valid key for an active
56+
subscription and use a correct regional API endpoint for your resource."}}'
57+
headers:
58+
Content-Length:
59+
- '224'
60+
Content-Type:
61+
- application/json
62+
Date:
63+
- Wed, 20 Aug 2025 13:52:24 GMT
64+
apim-request-id:
65+
- b198bf3e-4771-4a89-896f-1ba58f66e45d
66+
status:
67+
code: 401
68+
message: PermissionDenied
69+
version: 1

‎packages/dd-trace/test/llmobs/plugins/ai/index.spec.js‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const {
1313
expectedLLMObsNonLLMSpanEvent,
1414
deepEqualWithMockValues,
1515
MOCK_STRING,
16-
useLlmobs,
16+
useLlmObs,
1717
MOCK_NUMBER,
1818
MOCK_OBJECT
1919
} = require('../../util')
@@ -29,16 +29,15 @@ function getAiSdkOpenAiPackage (vercelAiVersion) {
2929

3030
describe('Plugin', () => {
3131
useEnv({
32-
OPENAI_API_KEY: '<not-a-real-key>',
33-
_DD_LLMOBS_FLUSH_INTERVAL: 0
32+
OPENAI_API_KEY: '<not-a-real-key>'
3433
})
3534

35+
const getEvents = useLlmObs({ plugin: 'ai' })
36+
3637
withVersions('ai', 'ai', range, (version, _, realVersion) => {
3738
let ai
3839
let openai
3940

40-
const getEvents = useLlmobs({ plugin: 'ai' })
41-
4241
beforeEach(function () {
4342
ai = require(`../../../../../../versions/ai@${version}`).get()
4443

‎packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ const { describe, it, before } = require('mocha')
55

66
const { withVersions } = require('../../../setup/mocha')
77

8-
const { expectedLLMObsLLMSpanEvent, deepEqualWithMockValues, useLlmobs } = require('../../util')
8+
const { expectedLLMObsLLMSpanEvent, deepEqualWithMockValues, useLlmObs } = require('../../util')
99
const { models, modelConfig } = require('../../../../../datadog-plugin-aws-sdk/test/fixtures/bedrockruntime')
1010
const { useEnv } = require('../../../../../../integration-tests/helpers')
1111

@@ -19,10 +19,11 @@ describe('Plugin', () => {
1919
describe('aws-sdk (bedrockruntime)', function () {
2020
useEnv({
2121
AWS_SECRET_ACCESS_KEY: '0000000000/00000000000000000000000000000',
22-
AWS_ACCESS_KEY_ID: '00000000000000000000',
23-
_DD_LLMOBS_FLUSH_INTERVAL: 0
22+
AWS_ACCESS_KEY_ID: '00000000000000000000'
2423
})
2524

25+
const getEvents = useLlmObs({ plugin: 'aws-sdk' })
26+
2627
withVersions('aws-sdk', ['@aws-sdk/smithy-client', 'aws-sdk'], '>=3', (version, moduleName) => {
2728
let AWS
2829
let bedrockRuntimeClient
@@ -31,8 +32,6 @@ describe('Plugin', () => {
3132
moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-bedrock-runtime' : 'aws-sdk'
3233

3334
describe('with configuration', () => {
34-
const getEvents = useLlmobs({ plugin: 'aws-sdk' })
35-
3635
before(() => {
3736
const requireVersion = version === '3.0.0' ? '3.422.0' : '>=3.422.0'
3837
AWS = require(`../../../../../../versions/${bedrockRuntimeClientName}@${requireVersion}`).get()

‎packages/dd-trace/test/llmobs/plugins/openai/openaiv4.spec.js‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,7 @@ describe('integrations', () => {
5555
return agent.close({ ritmReset: false, wipe: true })
5656
})
5757

58-
// TODO: Remove the range cap once we support openai 5
59-
withVersions('openai', 'openai', '>=4 <5', version => {
58+
withVersions('openai', 'openai', '>=4', version => {
6059
const moduleRequirePath = `../../../../../../versions/openai@${version}`
6160
let realVersion
6261

@@ -75,13 +74,13 @@ describe('integrations', () => {
7574
const AzureOpenAI = OpenAI.AzureOpenAI ?? OpenAI
7675
if (OpenAI.AzureOpenAI) {
7776
azureOpenai = new AzureOpenAI({
78-
endpoint: 'https://dd.openai.azure.com/',
77+
endpoint: 'http://127.0.0.1:9126/vcr/azure-openai',
7978
apiKey: 'test',
8079
apiVersion: '2024-05-01-preview'
8180
})
8281
} else {
8382
azureOpenai = new OpenAI({
84-
baseURL: 'https://dd.openai.azure.com/',
83+
baseURL: 'http://127.0.0.1:9126/vcr/azure-openai',
8584
apiKey: 'test',
8685
apiVersion: '2024-05-01-preview'
8786
})

‎packages/dd-trace/test/llmobs/util.js‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ function fromBuffer (spanProperty, isNumber = false) {
197197

198198
const agent = require('../plugins/agent')
199199
const assert = require('node:assert')
200+
const { useEnv } = require('../../../../integration-tests/helpers')
200201

201202
/**
202203
* @param {Object} options
@@ -205,15 +206,15 @@ const assert = require('node:assert')
205206
* @param {Object} options.closeOptions
206207
* @returns {function(): Promise<{ apmSpans: Array, llmobsSpans: Array }>}
207208
*/
208-
function useLlmobs ({
209+
function useLlmObs ({
209210
plugin,
210211
tracerConfigOptions = {
211212
llmobs: {
212213
mlApp: 'test',
213214
agentlessEnabled: false
214215
}
215216
},
216-
closeOptions = { ritmReset: false, wipe: true }
217+
closeOptions = { ritmReset: false }
217218
}) {
218219
if (!plugin) {
219220
throw new TypeError(
@@ -230,6 +231,10 @@ function useLlmobs ({
230231
let apmTracesPromise
231232
let llmobsTracesPromise
232233

234+
useEnv({
235+
_DD_LLMOBS_FLUSH_INTERVAL: 0
236+
})
237+
233238
before(() => {
234239
return agent.load(plugin, {}, tracerConfigOptions)
235240
})
@@ -264,7 +269,7 @@ module.exports = {
264269
expectedLLMObsLLMSpanEvent,
265270
expectedLLMObsNonLLMSpanEvent,
266271
deepEqualWithMockValues,
267-
useLlmobs,
272+
useLlmObs,
268273
MOCK_ANY,
269274
MOCK_NUMBER,
270275
MOCK_STRING,

0 commit comments

Comments
 (0)