From c966dcf11a90c4ff9e9da16c337fdc5dc51d9243 Mon Sep 17 00:00:00 2001 From: Nick Hale <4175918+njhale@users.noreply.github.com> Date: Mon, 14 Oct 2024 17:44:07 -0400 Subject: [PATCH 1/2] test: tweak smoke test tool bodies to standardize response text Tweak the tool bodies for smoke test GPTScripts to reduce ambiguity in the response. This prevents models -- like gpt-4o -- from doing things like failing to interpolate strings consistently between runs. Signed-off-by: Nick Hale <4175918+njhale@users.noreply.github.com> --- pkg/tests/smoke/testdata/Bob/test.gpt | 4 ++-- pkg/tests/smoke/testdata/BobAsShell/test.gpt | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/tests/smoke/testdata/Bob/test.gpt b/pkg/tests/smoke/testdata/Bob/test.gpt index 20f533e2..535790df 100644 --- a/pkg/tests/smoke/testdata/Bob/test.gpt +++ b/pkg/tests/smoke/testdata/Bob/test.gpt @@ -1,10 +1,10 @@ tools: bob -Ask Bob "how are you doing" and repeat his reply exactly. +Ask Bob "how are you doing" and repeat the response text exactly as given without saying anything else. --- name: bob description: I'm Bob, a friendly guy. args: question: The question to ask Bob. -When asked how I am doing, respond with exactly "Thanks for asking "${QUESTION}", I'm doing great fellow friendly AI tool!" +When asked how I am doing, respond with the following exactly: "Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!" with ${QUESTION} replaced with the question text as given. diff --git a/pkg/tests/smoke/testdata/BobAsShell/test.gpt b/pkg/tests/smoke/testdata/BobAsShell/test.gpt index a0edb9c4..6fdc514b 100644 --- a/pkg/tests/smoke/testdata/BobAsShell/test.gpt +++ b/pkg/tests/smoke/testdata/BobAsShell/test.gpt @@ -1,7 +1,6 @@ - tools: bob -Ask Bob "how are you doing" and repeat his reply exactly. +Ask Bob "how are you doing" and repeat the response text exactly as given without saying anything else. --- name: bob From 1b6f172ee636baf5feb848f4a2696b0c064c5c6e Mon Sep 17 00:00:00 2001 From: Nick Hale <4175918+njhale@users.noreply.github.com> Date: Mon, 14 Oct 2024 17:50:36 -0400 Subject: [PATCH 2/2] test: regenerate golden files for smoke tests Signed-off-by: Nick Hale <4175918+njhale@users.noreply.github.com> --- .../claude-3-5-sonnet-20240620-expected.json | 122 +++++----- .../Bob/gpt-4o-2024-08-06-expected.json | 140 +++++------ .../Bob/gpt-4o-mini-2024-07-18-expected.json | 140 +++++------ .../Bob/mistral-large-2402-expected.json | 154 ++++++------- .../claude-3-5-sonnet-20240620-expected.json | 198 ++++++---------- .../gpt-4o-2024-08-06-expected.json | 116 +++++----- .../gpt-4o-mini-2024-07-18-expected.json | 116 +++++----- .../mistral-large-2402-expected.json | 218 ++++++------------ 8 files changed, 534 insertions(+), 670 deletions(-) diff --git a/pkg/tests/smoke/testdata/Bob/claude-3-5-sonnet-20240620-expected.json b/pkg/tests/smoke/testdata/Bob/claude-3-5-sonnet-20240620-expected.json index 1b6342c4..52d975c5 100644 --- a/pkg/tests/smoke/testdata/Bob/claude-3-5-sonnet-20240620-expected.json +++ b/pkg/tests/smoke/testdata/Bob/claude-3-5-sonnet-20240620-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-10-14T12:30:37.766793-04:00", + "time": "2024-10-14T18:59:12.228692-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T12:30:37.767629-04:00", + "time": "2024-10-14T18:59:12.229038-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -41,14 +41,14 @@ "usage": {} }, { - "time": "2024-10-14T12:30:38.791552-04:00", + "time": "2024-10-14T18:59:13.520962-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T12:30:38.791851-04:00", + "time": "2024-10-14T18:59:13.521331-04:00", "callContext": { - "id": "1728923439", + "id": "1728946754", "tool": { "name": "Anthropic Claude3 Model Provider", "description": "Model provider for Anthropic hosted Claude3 models", @@ -93,9 +93,9 @@ "usage": {} }, { - "time": "2024-10-14T12:30:39.80734-04:00", + "time": "2024-10-14T18:59:14.541348-04:00", "callContext": { - "id": "1728923439", + "id": "1728946754", "tool": { "name": "Anthropic Claude3 Model Provider", "description": "Model provider for Anthropic hosted Claude3 models", @@ -138,24 +138,24 @@ }, "type": "callFinish", "usage": {}, - "content": "http://127.0.0.1:10940" + "content": "http://127.0.0.1:10258" }, { - "time": "2024-10-14T12:30:39.80752-04:00", + "time": "2024-10-14T18:59:14.541518-04:00", "type": "runFinish", "usage": {} }, { - "time": "2024-10-14T12:30:39.807592-04:00", + "time": "2024-10-14T18:59:14.541566-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -179,7 +179,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728923440", + "chatCompletionId": "1728946755", "usage": {}, "chatRequest": { "model": "", @@ -187,16 +187,16 @@ } }, { - "time": "2024-10-14T12:30:41.840024-04:00", + "time": "2024-10-14T18:59:17.304351-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -220,7 +220,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728923440", + "chatCompletionId": "1728946755", "usage": {}, "chatResponse": { "role": "assistant", @@ -228,7 +228,7 @@ { "toolCall": { "index": 0, - "id": "toolu_01B2uNGCcfcK9K5oGmBeix8b", + "id": "toolu_01KtYnAwnQ2cyRieDu98Jopb", "function": { "name": "bob", "arguments": "{\"question\": \"how are you doing\"}" @@ -240,16 +240,16 @@ } }, { - "time": "2024-10-14T12:30:41.840092-04:00", + "time": "2024-10-14T18:59:17.304441-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -273,7 +273,7 @@ "inputContext": null }, "toolSubCalls": { - "toolu_01B2uNGCcfcK9K5oGmBeix8b": { + "toolu_01KtYnAwnQ2cyRieDu98Jopb": { "toolID": "testdata/Bob/test.gpt:bob", "input": "{\"question\": \"how are you doing\"}" } @@ -282,9 +282,9 @@ "usage": {} }, { - "time": "2024-10-14T12:30:41.840134-04:00", + "time": "2024-10-14T18:59:17.304485-04:00", "callContext": { - "id": "toolu_01B2uNGCcfcK9K5oGmBeix8b", + "id": "toolu_01KtYnAwnQ2cyRieDu98Jopb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -299,7 +299,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -314,16 +314,16 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728923438" + "parentID": "1728946753" }, "type": "callStart", "usage": {}, "content": "{\"question\": \"how are you doing\"}" }, { - "time": "2024-10-14T12:30:42.553374-04:00", + "time": "2024-10-14T18:59:17.394841-04:00", "callContext": { - "id": "toolu_01B2uNGCcfcK9K5oGmBeix8b", + "id": "toolu_01KtYnAwnQ2cyRieDu98Jopb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -338,7 +338,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -353,10 +353,10 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728923438" + "parentID": "1728946753" }, "type": "callChat", - "chatCompletionId": "1728923441", + "chatCompletionId": "1728946756", "usage": {}, "chatRequest": { "model": "", @@ -364,9 +364,9 @@ } }, { - "time": "2024-10-14T12:30:43.320476-04:00", + "time": "2024-10-14T18:59:18.202926-04:00", "callContext": { - "id": "toolu_01B2uNGCcfcK9K5oGmBeix8b", + "id": "toolu_01KtYnAwnQ2cyRieDu98Jopb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -381,7 +381,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -396,25 +396,25 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728923438" + "parentID": "1728946753" }, "type": "callChat", - "chatCompletionId": "1728923441", + "chatCompletionId": "1728946756", "usage": {}, "chatResponse": { "role": "assistant", "content": [ { - "text": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": {} } }, { - "time": "2024-10-14T12:30:43.320527-04:00", + "time": "2024-10-14T18:59:18.202988-04:00", "callContext": { - "id": "toolu_01B2uNGCcfcK9K5oGmBeix8b", + "id": "toolu_01KtYnAwnQ2cyRieDu98Jopb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -429,7 +429,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -444,23 +444,23 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728923438" + "parentID": "1728946753" }, "type": "callFinish", "usage": {}, - "content": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T12:30:43.320565-04:00", + "time": "2024-10-14T18:59:18.203022-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -488,16 +488,16 @@ "usage": {} }, { - "time": "2024-10-14T12:30:43.676243-04:00", + "time": "2024-10-14T18:59:18.295164-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -521,7 +521,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728923442", + "chatCompletionId": "1728946757", "usage": {}, "chatRequest": { "model": "", @@ -529,16 +529,16 @@ } }, { - "time": "2024-10-14T12:30:45.165846-04:00", + "time": "2024-10-14T18:59:19.737028-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -562,29 +562,29 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728923442", + "chatCompletionId": "1728946757", "usage": {}, "chatResponse": { "role": "assistant", "content": [ { - "text": "Bob's reply was: \"Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!\"" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": {} } }, { - "time": "2024-10-14T12:30:45.165883-04:00", + "time": "2024-10-14T18:59:19.737045-04:00", "callContext": { - "id": "1728923438", + "id": "1728946753", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -609,10 +609,10 @@ }, "type": "callFinish", "usage": {}, - "content": "Bob's reply was: \"Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!\"" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T12:30:45.165904-04:00", + "time": "2024-10-14T18:59:19.737061-04:00", "type": "runFinish", "usage": {} } diff --git a/pkg/tests/smoke/testdata/Bob/gpt-4o-2024-08-06-expected.json b/pkg/tests/smoke/testdata/Bob/gpt-4o-2024-08-06-expected.json index 3e89e10b..67d9742b 100644 --- a/pkg/tests/smoke/testdata/Bob/gpt-4o-2024-08-06-expected.json +++ b/pkg/tests/smoke/testdata/Bob/gpt-4o-2024-08-06-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-10-14T15:00:24.05439-04:00", + "time": "2024-10-14T18:59:07.751937-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T15:00:24.054825-04:00", + "time": "2024-10-14T18:59:07.752324-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -41,16 +41,16 @@ "usage": {} }, { - "time": "2024-10-14T15:00:24.054884-04:00", + "time": "2024-10-14T18:59:07.75237-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -74,7 +74,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932426", + "chatCompletionId": "1728946749", "usage": {}, "chatRequest": { "model": "", @@ -82,16 +82,16 @@ } }, { - "time": "2024-10-14T15:00:25.474693-04:00", + "time": "2024-10-14T18:59:08.602251-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -115,11 +115,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932426", + "chatCompletionId": "1728946749", "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 }, "chatResponse": { "role": "assistant", @@ -127,7 +127,7 @@ { "toolCall": { "index": 0, - "id": "call_eVgG6VS6jTHyYl4nPzwWqYnT", + "id": "call_1nuYJNsE6SIQrXe4wyoMb8sh", "function": { "name": "bob", "arguments": "{\"question\":\"how are you doing\"}" @@ -136,23 +136,23 @@ } ], "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 } } }, { - "time": "2024-10-14T15:00:25.475061-04:00", + "time": "2024-10-14T18:59:08.602522-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -176,7 +176,7 @@ "inputContext": null }, "toolSubCalls": { - "call_eVgG6VS6jTHyYl4nPzwWqYnT": { + "call_1nuYJNsE6SIQrXe4wyoMb8sh": { "toolID": "testdata/Bob/test.gpt:bob", "input": "{\"question\":\"how are you doing\"}" } @@ -185,9 +185,9 @@ "usage": {} }, { - "time": "2024-10-14T15:00:25.475224-04:00", + "time": "2024-10-14T18:59:08.602683-04:00", "callContext": { - "id": "call_eVgG6VS6jTHyYl4nPzwWqYnT", + "id": "call_1nuYJNsE6SIQrXe4wyoMb8sh", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -202,7 +202,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -217,16 +217,16 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932425" + "parentID": "1728946748" }, "type": "callStart", "usage": {}, "content": "{\"question\":\"how are you doing\"}" }, { - "time": "2024-10-14T15:00:25.475415-04:00", + "time": "2024-10-14T18:59:08.602885-04:00", "callContext": { - "id": "call_eVgG6VS6jTHyYl4nPzwWqYnT", + "id": "call_1nuYJNsE6SIQrXe4wyoMb8sh", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -241,7 +241,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -256,10 +256,10 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932425" + "parentID": "1728946748" }, "type": "callChat", - "chatCompletionId": "1728932427", + "chatCompletionId": "1728946750", "usage": {}, "chatRequest": { "model": "", @@ -267,9 +267,9 @@ } }, { - "time": "2024-10-14T15:00:26.285181-04:00", + "time": "2024-10-14T18:59:09.291815-04:00", "callContext": { - "id": "call_eVgG6VS6jTHyYl4nPzwWqYnT", + "id": "call_1nuYJNsE6SIQrXe4wyoMb8sh", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -284,7 +284,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -299,33 +299,33 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932425" + "parentID": "1728946748" }, "type": "callChat", - "chatCompletionId": "1728932427", + "chatCompletionId": "1728946750", "usage": { - "promptTokens": 122, - "completionTokens": 14, - "totalTokens": 136 + "promptTokens": 137, + "completionTokens": 16, + "totalTokens": 153 }, "chatResponse": { "role": "assistant", "content": [ { - "text": "Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": { - "promptTokens": 122, - "completionTokens": 14, - "totalTokens": 136 + "promptTokens": 137, + "completionTokens": 16, + "totalTokens": 153 } } }, { - "time": "2024-10-14T15:00:26.285293-04:00", + "time": "2024-10-14T18:59:09.291883-04:00", "callContext": { - "id": "call_eVgG6VS6jTHyYl4nPzwWqYnT", + "id": "call_1nuYJNsE6SIQrXe4wyoMb8sh", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -340,7 +340,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -355,23 +355,23 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932425" + "parentID": "1728946748" }, "type": "callFinish", "usage": {}, - "content": "Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T15:00:26.285444-04:00", + "time": "2024-10-14T18:59:09.291934-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -399,16 +399,16 @@ "usage": {} }, { - "time": "2024-10-14T15:00:26.285687-04:00", + "time": "2024-10-14T18:59:09.292559-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -432,7 +432,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932428", + "chatCompletionId": "1728946751", "usage": {}, "chatRequest": { "model": "", @@ -440,16 +440,16 @@ } }, { - "time": "2024-10-14T15:00:27.147422-04:00", + "time": "2024-10-14T18:59:10.065468-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -473,37 +473,37 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932428", + "chatCompletionId": "1728946751", "usage": { - "promptTokens": 176, - "completionTokens": 18, - "totalTokens": 194 + "promptTokens": 185, + "completionTokens": 17, + "totalTokens": 202 }, "chatResponse": { "role": "assistant", "content": [ { - "text": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": { - "promptTokens": 176, - "completionTokens": 18, - "totalTokens": 194 + "promptTokens": 185, + "completionTokens": 17, + "totalTokens": 202 } } }, { - "time": "2024-10-14T15:00:27.147479-04:00", + "time": "2024-10-14T18:59:10.065547-04:00", "callContext": { - "id": "1728932425", + "id": "1728946748", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -528,10 +528,10 @@ }, "type": "callFinish", "usage": {}, - "content": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T15:00:27.147523-04:00", + "time": "2024-10-14T18:59:10.065614-04:00", "type": "runFinish", "usage": {} } diff --git a/pkg/tests/smoke/testdata/Bob/gpt-4o-mini-2024-07-18-expected.json b/pkg/tests/smoke/testdata/Bob/gpt-4o-mini-2024-07-18-expected.json index 3c1aebc9..e01dec10 100644 --- a/pkg/tests/smoke/testdata/Bob/gpt-4o-mini-2024-07-18-expected.json +++ b/pkg/tests/smoke/testdata/Bob/gpt-4o-mini-2024-07-18-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-10-14T11:31:46.97662-04:00", + "time": "2024-10-14T18:59:01.651525-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T11:31:46.977148-04:00", + "time": "2024-10-14T18:59:01.651887-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -41,16 +41,16 @@ "usage": {} }, { - "time": "2024-10-14T11:31:46.977209-04:00", + "time": "2024-10-14T18:59:01.651929-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -74,7 +74,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919908", + "chatCompletionId": "1728946743", "usage": {}, "chatRequest": { "model": "", @@ -82,16 +82,16 @@ } }, { - "time": "2024-10-14T11:31:49.170338-04:00", + "time": "2024-10-14T18:59:05.893238-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -115,11 +115,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919908", + "chatCompletionId": "1728946743", "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 }, "chatResponse": { "role": "assistant", @@ -127,7 +127,7 @@ { "toolCall": { "index": 0, - "id": "call_qBi5ZvQ2pFwXdENJXmuCb6Oy", + "id": "call_mcJFw1oe8YYFRPD1ZvFR4uZb", "function": { "name": "bob", "arguments": "{\"question\":\"how are you doing\"}" @@ -136,23 +136,23 @@ } ], "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 } } }, { - "time": "2024-10-14T11:31:49.170563-04:00", + "time": "2024-10-14T18:59:05.893515-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -176,7 +176,7 @@ "inputContext": null }, "toolSubCalls": { - "call_qBi5ZvQ2pFwXdENJXmuCb6Oy": { + "call_mcJFw1oe8YYFRPD1ZvFR4uZb": { "toolID": "testdata/Bob/test.gpt:bob", "input": "{\"question\":\"how are you doing\"}" } @@ -185,9 +185,9 @@ "usage": {} }, { - "time": "2024-10-14T11:31:49.171155-04:00", + "time": "2024-10-14T18:59:05.893776-04:00", "callContext": { - "id": "call_qBi5ZvQ2pFwXdENJXmuCb6Oy", + "id": "call_mcJFw1oe8YYFRPD1ZvFR4uZb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -202,7 +202,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -217,16 +217,16 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919907" + "parentID": "1728946742" }, "type": "callStart", "usage": {}, "content": "{\"question\":\"how are you doing\"}" }, { - "time": "2024-10-14T11:31:49.171395-04:00", + "time": "2024-10-14T18:59:05.894101-04:00", "callContext": { - "id": "call_qBi5ZvQ2pFwXdENJXmuCb6Oy", + "id": "call_mcJFw1oe8YYFRPD1ZvFR4uZb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -241,7 +241,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -256,10 +256,10 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919907" + "parentID": "1728946742" }, "type": "callChat", - "chatCompletionId": "1728919909", + "chatCompletionId": "1728946744", "usage": {}, "chatRequest": { "model": "", @@ -267,9 +267,9 @@ } }, { - "time": "2024-10-14T11:31:50.446571-04:00", + "time": "2024-10-14T18:59:08.315365-04:00", "callContext": { - "id": "call_qBi5ZvQ2pFwXdENJXmuCb6Oy", + "id": "call_mcJFw1oe8YYFRPD1ZvFR4uZb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -284,7 +284,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -299,33 +299,33 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919907" + "parentID": "1728946742" }, "type": "callChat", - "chatCompletionId": "1728919909", + "chatCompletionId": "1728946744", "usage": { - "promptTokens": 122, - "completionTokens": 17, - "totalTokens": 139 + "promptTokens": 137, + "completionTokens": 16, + "totalTokens": 153 }, "chatResponse": { "role": "assistant", "content": [ { - "text": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": { - "promptTokens": 122, - "completionTokens": 17, - "totalTokens": 139 + "promptTokens": 137, + "completionTokens": 16, + "totalTokens": 153 } } }, { - "time": "2024-10-14T11:31:50.446692-04:00", + "time": "2024-10-14T18:59:08.315556-04:00", "callContext": { - "id": "call_qBi5ZvQ2pFwXdENJXmuCb6Oy", + "id": "call_mcJFw1oe8YYFRPD1ZvFR4uZb", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -340,7 +340,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -355,23 +355,23 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919907" + "parentID": "1728946742" }, "type": "callFinish", "usage": {}, - "content": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T11:31:50.446773-04:00", + "time": "2024-10-14T18:59:08.315661-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -399,16 +399,16 @@ "usage": {} }, { - "time": "2024-10-14T11:31:50.446939-04:00", + "time": "2024-10-14T18:59:08.315834-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -432,7 +432,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919910", + "chatCompletionId": "1728946745", "usage": {}, "chatRequest": { "model": "", @@ -440,16 +440,16 @@ } }, { - "time": "2024-10-14T11:31:52.118055-04:00", + "time": "2024-10-14T18:59:09.27109-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -473,37 +473,37 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919910", + "chatCompletionId": "1728946745", "usage": { - "promptTokens": 179, - "completionTokens": 18, - "totalTokens": 197 + "promptTokens": 185, + "completionTokens": 17, + "totalTokens": 202 }, "chatResponse": { "role": "assistant", "content": [ { - "text": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": { - "promptTokens": 179, - "completionTokens": 18, - "totalTokens": 197 + "promptTokens": 185, + "completionTokens": 17, + "totalTokens": 202 } } }, { - "time": "2024-10-14T11:31:52.118196-04:00", + "time": "2024-10-14T18:59:09.271259-04:00", "callContext": { - "id": "1728919907", + "id": "1728946742", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -528,10 +528,10 @@ }, "type": "callFinish", "usage": {}, - "content": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T11:31:52.118256-04:00", + "time": "2024-10-14T18:59:09.271406-04:00", "type": "runFinish", "usage": {} } diff --git a/pkg/tests/smoke/testdata/Bob/mistral-large-2402-expected.json b/pkg/tests/smoke/testdata/Bob/mistral-large-2402-expected.json index c2971d57..ca392c03 100644 --- a/pkg/tests/smoke/testdata/Bob/mistral-large-2402-expected.json +++ b/pkg/tests/smoke/testdata/Bob/mistral-large-2402-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-10-14T12:20:24.700667-04:00", + "time": "2024-10-14T18:59:18.199427-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T12:20:24.701071-04:00", + "time": "2024-10-14T18:59:18.19975-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -41,14 +41,14 @@ "usage": {} }, { - "time": "2024-10-14T12:20:25.518655-04:00", + "time": "2024-10-14T18:59:19.063682-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T12:20:25.518946-04:00", + "time": "2024-10-14T18:59:19.063951-04:00", "callContext": { - "id": "1728922826", + "id": "1728946760", "tool": { "name": "Mistral La Plateforme Provider", "description": "Model provider for Mistral models running on La Plateforme", @@ -93,9 +93,9 @@ "usage": {} }, { - "time": "2024-10-14T12:20:26.534361-04:00", + "time": "2024-10-14T18:59:20.078127-04:00", "callContext": { - "id": "1728922826", + "id": "1728946760", "tool": { "name": "Mistral La Plateforme Provider", "description": "Model provider for Mistral models running on La Plateforme", @@ -138,24 +138,24 @@ }, "type": "callFinish", "usage": {}, - "content": "http://127.0.0.1:11149" + "content": "http://127.0.0.1:10912" }, { - "time": "2024-10-14T12:20:26.534546-04:00", + "time": "2024-10-14T18:59:20.078235-04:00", "type": "runFinish", "usage": {} }, { - "time": "2024-10-14T12:20:26.534598-04:00", + "time": "2024-10-14T18:59:20.078285-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -179,7 +179,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728922827", + "chatCompletionId": "1728946761", "usage": {}, "chatRequest": { "model": "", @@ -187,16 +187,16 @@ } }, { - "time": "2024-10-14T12:20:27.793767-04:00", + "time": "2024-10-14T18:59:21.857633-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -220,11 +220,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728922827", + "chatCompletionId": "1728946761", "usage": { - "promptTokens": 188, + "promptTokens": 195, "completionTokens": 23, - "totalTokens": 211 + "totalTokens": 218 }, "chatResponse": { "role": "assistant", @@ -232,7 +232,7 @@ { "toolCall": { "index": 0, - "id": "jSMVlVVyb", + "id": "pIj9ljPqt", "function": { "name": "bob", "arguments": "{\"question\": \"how are you doing\"}" @@ -241,23 +241,23 @@ } ], "usage": { - "promptTokens": 188, + "promptTokens": 195, "completionTokens": 23, - "totalTokens": 211 + "totalTokens": 218 } } }, { - "time": "2024-10-14T12:20:27.793996-04:00", + "time": "2024-10-14T18:59:21.858005-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -281,7 +281,7 @@ "inputContext": null }, "toolSubCalls": { - "jSMVlVVyb": { + "pIj9ljPqt": { "toolID": "testdata/Bob/test.gpt:bob", "input": "{\"question\": \"how are you doing\"}" } @@ -290,9 +290,9 @@ "usage": {} }, { - "time": "2024-10-14T12:20:27.794146-04:00", + "time": "2024-10-14T18:59:21.858212-04:00", "callContext": { - "id": "jSMVlVVyb", + "id": "pIj9ljPqt", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -307,7 +307,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -322,16 +322,16 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728922825" + "parentID": "1728946759" }, "type": "callStart", "usage": {}, "content": "{\"question\": \"how are you doing\"}" }, { - "time": "2024-10-14T12:20:28.306793-04:00", + "time": "2024-10-14T18:59:22.381191-04:00", "callContext": { - "id": "jSMVlVVyb", + "id": "pIj9ljPqt", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -346,7 +346,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -361,10 +361,10 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728922825" + "parentID": "1728946759" }, "type": "callChat", - "chatCompletionId": "1728922828", + "chatCompletionId": "1728946762", "usage": {}, "chatRequest": { "model": "", @@ -372,9 +372,9 @@ } }, { - "time": "2024-10-14T12:20:29.060571-04:00", + "time": "2024-10-14T18:59:23.160275-04:00", "callContext": { - "id": "jSMVlVVyb", + "id": "pIj9ljPqt", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -389,7 +389,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -404,33 +404,33 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728922825" + "parentID": "1728946759" }, "type": "callChat", - "chatCompletionId": "1728922828", + "chatCompletionId": "1728946762", "usage": { - "promptTokens": 145, - "completionTokens": 19, - "totalTokens": 164 + "promptTokens": 163, + "completionTokens": 18, + "totalTokens": 181 }, "chatResponse": { "role": "assistant", "content": [ { - "text": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": { - "promptTokens": 145, - "completionTokens": 19, - "totalTokens": 164 + "promptTokens": 163, + "completionTokens": 18, + "totalTokens": 181 } } }, { - "time": "2024-10-14T12:20:29.060766-04:00", + "time": "2024-10-14T18:59:23.160433-04:00", "callContext": { - "id": "jSMVlVVyb", + "id": "pIj9ljPqt", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -445,7 +445,7 @@ }, "type": "object" }, - "instructions": "When asked how I am doing, respond with exactly \"Thanks for asking \"${QUESTION}\", I'm doing great fellow friendly AI tool!\"", + "instructions": "When asked how I am doing, respond with the following exactly: \"Thanks for asking ${QUESTION}! I'm doing great fellow friendly AI tool!\" with ${QUESTION} replaced with the question text as given.", "id": "testdata/Bob/test.gpt:bob", "localTools": { "": "testdata/Bob/test.gpt:", @@ -460,23 +460,23 @@ "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728922825" + "parentID": "1728946759" }, "type": "callFinish", "usage": {}, - "content": "Thanks for asking \"how are you doing\", I'm doing great fellow friendly AI tool!" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T12:20:29.060906-04:00", + "time": "2024-10-14T18:59:23.160522-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -504,16 +504,16 @@ "usage": {} }, { - "time": "2024-10-14T12:20:29.203429-04:00", + "time": "2024-10-14T18:59:23.531261-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -537,7 +537,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728922829", + "chatCompletionId": "1728946763", "usage": {}, "chatRequest": { "model": "", @@ -545,16 +545,16 @@ } }, { - "time": "2024-10-14T12:20:30.272631-04:00", + "time": "2024-10-14T18:59:24.303745-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -578,37 +578,37 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728922829", + "chatCompletionId": "1728946763", "usage": { - "promptTokens": 246, - "completionTokens": 23, - "totalTokens": 269 + "promptTokens": 252, + "completionTokens": 18, + "totalTokens": 270 }, "chatResponse": { "role": "assistant", "content": [ { - "text": "Bob said, \"Thanks for asking 'how are you doing', I'm doing great fellow friendly AI tool!\"" + "text": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" } ], "usage": { - "promptTokens": 246, - "completionTokens": 23, - "totalTokens": 269 + "promptTokens": 252, + "completionTokens": 18, + "totalTokens": 270 } } }, { - "time": "2024-10-14T12:20:30.27277-04:00", + "time": "2024-10-14T18:59:24.303903-04:00", "callContext": { - "id": "1728922825", + "id": "1728946759", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/Bob/test.gpt:", "toolMapping": { "bob": [ @@ -633,10 +633,10 @@ }, "type": "callFinish", "usage": {}, - "content": "Bob said, \"Thanks for asking 'how are you doing', I'm doing great fellow friendly AI tool!\"" + "content": "Thanks for asking how are you doing! I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T12:20:30.27283-04:00", + "time": "2024-10-14T18:59:24.303961-04:00", "type": "runFinish", "usage": {} } diff --git a/pkg/tests/smoke/testdata/BobAsShell/claude-3-5-sonnet-20240620-expected.json b/pkg/tests/smoke/testdata/BobAsShell/claude-3-5-sonnet-20240620-expected.json index 3a2838ab..22fe9514 100644 --- a/pkg/tests/smoke/testdata/BobAsShell/claude-3-5-sonnet-20240620-expected.json +++ b/pkg/tests/smoke/testdata/BobAsShell/claude-3-5-sonnet-20240620-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-08-23T12:02:17.549859-04:00", + "time": "2024-10-14T17:38:39.518668-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-08-23T12:02:17.55023-04:00", + "time": "2024-10-14T17:38:39.519079-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -41,14 +41,14 @@ "usage": {} }, { - "time": "2024-08-23T12:02:18.283201-04:00", + "time": "2024-10-14T17:38:40.155982-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-08-23T12:02:18.28339-04:00", + "time": "2024-10-14T17:38:40.156405-04:00", "callContext": { - "id": "1724428939", + "id": "1728941921", "tool": { "name": "Anthropic Claude3 Model Provider", "description": "Model provider for Anthropic hosted Claude3 models", @@ -64,7 +64,7 @@ "github.com/gptscript-ai/credential as github.com/gptscript-ai/claude3-anthropic-provider/credential with \"Please enter your Anthropic API Key\" as message and token as field and \"ANTHROPIC_API_KEY\" as env": [ { "reference": "github.com/gptscript-ai/credential as github.com/gptscript-ai/claude3-anthropic-provider/credential with \"Please enter your Anthropic API Key\" as message and token as field and \"ANTHROPIC_API_KEY\" as env", - "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/37e5f870e195b896438c0bc35867403a42f82e89/tool.gpt:token" + "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/de2fada1c51a1dbb5c3e9ef268ea6740d1b52f80/tool.gpt:token" } ] }, @@ -93,9 +93,9 @@ "usage": {} }, { - "time": "2024-08-23T12:02:19.295369-04:00", + "time": "2024-10-14T17:38:41.173004-04:00", "callContext": { - "id": "1724428939", + "id": "1728941921", "tool": { "name": "Anthropic Claude3 Model Provider", "description": "Model provider for Anthropic hosted Claude3 models", @@ -111,7 +111,7 @@ "github.com/gptscript-ai/credential as github.com/gptscript-ai/claude3-anthropic-provider/credential with \"Please enter your Anthropic API Key\" as message and token as field and \"ANTHROPIC_API_KEY\" as env": [ { "reference": "github.com/gptscript-ai/credential as github.com/gptscript-ai/claude3-anthropic-provider/credential with \"Please enter your Anthropic API Key\" as message and token as field and \"ANTHROPIC_API_KEY\" as env", - "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/37e5f870e195b896438c0bc35867403a42f82e89/tool.gpt:token" + "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/de2fada1c51a1dbb5c3e9ef268ea6740d1b52f80/tool.gpt:token" } ] }, @@ -138,24 +138,24 @@ }, "type": "callFinish", "usage": {}, - "content": "http://127.0.0.1:10739" + "content": "http://127.0.0.1:10787" }, { - "time": "2024-08-23T12:02:19.295542-04:00", + "time": "2024-10-14T17:38:41.173175-04:00", "type": "runFinish", "usage": {} }, { - "time": "2024-08-23T12:02:19.295604-04:00", + "time": "2024-10-14T17:38:41.173247-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -179,48 +179,24 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428940", + "chatCompletionId": "1728941922", "usage": {}, "chatRequest": { - "model": "claude-3-5-sonnet-20240620", - "messages": [ - { - "role": "system", - "content": "Ask Bob \"how are you doing\" and repeat his reply exactly." - } - ], - "temperature": 0, - "tools": [ - { - "type": "function", - "function": { - "name": "bob", - "description": "I'm Bob, a friendly guy.", - "parameters": { - "properties": { - "question": { - "description": "The question to ask Bob.", - "type": "string" - } - }, - "type": "object" - } - } - } - ] + "model": "", + "messages": null } }, { - "time": "2024-08-23T12:02:21.136785-04:00", + "time": "2024-10-14T17:38:43.937061-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -244,7 +220,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428940", + "chatCompletionId": "1728941922", "usage": {}, "chatResponse": { "role": "assistant", @@ -252,7 +228,7 @@ { "toolCall": { "index": 0, - "id": "toolu_01XzHFJpwHD8hzowvAqGgfSz", + "id": "toolu_01PQYSGxbwRLw8XuUUkgKvbe", "function": { "name": "bob", "arguments": "{\"question\": \"how are you doing\"}" @@ -264,16 +240,16 @@ } }, { - "time": "2024-08-23T12:02:21.136848-04:00", + "time": "2024-10-14T17:38:43.937155-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -297,7 +273,7 @@ "inputContext": null }, "toolSubCalls": { - "toolu_01XzHFJpwHD8hzowvAqGgfSz": { + "toolu_01PQYSGxbwRLw8XuUUkgKvbe": { "toolID": "testdata/BobAsShell/test.gpt:bob", "input": "{\"question\": \"how are you doing\"}" } @@ -306,9 +282,9 @@ "usage": {} }, { - "time": "2024-08-23T12:02:21.136877-04:00", + "time": "2024-10-14T17:38:43.937193-04:00", "callContext": { - "id": "toolu_01XzHFJpwHD8hzowvAqGgfSz", + "id": "toolu_01PQYSGxbwRLw8XuUUkgKvbe", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -331,14 +307,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428938", + "parentID": "1728941920", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callStart", @@ -346,9 +322,9 @@ "content": "{\"question\": \"how are you doing\"}" }, { - "time": "2024-08-23T12:02:21.137223-04:00", + "time": "2024-10-14T17:38:43.938264-04:00", "callContext": { - "id": "toolu_01XzHFJpwHD8hzowvAqGgfSz", + "id": "toolu_01PQYSGxbwRLw8XuUUkgKvbe", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -371,18 +347,18 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428938", + "parentID": "1728941920", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1724428941", + "chatCompletionId": "1728941923", "usage": {}, "chatRequest": { "model": "", @@ -390,9 +366,9 @@ } }, { - "time": "2024-08-23T12:02:21.142624-04:00", + "time": "2024-10-14T17:38:43.943625-04:00", "callContext": { - "id": "toolu_01XzHFJpwHD8hzowvAqGgfSz", + "id": "toolu_01PQYSGxbwRLw8XuUUkgKvbe", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -415,27 +391,27 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428938", + "parentID": "1728941920", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1724428941", + "chatCompletionId": "1728941923", "usage": {}, "chatResponse": { "usage": {} } }, { - "time": "2024-08-23T12:02:21.142691-04:00", + "time": "2024-10-14T17:38:43.943703-04:00", "callContext": { - "id": "toolu_01XzHFJpwHD8hzowvAqGgfSz", + "id": "toolu_01PQYSGxbwRLw8XuUUkgKvbe", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -458,14 +434,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428938", + "parentID": "1728941920", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callFinish", @@ -473,16 +449,16 @@ "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\n" }, { - "time": "2024-08-23T12:02:21.142723-04:00", + "time": "2024-10-14T17:38:43.943766-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -510,16 +486,16 @@ "usage": {} }, { - "time": "2024-08-23T12:02:21.371211-04:00", + "time": "2024-10-14T17:38:44.494388-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -543,68 +519,24 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428942", + "chatCompletionId": "1728941924", "usage": {}, "chatRequest": { - "model": "claude-3-5-sonnet-20240620", - "messages": [ - { - "role": "system", - "content": "Ask Bob \"how are you doing\" and repeat his reply exactly." - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "toolu_01XzHFJpwHD8hzowvAqGgfSz", - "type": "function", - "function": { - "name": "bob", - "arguments": "{\"question\": \"how are you doing\"}" - } - } - ] - }, - { - "role": "tool", - "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\n", - "name": "bob", - "tool_call_id": "toolu_01XzHFJpwHD8hzowvAqGgfSz" - } - ], - "temperature": 0, - "tools": [ - { - "type": "function", - "function": { - "name": "bob", - "description": "I'm Bob, a friendly guy.", - "parameters": { - "properties": { - "question": { - "description": "The question to ask Bob.", - "type": "string" - } - }, - "type": "object" - } - } - } - ] + "model": "", + "messages": null } }, { - "time": "2024-08-23T12:02:23.102371-04:00", + "time": "2024-10-14T17:38:45.659797-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -628,29 +560,29 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428942", + "chatCompletionId": "1728941924", "usage": {}, "chatResponse": { "role": "assistant", "content": [ { - "text": "Bob's reply was: \"Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\"" + "text": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!" } ], "usage": {} } }, { - "time": "2024-08-23T12:02:23.102422-04:00", + "time": "2024-10-14T17:38:45.659891-04:00", "callContext": { - "id": "1724428938", + "id": "1728941920", "tool": { "modelName": "claude-3-5-sonnet-20240620 from github.com/gptscript-ai/claude3-anthropic-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -675,10 +607,10 @@ }, "type": "callFinish", "usage": {}, - "content": "Bob's reply was: \"Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\"" + "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!" }, { - "time": "2024-08-23T12:02:23.102441-04:00", + "time": "2024-10-14T17:38:45.659921-04:00", "type": "runFinish", "usage": {} } diff --git a/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-2024-08-06-expected.json b/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-2024-08-06-expected.json index 694e021c..91b6a636 100644 --- a/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-2024-08-06-expected.json +++ b/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-2024-08-06-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-10-14T15:00:27.184787-04:00", + "time": "2024-10-14T17:38:25.173529-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T15:00:27.185109-04:00", + "time": "2024-10-14T17:38:25.174285-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -41,16 +41,16 @@ "usage": {} }, { - "time": "2024-10-14T15:00:27.185153-04:00", + "time": "2024-10-14T17:38:25.174351-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -74,7 +74,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932429", + "chatCompletionId": "1728941907", "usage": {}, "chatRequest": { "model": "", @@ -82,16 +82,16 @@ } }, { - "time": "2024-10-14T15:00:28.310827-04:00", + "time": "2024-10-14T17:38:26.165696-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -115,11 +115,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932429", + "chatCompletionId": "1728941907", "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 }, "chatResponse": { "role": "assistant", @@ -127,7 +127,7 @@ { "toolCall": { "index": 0, - "id": "call_c8HV8M6DRtYLd8MEHgaHAWtZ", + "id": "call_95p8Knb4mdiEgxt5iXxnxOKC", "function": { "name": "bob", "arguments": "{\"question\":\"how are you doing\"}" @@ -136,23 +136,23 @@ } ], "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 } } }, { - "time": "2024-10-14T15:00:28.311081-04:00", + "time": "2024-10-14T17:38:26.165858-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -176,7 +176,7 @@ "inputContext": null }, "toolSubCalls": { - "call_c8HV8M6DRtYLd8MEHgaHAWtZ": { + "call_95p8Knb4mdiEgxt5iXxnxOKC": { "toolID": "testdata/BobAsShell/test.gpt:bob", "input": "{\"question\":\"how are you doing\"}" } @@ -185,9 +185,9 @@ "usage": {} }, { - "time": "2024-10-14T15:00:28.311186-04:00", + "time": "2024-10-14T17:38:26.165964-04:00", "callContext": { - "id": "call_c8HV8M6DRtYLd8MEHgaHAWtZ", + "id": "call_95p8Knb4mdiEgxt5iXxnxOKC", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -210,14 +210,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932428", + "parentID": "1728941906", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callStart", @@ -225,9 +225,9 @@ "content": "{\"question\":\"how are you doing\"}" }, { - "time": "2024-10-14T15:00:28.312343-04:00", + "time": "2024-10-14T17:38:26.167235-04:00", "callContext": { - "id": "call_c8HV8M6DRtYLd8MEHgaHAWtZ", + "id": "call_95p8Knb4mdiEgxt5iXxnxOKC", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -250,18 +250,18 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932428", + "parentID": "1728941906", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1728932430", + "chatCompletionId": "1728941908", "usage": {}, "chatRequest": { "model": "", @@ -269,9 +269,9 @@ } }, { - "time": "2024-10-14T15:00:28.328987-04:00", + "time": "2024-10-14T17:38:26.178131-04:00", "callContext": { - "id": "call_c8HV8M6DRtYLd8MEHgaHAWtZ", + "id": "call_95p8Knb4mdiEgxt5iXxnxOKC", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -294,27 +294,27 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932428", + "parentID": "1728941906", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1728932430", + "chatCompletionId": "1728941908", "usage": {}, "chatResponse": { "usage": {} } }, { - "time": "2024-10-14T15:00:28.329156-04:00", + "time": "2024-10-14T17:38:26.178199-04:00", "callContext": { - "id": "call_c8HV8M6DRtYLd8MEHgaHAWtZ", + "id": "call_95p8Knb4mdiEgxt5iXxnxOKC", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -337,14 +337,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728932428", + "parentID": "1728941906", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callFinish", @@ -352,16 +352,16 @@ "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\n" }, { - "time": "2024-10-14T15:00:28.32924-04:00", + "time": "2024-10-14T17:38:26.178356-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -389,16 +389,16 @@ "usage": {} }, { - "time": "2024-10-14T15:00:28.329393-04:00", + "time": "2024-10-14T17:38:26.178539-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -422,7 +422,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932431", + "chatCompletionId": "1728941909", "usage": {}, "chatRequest": { "model": "", @@ -430,16 +430,16 @@ } }, { - "time": "2024-10-14T15:00:29.437703-04:00", + "time": "2024-10-14T17:38:27.001877-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -463,11 +463,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728932431", + "chatCompletionId": "1728941909", "usage": { - "promptTokens": 178, + "promptTokens": 185, "completionTokens": 17, - "totalTokens": 195 + "totalTokens": 202 }, "chatResponse": { "role": "assistant", @@ -477,23 +477,23 @@ } ], "usage": { - "promptTokens": 178, + "promptTokens": 185, "completionTokens": 17, - "totalTokens": 195 + "totalTokens": 202 } } }, { - "time": "2024-10-14T15:00:29.437754-04:00", + "time": "2024-10-14T17:38:27.001903-04:00", "callContext": { - "id": "1728932428", + "id": "1728941906", "tool": { "modelName": "gpt-4o-2024-08-06", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -521,7 +521,7 @@ "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T15:00:29.437838-04:00", + "time": "2024-10-14T17:38:27.001932-04:00", "type": "runFinish", "usage": {} } diff --git a/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-mini-2024-07-18-expected.json b/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-mini-2024-07-18-expected.json index 6354054d..bdfee87d 100644 --- a/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-mini-2024-07-18-expected.json +++ b/pkg/tests/smoke/testdata/BobAsShell/gpt-4o-mini-2024-07-18-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-10-14T11:31:52.152013-04:00", + "time": "2024-10-14T17:37:54.379122-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-10-14T11:31:52.152428-04:00", + "time": "2024-10-14T17:37:54.379631-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -41,16 +41,16 @@ "usage": {} }, { - "time": "2024-10-14T11:31:52.152473-04:00", + "time": "2024-10-14T17:37:54.379682-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -74,7 +74,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919914", + "chatCompletionId": "1728941876", "usage": {}, "chatRequest": { "model": "", @@ -82,16 +82,16 @@ } }, { - "time": "2024-10-14T11:31:54.107362-04:00", + "time": "2024-10-14T17:37:55.230509-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -115,11 +115,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919914", + "chatCompletionId": "1728941876", "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 }, "chatResponse": { "role": "assistant", @@ -127,7 +127,7 @@ { "toolCall": { "index": 0, - "id": "call_AmrlGivMXtyAzbP85T7lwFN9", + "id": "call_FInUoOxKSR90EOxzIHXivvSX", "function": { "name": "bob", "arguments": "{\"question\":\"how are you doing\"}" @@ -136,23 +136,23 @@ } ], "usage": { - "promptTokens": 138, + "promptTokens": 145, "completionTokens": 17, - "totalTokens": 155 + "totalTokens": 162 } } }, { - "time": "2024-10-14T11:31:54.107585-04:00", + "time": "2024-10-14T17:37:55.23069-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -176,7 +176,7 @@ "inputContext": null }, "toolSubCalls": { - "call_AmrlGivMXtyAzbP85T7lwFN9": { + "call_FInUoOxKSR90EOxzIHXivvSX": { "toolID": "testdata/BobAsShell/test.gpt:bob", "input": "{\"question\":\"how are you doing\"}" } @@ -185,9 +185,9 @@ "usage": {} }, { - "time": "2024-10-14T11:31:54.107715-04:00", + "time": "2024-10-14T17:37:55.230816-04:00", "callContext": { - "id": "call_AmrlGivMXtyAzbP85T7lwFN9", + "id": "call_FInUoOxKSR90EOxzIHXivvSX", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -210,14 +210,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919913", + "parentID": "1728941875", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callStart", @@ -225,9 +225,9 @@ "content": "{\"question\":\"how are you doing\"}" }, { - "time": "2024-10-14T11:31:54.108876-04:00", + "time": "2024-10-14T17:37:55.231913-04:00", "callContext": { - "id": "call_AmrlGivMXtyAzbP85T7lwFN9", + "id": "call_FInUoOxKSR90EOxzIHXivvSX", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -250,18 +250,18 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919913", + "parentID": "1728941875", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1728919915", + "chatCompletionId": "1728941877", "usage": {}, "chatRequest": { "model": "", @@ -269,9 +269,9 @@ } }, { - "time": "2024-10-14T11:31:54.121327-04:00", + "time": "2024-10-14T17:37:55.245261-04:00", "callContext": { - "id": "call_AmrlGivMXtyAzbP85T7lwFN9", + "id": "call_FInUoOxKSR90EOxzIHXivvSX", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -294,27 +294,27 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919913", + "parentID": "1728941875", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1728919915", + "chatCompletionId": "1728941877", "usage": {}, "chatResponse": { "usage": {} } }, { - "time": "2024-10-14T11:31:54.121533-04:00", + "time": "2024-10-14T17:37:55.245348-04:00", "callContext": { - "id": "call_AmrlGivMXtyAzbP85T7lwFN9", + "id": "call_FInUoOxKSR90EOxzIHXivvSX", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -337,14 +337,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1728919913", + "parentID": "1728941875", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callFinish", @@ -352,16 +352,16 @@ "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\n" }, { - "time": "2024-10-14T11:31:54.121596-04:00", + "time": "2024-10-14T17:37:55.245528-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -389,16 +389,16 @@ "usage": {} }, { - "time": "2024-10-14T11:31:54.121802-04:00", + "time": "2024-10-14T17:37:55.245751-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -422,7 +422,7 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919916", + "chatCompletionId": "1728941878", "usage": {}, "chatRequest": { "model": "", @@ -430,16 +430,16 @@ } }, { - "time": "2024-10-14T11:31:55.746879-04:00", + "time": "2024-10-14T17:37:56.345692-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -463,11 +463,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1728919916", + "chatCompletionId": "1728941878", "usage": { - "promptTokens": 178, + "promptTokens": 185, "completionTokens": 17, - "totalTokens": 195 + "totalTokens": 202 }, "chatResponse": { "role": "assistant", @@ -477,23 +477,23 @@ } ], "usage": { - "promptTokens": 178, + "promptTokens": 185, "completionTokens": 17, - "totalTokens": 195 + "totalTokens": 202 } } }, { - "time": "2024-10-14T11:31:55.746965-04:00", + "time": "2024-10-14T17:37:56.345742-04:00", "callContext": { - "id": "1728919913", + "id": "1728941875", "tool": { "modelName": "gpt-4o-mini-2024-07-18", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -521,7 +521,7 @@ "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!" }, { - "time": "2024-10-14T11:31:55.747227-04:00", + "time": "2024-10-14T17:37:56.345803-04:00", "type": "runFinish", "usage": {} } diff --git a/pkg/tests/smoke/testdata/BobAsShell/mistral-large-2402-expected.json b/pkg/tests/smoke/testdata/BobAsShell/mistral-large-2402-expected.json index 6dc41a08..4506754b 100644 --- a/pkg/tests/smoke/testdata/BobAsShell/mistral-large-2402-expected.json +++ b/pkg/tests/smoke/testdata/BobAsShell/mistral-large-2402-expected.json @@ -1,20 +1,20 @@ [ { - "time": "2024-08-23T12:02:07.951538-04:00", + "time": "2024-10-14T17:38:47.018065-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-08-23T12:02:07.951742-04:00", + "time": "2024-10-14T17:38:47.018394-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -41,14 +41,14 @@ "usage": {} }, { - "time": "2024-08-23T12:02:08.65216-04:00", + "time": "2024-10-14T17:38:47.47198-04:00", "type": "runStart", "usage": {} }, { - "time": "2024-08-23T12:02:08.65229-04:00", + "time": "2024-10-14T17:38:47.472449-04:00", "callContext": { - "id": "1724428929", + "id": "1728941929", "tool": { "name": "Mistral La Plateforme Provider", "description": "Model provider for Mistral models running on La Plateforme", @@ -64,7 +64,7 @@ "github.com/gptscript-ai/credential as github.com/gptscript-ai/mistral-laplateforme-provider/credential with \"Please enter your Mistral La Plateforme API Key\" as message and token as field and \"MISTRAL_API_KEY\" as env": [ { "reference": "github.com/gptscript-ai/credential as github.com/gptscript-ai/mistral-laplateforme-provider/credential with \"Please enter your Mistral La Plateforme API Key\" as message and token as field and \"MISTRAL_API_KEY\" as env", - "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/37e5f870e195b896438c0bc35867403a42f82e89/tool.gpt:token" + "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/de2fada1c51a1dbb5c3e9ef268ea6740d1b52f80/tool.gpt:token" } ] }, @@ -93,9 +93,9 @@ "usage": {} }, { - "time": "2024-08-23T12:02:09.659962-04:00", + "time": "2024-10-14T17:38:50.566081-04:00", "callContext": { - "id": "1724428929", + "id": "1728941929", "tool": { "name": "Mistral La Plateforme Provider", "description": "Model provider for Mistral models running on La Plateforme", @@ -111,7 +111,7 @@ "github.com/gptscript-ai/credential as github.com/gptscript-ai/mistral-laplateforme-provider/credential with \"Please enter your Mistral La Plateforme API Key\" as message and token as field and \"MISTRAL_API_KEY\" as env": [ { "reference": "github.com/gptscript-ai/credential as github.com/gptscript-ai/mistral-laplateforme-provider/credential with \"Please enter your Mistral La Plateforme API Key\" as message and token as field and \"MISTRAL_API_KEY\" as env", - "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/37e5f870e195b896438c0bc35867403a42f82e89/tool.gpt:token" + "toolID": "https://raw.githubusercontent.com/gptscript-ai/credential/de2fada1c51a1dbb5c3e9ef268ea6740d1b52f80/tool.gpt:token" } ] }, @@ -138,24 +138,24 @@ }, "type": "callFinish", "usage": {}, - "content": "http://127.0.0.1:10532" + "content": "http://127.0.0.1:11133" }, { - "time": "2024-08-23T12:02:09.66007-04:00", + "time": "2024-10-14T17:38:50.56681-04:00", "type": "runFinish", "usage": {} }, { - "time": "2024-08-23T12:02:09.660117-04:00", + "time": "2024-10-14T17:38:50.567218-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -179,48 +179,24 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428930", + "chatCompletionId": "1728941930", "usage": {}, "chatRequest": { - "model": "mistral-large-2402", - "messages": [ - { - "role": "system", - "content": "Ask Bob \"how are you doing\" and repeat his reply exactly." - } - ], - "temperature": 0, - "tools": [ - { - "type": "function", - "function": { - "name": "bob", - "description": "I'm Bob, a friendly guy.", - "parameters": { - "properties": { - "question": { - "description": "The question to ask Bob.", - "type": "string" - } - }, - "type": "object" - } - } - } - ] + "model": "", + "messages": null } }, { - "time": "2024-08-23T12:02:11.140706-04:00", + "time": "2024-10-14T17:38:51.51096-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -244,11 +220,11 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428930", + "chatCompletionId": "1728941930", "usage": { - "promptTokens": 188, + "promptTokens": 195, "completionTokens": 23, - "totalTokens": 211 + "totalTokens": 218 }, "chatResponse": { "role": "assistant", @@ -256,7 +232,7 @@ { "toolCall": { "index": 0, - "id": "r1wQzUugN", + "id": "KLMoUpwIL", "function": { "name": "bob", "arguments": "{\"question\": \"how are you doing\"}" @@ -265,23 +241,23 @@ } ], "usage": { - "promptTokens": 188, + "promptTokens": 195, "completionTokens": 23, - "totalTokens": 211 + "totalTokens": 218 } } }, { - "time": "2024-08-23T12:02:11.140968-04:00", + "time": "2024-10-14T17:38:51.511569-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -305,7 +281,7 @@ "inputContext": null }, "toolSubCalls": { - "r1wQzUugN": { + "KLMoUpwIL": { "toolID": "testdata/BobAsShell/test.gpt:bob", "input": "{\"question\": \"how are you doing\"}" } @@ -314,9 +290,9 @@ "usage": {} }, { - "time": "2024-08-23T12:02:11.141094-04:00", + "time": "2024-10-14T17:38:51.511777-04:00", "callContext": { - "id": "r1wQzUugN", + "id": "KLMoUpwIL", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -339,14 +315,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428928", + "parentID": "1728941928", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callStart", @@ -354,9 +330,9 @@ "content": "{\"question\": \"how are you doing\"}" }, { - "time": "2024-08-23T12:02:11.141978-04:00", + "time": "2024-10-14T17:38:51.513152-04:00", "callContext": { - "id": "r1wQzUugN", + "id": "KLMoUpwIL", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -379,18 +355,18 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428928", + "parentID": "1728941928", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1724428931", + "chatCompletionId": "1728941931", "usage": {}, "chatRequest": { "model": "", @@ -398,9 +374,9 @@ } }, { - "time": "2024-08-23T12:02:11.153328-04:00", + "time": "2024-10-14T17:38:51.528154-04:00", "callContext": { - "id": "r1wQzUugN", + "id": "KLMoUpwIL", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -423,27 +399,27 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428928", + "parentID": "1728941928", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callChat", - "chatCompletionId": "1724428931", + "chatCompletionId": "1728941931", "usage": {}, "chatResponse": { "usage": {} } }, { - "time": "2024-08-23T12:02:11.153471-04:00", + "time": "2024-10-14T17:38:51.528298-04:00", "callContext": { - "id": "r1wQzUugN", + "id": "KLMoUpwIL", "tool": { "name": "bob", "description": "I'm Bob, a friendly guy.", @@ -466,14 +442,14 @@ }, "source": { "location": "testdata/BobAsShell/test.gpt", - "lineNo": 7 + "lineNo": 6 }, "workingDir": "testdata/BobAsShell" }, "currentAgent": {}, "inputContext": null, "toolName": "bob", - "parentID": "1724428928", + "parentID": "1728941928", "displayText": "Running bob from testdata/BobAsShell/test.gpt" }, "type": "callFinish", @@ -481,16 +457,16 @@ "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\n" }, { - "time": "2024-08-23T12:02:11.153544-04:00", + "time": "2024-10-14T17:38:51.528421-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -518,16 +494,16 @@ "usage": {} }, { - "time": "2024-08-23T12:02:11.41447-04:00", + "time": "2024-10-14T17:38:51.894619-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -551,68 +527,24 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428932", + "chatCompletionId": "1728941932", "usage": {}, "chatRequest": { - "model": "mistral-large-2402", - "messages": [ - { - "role": "system", - "content": "Ask Bob \"how are you doing\" and repeat his reply exactly." - }, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "r1wQzUugN", - "type": "function", - "function": { - "name": "bob", - "arguments": "{\"question\": \"how are you doing\"}" - } - } - ] - }, - { - "role": "tool", - "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\n", - "name": "bob", - "tool_call_id": "r1wQzUugN" - } - ], - "temperature": 0, - "tools": [ - { - "type": "function", - "function": { - "name": "bob", - "description": "I'm Bob, a friendly guy.", - "parameters": { - "properties": { - "question": { - "description": "The question to ask Bob.", - "type": "string" - } - }, - "type": "object" - } - } - } - ] + "model": "", + "messages": null } }, { - "time": "2024-08-23T12:02:12.424283-04:00", + "time": "2024-10-14T17:38:52.586731-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -636,37 +568,37 @@ "inputContext": null }, "type": "callChat", - "chatCompletionId": "1724428932", + "chatCompletionId": "1728941932", "usage": { - "promptTokens": 247, - "completionTokens": 22, - "totalTokens": 269 + "promptTokens": 254, + "completionTokens": 18, + "totalTokens": 272 }, "chatResponse": { "role": "assistant", "content": [ { - "text": "Bob said, \"Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\"" + "text": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!" } ], "usage": { - "promptTokens": 247, - "completionTokens": 22, - "totalTokens": 269 + "promptTokens": 254, + "completionTokens": 18, + "totalTokens": 272 } } }, { - "time": "2024-08-23T12:02:12.42432-04:00", + "time": "2024-10-14T17:38:52.587128-04:00", "callContext": { - "id": "1724428928", + "id": "1728941928", "tool": { "modelName": "mistral-large-2402 from github.com/gptscript-ai/mistral-laplateforme-provider", "internalPrompt": null, "tools": [ "bob" ], - "instructions": "Ask Bob \"how are you doing\" and repeat his reply exactly.", + "instructions": "Ask Bob \"how are you doing\" and repeat the response text exactly as given without saying anything else.", "id": "testdata/BobAsShell/test.gpt:", "toolMapping": { "bob": [ @@ -691,10 +623,10 @@ }, "type": "callFinish", "usage": {}, - "content": "Bob said, \"Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!\"" + "content": "Thanks for asking how are you doing, I'm doing great fellow friendly AI tool!" }, { - "time": "2024-08-23T12:02:12.424439-04:00", + "time": "2024-10-14T17:38:52.587221-04:00", "type": "runFinish", "usage": {} }