From fa23e9da13c49a9c3bd364f6923af69d43da5023 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 10 Oct 2025 18:40:05 +0530 Subject: [PATCH 01/89] SimpleChatToolCalling: Test/Explore srvr initial hs using cmdline --- .../public_simplechat/test-tools-cmdline.sh | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tools/server/public_simplechat/test-tools-cmdline.sh diff --git a/tools/server/public_simplechat/test-tools-cmdline.sh b/tools/server/public_simplechat/test-tools-cmdline.sh new file mode 100644 index 0000000000000..6aa97753c904e --- /dev/null +++ b/tools/server/public_simplechat/test-tools-cmdline.sh @@ -0,0 +1,83 @@ +echo "DONT FORGET TO RUN llama-server" +echo "build/bin/llama-server -m ~/Downloads/GenAi.Text/gemma-3n-E4B-it-Q8_0.gguf --path tools/server/public_simplechat --jinja" +curl http://localhost:8080/v1/chat/completions -d '{ + "model": "gpt-3.5-turbo", + "tools": [ + { + "type":"function", + "function":{ + "name":"javascript", + "description":"Runs code in an javascript interpreter and returns the result of the execution after 60 seconds.", + "parameters":{ + "type":"object", + "properties":{ + "code":{ + "type":"string", + "description":"The code to run in the javascript interpreter." + } + }, + "required":["code"] + } + } + }, + { + "type":"function", + "function":{ + "name":"web_fetch", + "description":"Connects to the internet and fetches the specified url, may take few seconds", + "parameters":{ + "type":"object", + "properties":{ + "url":{ + "type":"string", + "description":"The url to fetch from internet." + } + }, + "required":["url"] + } + } + }, + { + "type":"function", + "function":{ + "name":"simple_calc", + "description":"Calculates the provided arithmatic expression using javascript interpreter and returns the result of the execution after few seconds.", + "parameters":{ + "type":"object", + "properties":{ + "arithexp":{ + "type":"string", + "description":"The arithmatic expression that will be calculated using javascript interpreter." + } + }, + "required":["arithexp"] + } + } + } + ], + "messages": [ + { + "role": "user", + "content": "Add 324 to todays temperature in celcius in kochi. Dont forget to get todays weather info about kochi so that the temperature is valid. Use a free weather info site which doesnt require any api keys to get the info" + } + ] +}' + + +exit + + + "content": "Print a hello world message with python." + "content": "Print a hello world message with javascript." + "content": "Calculate the sum of 5 and 27." + "content": "Can you get me todays date." + "content": "Can you get todays date. And inturn add 10 to todays date" + "content": "Who is known as father of the nation in India, also is there a similar figure for USA as well as UK" + "content": "Who is known as father of the nation in India, Add 10 to double his year of birth and show me the results." + "content": "How is the weather today in london." + "content": "How is the weather today in london. Add 324 to todays temperature in celcius in london" + "content": "How is the weather today in london. Add 324 to todays temperature in celcius in kochi" + "content": "Add 324 to todays temperature in celcius in london" + "content": "Add 324 to todays temperature in celcius in delhi. Dont forget to get todays weather info about delhi so that the temperature is valid" + "content": "Add 324 to todays temperature in celcius in mumbai. Dont forget to get todays weather info about mumbai so that the temperature is valid. Use a free weather info site which doesnt require any api keys to get the info" + "content": "Can you get the cutoff rank for all the deemed medical universities in India for UGNeet 25" From 75ce9e47e1035c3a1e33b70188337f1b535986a1 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 10 Oct 2025 20:18:16 +0530 Subject: [PATCH 02/89] SimpleChatTools: Add boolean to allow user control of tools use --- tools/server/public_simplechat/simplechat.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 2fcd24a860bd4..bed0afbcdf559 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -726,6 +726,7 @@ class Me { this.defaultChatIds = [ "Default", "Other" ]; this.multiChat = new MultiChatUI(); this.bStream = true; + this.bTools = false; this.bCompletionFreshChatAlways = true; this.bCompletionInsertStandardRolePrefix = false; this.bTrimGarbage = true; @@ -804,6 +805,8 @@ class Me { ui.el_create_append_p(`bStream:${this.bStream}`, elDiv); + ui.el_create_append_p(`bTools:${this.bTools}`, elDiv); + ui.el_create_append_p(`bTrimGarbage:${this.bTrimGarbage}`, elDiv); ui.el_create_append_p(`ApiEndPoint:${this.apiEP}`, elDiv); @@ -878,6 +881,11 @@ class Me { }); elDiv.appendChild(bb.div); + bb = ui.el_creatediv_boolbutton("SetTools", "Tools", {true: "[+] yes tools", false: "[-] no tools"}, this.bTools, (val)=>{ + this.bTools = val; + }); + elDiv.appendChild(bb.div); + bb = ui.el_creatediv_boolbutton("SetTrimGarbage", "TrimGarbage", {true: "[+] yes trim", false: "[-] dont trim"}, this.bTrimGarbage, (val)=>{ this.bTrimGarbage = val; }); From 68fc28f0ec7e4f11f5a00182a9c68c3ed8c23b42 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 10 Oct 2025 23:02:30 +0530 Subject: [PATCH 03/89] SimpleChatTC: Update test shell script a bit Enable streaming by default, to check the handshake before going on to change the code, given that havent looked into this for more than a year now and have been busy with totally different stuff. Also updated the user messages used for testing a bit --- tools/server/public_simplechat/test-tools-cmdline.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/server/public_simplechat/test-tools-cmdline.sh b/tools/server/public_simplechat/test-tools-cmdline.sh index 6aa97753c904e..e5fb7652f4727 100644 --- a/tools/server/public_simplechat/test-tools-cmdline.sh +++ b/tools/server/public_simplechat/test-tools-cmdline.sh @@ -1,7 +1,10 @@ echo "DONT FORGET TO RUN llama-server" echo "build/bin/llama-server -m ~/Downloads/GenAi.Text/gemma-3n-E4B-it-Q8_0.gguf --path tools/server/public_simplechat --jinja" +echo "Note: Remove stream: true line below, if you want one shot instead of streaming response from ai server" +echo "Note: Using different locations below, as the mechanism / url used to fetch will / may need to change" curl http://localhost:8080/v1/chat/completions -d '{ "model": "gpt-3.5-turbo", + "stream": true, "tools": [ { "type":"function", @@ -58,7 +61,7 @@ curl http://localhost:8080/v1/chat/completions -d '{ "messages": [ { "role": "user", - "content": "Add 324 to todays temperature in celcius in kochi. Dont forget to get todays weather info about kochi so that the temperature is valid. Use a free weather info site which doesnt require any api keys to get the info" + "content": "what is your name." } ] }' @@ -67,6 +70,7 @@ curl http://localhost:8080/v1/chat/completions -d '{ exit + "content": "what is your name." "content": "Print a hello world message with python." "content": "Print a hello world message with javascript." "content": "Calculate the sum of 5 and 27." @@ -76,8 +80,9 @@ exit "content": "Who is known as father of the nation in India, Add 10 to double his year of birth and show me the results." "content": "How is the weather today in london." "content": "How is the weather today in london. Add 324 to todays temperature in celcius in london" - "content": "How is the weather today in london. Add 324 to todays temperature in celcius in kochi" + "content": "How is the weather today in bengaluru. Add 324 to todays temperature in celcius in kochi" "content": "Add 324 to todays temperature in celcius in london" + "content": "Add 324 to todays temperature in celcius in delhi" "content": "Add 324 to todays temperature in celcius in delhi. Dont forget to get todays weather info about delhi so that the temperature is valid" - "content": "Add 324 to todays temperature in celcius in mumbai. Dont forget to get todays weather info about mumbai so that the temperature is valid. Use a free weather info site which doesnt require any api keys to get the info" + "content": "Add 324 to todays temperature in celcius in bengaluru. Dont forget to get todays weather info about bengaluru so that the temperature is valid. Use a free weather info site which doesnt require any api keys to get the info" "content": "Can you get the cutoff rank for all the deemed medical universities in India for UGNeet 25" From 85845a011cb7f21326ef1a29c0b0b4061bbd9fec Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 10 Oct 2025 23:31:36 +0530 Subject: [PATCH 04/89] SimpleChatTC: Add skeleton for a javascript interpretor tool call Define the meta that needs to be passed to the GenAi Engine. Define the logic that implements the tool call, if called. Implement the flow/structure such that a single tool calls implementation file can define multiple tool calls. --- tools/server/public_simplechat/tooljs.mjs | 40 +++++++++++++++++++++++ tools/server/public_simplechat/tools.mjs | 7 ++++ 2 files changed, 47 insertions(+) create mode 100644 tools/server/public_simplechat/tooljs.mjs create mode 100644 tools/server/public_simplechat/tools.mjs diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs new file mode 100644 index 0000000000000..cfbdc61e56b63 --- /dev/null +++ b/tools/server/public_simplechat/tooljs.mjs @@ -0,0 +1,40 @@ +//@ts-check +// Helpers to handle tools/functions calling +// by Humans for All +// + + +let metas = [ + { + "type":"function", + "function":{ + "name": "javascript", + "description":"Runs code in an javascript interpreter and returns the result of the execution after 60 seconds.", + "parameters":{ + "type":"object", + "properties":{ + "code":{ + "type":"string", + "description":"The code to run in the javascript interpreter." + } + }, + "required":["code"] + } + } + } +] + + +/** + * Implementation of the javascript interpretor logic. Minimal skeleton for now. + * @param {any} obj + */ +function tool_run(obj) { + let func = new Function(obj["code"]) + func() +} + +let tswitch = { + "javascript": tool_run, +} + diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs new file mode 100644 index 0000000000000..6d0b375447cb7 --- /dev/null +++ b/tools/server/public_simplechat/tools.mjs @@ -0,0 +1,7 @@ +//@ts-check +// Helpers to handle tools/functions calling +// by Humans for All +// + + +import * as tjs from './tooljs.mjs' \ No newline at end of file From bbaae70503d56fc24dc2618beee951b12eae20c0 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 00:02:32 +0530 Subject: [PATCH 05/89] SimpleChatTC: More generic tooljs, SimpCalc, some main skeleton Make tooljs structure and flow more generic Add a simple_calculator tool/function call logic Add initial skeleton wrt the main tools.mjs file. --- tools/server/public_simplechat/tooljs.mjs | 77 ++++++++++++++++++----- tools/server/public_simplechat/tools.mjs | 26 +++++++- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index cfbdc61e56b63..1796bfaa2b1f0 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -1,40 +1,83 @@ //@ts-check -// Helpers to handle tools/functions calling +// DANGER DANGER DANGER - Simple and Stupid - Use from a discardable VM only +// Helpers to handle tools/functions calling wrt +// * javascript interpreter +// * simple arithmatic calculator // by Humans for All // -let metas = [ - { - "type":"function", - "function":{ +let js_meta = { + "type": "function", + "function": { "name": "javascript", - "description":"Runs code in an javascript interpreter and returns the result of the execution after 60 seconds.", - "parameters":{ - "type":"object", - "properties":{ - "code":{ - "type":"string", - "description":"The code to run in the javascript interpreter." + "description": "Runs code in an javascript interpreter and returns the result of the execution after few seconds", + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "The code to run in the javascript interpreter." } }, - "required":["code"] + "required": ["code"] } } } -] /** * Implementation of the javascript interpretor logic. Minimal skeleton for now. + * ALERT: Has access to the javascript environment and can mess with it and beyond * @param {any} obj */ -function tool_run(obj) { +function js_run(obj) { let func = new Function(obj["code"]) func() } -let tswitch = { - "javascript": tool_run, + +let calc_meta = { + "type": "function", + "function": { + "name": "simple_calculator", + "description": "Calculates the provided arithmatic expression using javascript interpreter and returns the result of the execution after few seconds", + "parameters": { + "type": "object", + "properties": { + "arithexpr":{ + "type":"string", + "description":"The arithmatic expression that will be calculated using javascript interpreter." + } + }, + "required": ["arithexpr"] + } + } + } + + +/** + * Implementation of the simple calculator logic. Minimal skeleton for now. + * ALERT: Has access to the javascript environment and can mess with it and beyond + * @param {any} obj + */ +function calc_run(obj) { + let func = new Function(obj["arithexpr"]) + func() +} + + +/** + * @type {Object} + */ +export let tc_switch = { + "javascript": { + "handler": js_run, + "meta": js_meta + }, + "simple_calculator": { + "handler": calc_run, + "meta": calc_meta + } } diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index 6d0b375447cb7..ba80e30a91347 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -1,7 +1,29 @@ //@ts-check -// Helpers to handle tools/functions calling +// DANGER DANGER DANGER - Simple and Stupid - Use from a discardable VM only +// Helpers to handle tools/functions calling in a direct and dangerous way // by Humans for All // -import * as tjs from './tooljs.mjs' \ No newline at end of file +import * as tjs from './tooljs.mjs' + + +/** + * @type {Object} + */ +let tc_switch = {} + +export function setup() { + for (const key in tjs.tc_switch) { + tc_switch[key] = tjs.tc_switch[key] + } +} + +export function meta() { + let tools = [] + for (const key in tc_switch) { + tools.push(tc_switch[key]["meta"]) + } + return tools +} + From f09156831c4399cab69e11686d4cd743b022fa84 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 00:18:37 +0530 Subject: [PATCH 06/89] SimpleChatTC: Bring in the tools meta into the main flow --- tools/server/public_simplechat/simplechat.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index bed0afbcdf559..6f33457e733f2 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -4,6 +4,8 @@ import * as du from "./datautils.mjs"; import * as ui from "./ui.mjs" +import * as tools from "./tools.mjs" + class Roles { static System = "system"; @@ -228,6 +230,9 @@ class SimpleChat { if (gMe.bStream) { obj["stream"] = true; } + if (gMe.bTools) { + obj["tools"] = tools.meta(); + } return JSON.stringify(obj); } @@ -927,6 +932,8 @@ function startme() { gMe.debug_disable(); document["gMe"] = gMe; document["du"] = du; + document["tools"] = tools; + tools.setup() for (let cid of gMe.defaultChatIds) { gMe.multiChat.new_chat_session(cid); } From 9d8be85b0eac91a901bf1693d6da5e029702d00c Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 01:06:23 +0530 Subject: [PATCH 07/89] SimpleChatTC: use tcpdump to dbg hs; check if ai aware of tools --- tools/server/public_simplechat/test-tools-cmdline.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/test-tools-cmdline.sh b/tools/server/public_simplechat/test-tools-cmdline.sh index e5fb7652f4727..adea59cb6c279 100644 --- a/tools/server/public_simplechat/test-tools-cmdline.sh +++ b/tools/server/public_simplechat/test-tools-cmdline.sh @@ -2,6 +2,7 @@ echo "DONT FORGET TO RUN llama-server" echo "build/bin/llama-server -m ~/Downloads/GenAi.Text/gemma-3n-E4B-it-Q8_0.gguf --path tools/server/public_simplechat --jinja" echo "Note: Remove stream: true line below, if you want one shot instead of streaming response from ai server" echo "Note: Using different locations below, as the mechanism / url used to fetch will / may need to change" +echo "Note: sudo tcpdump -i lo -s 0 -vvv -A host 127.0.0.1 and port 8080 | tee /tmp/td.log can be used to capture the hs" curl http://localhost:8080/v1/chat/completions -d '{ "model": "gpt-3.5-turbo", "stream": true, @@ -61,7 +62,7 @@ curl http://localhost:8080/v1/chat/completions -d '{ "messages": [ { "role": "user", - "content": "what is your name." + "content": "What and all tools you have access to" } ] }' @@ -71,6 +72,7 @@ exit "content": "what is your name." + "content": "What and all tools you have access to" "content": "Print a hello world message with python." "content": "Print a hello world message with javascript." "content": "Calculate the sum of 5 and 27." From 2e4693cf0dfa0f06023eb81ba79190562da0a42e Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 01:17:19 +0530 Subject: [PATCH 08/89] SimpleChatTC: Skeleton to handle diff fields when streaming Changed latestResponse type to an object instead of a string. Inturn it contains entries for content, toolname and toolargs. Added a custom clear logic due to the same and used it to replace the previously simple assigning of empty string to latestResponse. For now in all places where latestReponse is used, I have replaced with latestReponse.content. Next need to handle identifying the field being streamed and inturn append to it. Also need to add logic to call tool, when tool_call triggered by genai. --- tools/server/public_simplechat/simplechat.js | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 6f33457e733f2..bc3d2f00f566f 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -72,7 +72,7 @@ class SimpleChat { */ this.xchat = []; this.iLastSys = -1; - this.latestResponse = ""; + this.latestResponse = { content: "", toolname: "", toolargs: "" }; } clear() { @@ -80,6 +80,10 @@ class SimpleChat { this.iLastSys = -1; } + clear_latestresponse() { + this.latestResponse = { content: "", toolname: "", toolargs: "" }; + } + ods_key() { return `SimpleChat-${this.chatId}` } @@ -151,7 +155,7 @@ class SimpleChat { * @param {string} content */ append_response(content) { - this.latestResponse += content; + this.latestResponse.content += content; } /** @@ -392,7 +396,7 @@ class SimpleChat { } let tdUtf8 = new TextDecoder("utf-8"); let rr = resp.body.getReader(); - this.latestResponse = ""; + this.clear_latestresponse() let xLines = new du.NewLines(); while(true) { let { value: cur, done: done } = await rr.read(); @@ -419,14 +423,14 @@ class SimpleChat { console.debug("DBUG:SC:PART:Json:", curJson); this.append_response(this.response_extract_stream(curJson, apiEP)); } - elP.innerText = this.latestResponse; + elP.innerText = this.latestResponse.content; elP.scrollIntoView(false); if (done) { break; } } - console.debug("DBUG:SC:PART:Full:", this.latestResponse); - return this.latestResponse; + console.debug("DBUG:SC:PART:Full:", this.latestResponse.content); + return this.latestResponse.content; } /** @@ -455,11 +459,11 @@ class SimpleChat { if (gMe.bStream) { try { theResp.assistant = await this.handle_response_multipart(resp, apiEP, elDiv); - this.latestResponse = ""; + this.clear_latestresponse() } catch (error) { - theResp.assistant = this.latestResponse; + theResp.assistant = this.latestResponse.content; this.add(Roles.Assistant, theResp.assistant); - this.latestResponse = ""; + this.clear_latestresponse() throw error; } } else { From 27161cb110fda1b534802b636c0788aa3f77e36b Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 01:46:23 +0530 Subject: [PATCH 09/89] SimpleChatTC: Extract streamed field - assume only 1f at any time Update response_extract_stream to check for which field is being currently streamed ie is it normal content or tool call func name or tool call func args and then return the field name and extracted value. Previously it was always assumed that only normal content will be returned. Currently it is assumed that the server will only stream one of the 3 supported fields at any time and not more than one of them at the same time. TODO: Have to also add logic to extract the reasoning field later, ie wrt gen ai models which give out their thinking. Have updated append_response to expect both the key and the value wrt the latestResponse object, which it will be manipualted. Previously it was always assumed that content is what will be got and inturn appended. --- tools/server/public_simplechat/simplechat.js | 27 +++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index bc3d2f00f566f..5adae099715ac 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -152,10 +152,10 @@ class SimpleChat { /** * Collate the latest response from the server/ai-model, as it is becoming available. * This is mainly useful for the stream mode. - * @param {string} content + * @param {{key: string, value: string}} resp */ - append_response(content) { - this.latestResponse.content += content; + append_response(resp) { + this.latestResponse[resp.key] += resp.value; } /** @@ -311,10 +311,25 @@ class SimpleChat { * @param {string} apiEP */ response_extract_stream(respBody, apiEP) { + let key = "content" let assistant = ""; if (apiEP == ApiEP.Type.Chat) { - if (respBody["choices"][0]["finish_reason"] !== "stop") { - assistant = respBody["choices"][0]["delta"]["content"]; + if (respBody["choices"][0]["finish_reason"] !== null) { + if (respBody["choices"][0]["delta"]["content"] !== undefined) { + assistant = respBody["choices"][0]["delta"]["content"]; + } else { + if (respBody["choices"][0]["delta"]["tool_calls"] !== undefined) { + if (respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["name"] !== undefined) { + key = "toolname"; + assistant = respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["name"]; + } else { + if (respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] !== undefined) { + key = "toolargs"; + assistant = respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"]; + } + } + } + } } } else { try { @@ -323,7 +338,7 @@ class SimpleChat { assistant = respBody["content"]; } } - return assistant; + return { key: key, value: assistant }; } /** From 788d56aeee3d56c2c03d04a2e3c737b9ce6fdf5e Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 02:35:01 +0530 Subject: [PATCH 10/89] SimpleChatTC: Avoid null content, Fix oversight wrt finish_reason I was wrongly checking for finish_reason to be non null, before trying to extract the genai content/toolcalls, have fixed this oversight with the new flow in progress. I had added few debug logs to identify the above issue, need to remove them later. Note: given that debug logs are disabled by replacing the debug function during this program's initialisation, which I had forgotten about, I didnt get the debug messages and had to scratch my head a bit, before realising this and the other issue ;) Also either when I had originally implemented simplechat 1+ years back, or later due to changes on the server end, the streaming flow sends a initial null wrt the content, where it only sets the role. This was not handled in my flow on the client side, so a null was getting prepended to the chat messages/responses from the server. This has been fixed now in the new generic flow. --- tools/server/public_simplechat/simplechat.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 5adae099715ac..cd075f37bdf04 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -155,6 +155,10 @@ class SimpleChat { * @param {{key: string, value: string}} resp */ append_response(resp) { + if (resp.value == null) { + return + } + console.debug(resp.key, resp.value) this.latestResponse[resp.key] += resp.value; } @@ -311,10 +315,11 @@ class SimpleChat { * @param {string} apiEP */ response_extract_stream(respBody, apiEP) { + console.debug(respBody, apiEP) let key = "content" let assistant = ""; if (apiEP == ApiEP.Type.Chat) { - if (respBody["choices"][0]["finish_reason"] !== null) { + if (respBody["choices"][0]["finish_reason"] === null) { if (respBody["choices"][0]["delta"]["content"] !== undefined) { assistant = respBody["choices"][0]["delta"]["content"]; } else { From 4cbe1d291aba4714d73cd8460b2604e3f29001b4 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 02:58:03 +0530 Subject: [PATCH 11/89] SimpleChatTC: Show toolcall being generated by ai - Temp --- tools/server/public_simplechat/simplechat.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index cd075f37bdf04..4fefe48ea4ab5 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -443,7 +443,11 @@ class SimpleChat { console.debug("DBUG:SC:PART:Json:", curJson); this.append_response(this.response_extract_stream(curJson, apiEP)); } - elP.innerText = this.latestResponse.content; + if (this.latestResponse.content !== "") { + elP.innerText = this.latestResponse.content; + } else { + elP.innerText = `ToolCall:${this.latestResponse.toolname}:${this.latestResponse.toolargs}`; + } elP.scrollIntoView(false); if (done) { break; From 174b0b1548744cdf660a8bccc5284f83aafeb04a Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 03:14:27 +0530 Subject: [PATCH 12/89] SimpleChatTC: AssistantResponse class initial go Make latestResponse into a new class based type instance wrt ai assistant response, which is what it represents. Move clearing, appending fields' values and getting assistant's response info (irrespective of a content or toolcall response) into this new class and inturn use the same. --- tools/server/public_simplechat/simplechat.js | 70 ++++++++++++-------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 4fefe48ea4ab5..510749c17c88d 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -37,6 +37,39 @@ class ApiEP { } +class AssistantResponse { + + constructor() { + this.response = { content: "", toolname: "", toolargs: "" }; + } + + clear() { + this.response = { content: "", toolname: "", toolargs: "" }; + } + + /** + * Helps collate the latest response from the server/ai-model, as it is becoming available. + * This is mainly useful for the stream mode. + * @param {{key: string, value: string}} resp + */ + append_response(resp) { + if (resp.value == null) { + return + } + console.debug(resp.key, resp.value) + this.response[resp.key] += resp.value; + } + + content_equiv() { + if (this.response.content !== "") { + return this.response.content; + } else { + return `ToolCall:${this.response.toolname}:${this.response.toolargs}`; + } + } + +} + let gUsageMsg = `

Usage

@@ -72,7 +105,7 @@ class SimpleChat { */ this.xchat = []; this.iLastSys = -1; - this.latestResponse = { content: "", toolname: "", toolargs: "" }; + this.latestResponse = new AssistantResponse(); } clear() { @@ -80,10 +113,6 @@ class SimpleChat { this.iLastSys = -1; } - clear_latestresponse() { - this.latestResponse = { content: "", toolname: "", toolargs: "" }; - } - ods_key() { return `SimpleChat-${this.chatId}` } @@ -149,19 +178,6 @@ class SimpleChat { return rchat; } - /** - * Collate the latest response from the server/ai-model, as it is becoming available. - * This is mainly useful for the stream mode. - * @param {{key: string, value: string}} resp - */ - append_response(resp) { - if (resp.value == null) { - return - } - console.debug(resp.key, resp.value) - this.latestResponse[resp.key] += resp.value; - } - /** * Add an entry into xchat * @param {string} role @@ -416,7 +432,7 @@ class SimpleChat { } let tdUtf8 = new TextDecoder("utf-8"); let rr = resp.body.getReader(); - this.clear_latestresponse() + this.latestResponse.clear() let xLines = new du.NewLines(); while(true) { let { value: cur, done: done } = await rr.read(); @@ -441,20 +457,16 @@ class SimpleChat { } let curJson = JSON.parse(curLine); console.debug("DBUG:SC:PART:Json:", curJson); - this.append_response(this.response_extract_stream(curJson, apiEP)); - } - if (this.latestResponse.content !== "") { - elP.innerText = this.latestResponse.content; - } else { - elP.innerText = `ToolCall:${this.latestResponse.toolname}:${this.latestResponse.toolargs}`; + this.latestResponse.append_response(this.response_extract_stream(curJson, apiEP)); } + elP.innerText = this.latestResponse.content_equiv() elP.scrollIntoView(false); if (done) { break; } } - console.debug("DBUG:SC:PART:Full:", this.latestResponse.content); - return this.latestResponse.content; + console.debug("DBUG:SC:PART:Full:", this.latestResponse.content_equiv()); + return this.latestResponse; } /** @@ -483,11 +495,11 @@ class SimpleChat { if (gMe.bStream) { try { theResp.assistant = await this.handle_response_multipart(resp, apiEP, elDiv); - this.clear_latestresponse() + this.latestResponse.clear() } catch (error) { theResp.assistant = this.latestResponse.content; this.add(Roles.Assistant, theResp.assistant); - this.clear_latestresponse() + this.latestResponse.clear() throw error; } } else { From 10b1013eb9c30e307e5c424dcb0366de19305cd6 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 03:31:18 +0530 Subject: [PATCH 13/89] SimpleChatTC: AssistantResponse everywhere initial go Switch oneshot handler to use AssistantResponse, inturn currenlty only handle the normal content in the response. TODO: If any tool_calls in the oneshot response, it is currently not handled. Inturn switch the generic/toplevel handle response logic to use AssistantResponse class, given that both oneshot and the multipart/streaming flows use/return it. Inturn add trimmedContent member to AssistantResponse class and make the generic handle response logic to save the trimmed content into this. Update users of trimmed to work with this structure. --- tools/server/public_simplechat/simplechat.js | 39 ++++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 510749c17c88d..c38c740efe9e0 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -40,11 +40,11 @@ class ApiEP { class AssistantResponse { constructor() { - this.response = { content: "", toolname: "", toolargs: "" }; + this.response = { content: "", toolname: "", toolargs: "", trimmedContent: "" }; } clear() { - this.response = { content: "", toolname: "", toolargs: "" }; + this.response = { content: "", toolname: "", toolargs: "", trimmedContent: "" }; } /** @@ -312,14 +312,14 @@ class SimpleChat { * @param {string} apiEP */ response_extract(respBody, apiEP) { - let assistant = ""; + let assistant = new AssistantResponse(); if (apiEP == ApiEP.Type.Chat) { - assistant = respBody["choices"][0]["message"]["content"]; + assistant.response.content = respBody["choices"][0]["message"]["content"]; } else { try { - assistant = respBody["choices"][0]["text"]; + assistant.response.content = respBody["choices"][0]["text"]; } catch { - assistant = respBody["content"]; + assistant.response.content = respBody["content"]; } } return assistant; @@ -483,34 +483,33 @@ class SimpleChat { /** * Handle the response from the server be it in oneshot or multipart/stream mode. * Also take care of the optional garbage trimming. + * TODO: Need to handle tool calling and related flow, including how to show + * the assistant's request for tool calling and the response from tool. * @param {Response} resp * @param {string} apiEP * @param {HTMLDivElement} elDiv */ async handle_response(resp, apiEP, elDiv) { - let theResp = { - assistant: "", - trimmed: "", - } + let theResp = null if (gMe.bStream) { try { - theResp.assistant = await this.handle_response_multipart(resp, apiEP, elDiv); + theResp = await this.handle_response_multipart(resp, apiEP, elDiv); this.latestResponse.clear() } catch (error) { - theResp.assistant = this.latestResponse.content; - this.add(Roles.Assistant, theResp.assistant); + theResp = this.latestResponse; + this.add(Roles.Assistant, theResp.content_equiv()); this.latestResponse.clear() throw error; } } else { - theResp.assistant = await this.handle_response_oneshot(resp, apiEP); + theResp = await this.handle_response_oneshot(resp, apiEP); } if (gMe.bTrimGarbage) { - let origMsg = theResp.assistant; - theResp.assistant = du.trim_garbage_at_end(origMsg); - theResp.trimmed = origMsg.substring(theResp.assistant.length); + let origMsg = theResp.response.content; + theResp.response.content = du.trim_garbage_at_end(origMsg); + theResp.response.trimmedContent = origMsg.substring(theResp.response.content.length); } - this.add(Roles.Assistant, theResp.assistant); + this.add(Roles.Assistant, theResp.content_equiv()); return theResp; } @@ -678,8 +677,8 @@ class MultiChatUI { let theResp = await chat.handle_response(resp, apiEP, this.elDivChat); if (chatId == this.curChatId) { chat.show(this.elDivChat); - if (theResp.trimmed.length > 0) { - let p = ui.el_create_append_p(`TRIMMED:${theResp.trimmed}`, this.elDivChat); + if (theResp.response.trimmedContent.length > 0) { + let p = ui.el_create_append_p(`TRIMMED:${theResp.response.trimmedContent}`, this.elDivChat); p.className="role-trim"; } } else { From e4e29a245216301cfeb482a502e7890f27784b64 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 04:03:53 +0530 Subject: [PATCH 14/89] SimpleChatTC: twins wrt streamed response handling As there could be failure wrt getting the response from the ai server some where in between a long response spread over multiple parts, the logic uses the latestResponse to cache the response as it is being received. However once the full response is got, one needs to transfer it to a new instance of AssistantResponse class, so that latestResponse can be cleared, while the new instance can be used in other locations in the flow as needed. Achieve the same now. --- tools/server/public_simplechat/simplechat.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index c38c740efe9e0..7cce5b5f5f8da 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -39,8 +39,16 @@ class ApiEP { class AssistantResponse { - constructor() { - this.response = { content: "", toolname: "", toolargs: "", trimmedContent: "" }; + constructor(content="", toolname="", toolargs="", trimmedContent="") { + this.response = { content: content, toolname: toolname, toolargs: toolargs, trimmedContent: trimmedContent }; + } + + /** + * Create a new instance from an existing instance + * @param {AssistantResponse} old + */ + static newFrom(old) { + return new AssistantResponse(old.response.content, old.response.toolname, old.response.toolargs, old.response.trimmedContent) } clear() { @@ -466,7 +474,7 @@ class SimpleChat { } } console.debug("DBUG:SC:PART:Full:", this.latestResponse.content_equiv()); - return this.latestResponse; + return AssistantResponse.newFrom(this.latestResponse); } /** From d7f612ffd03c69e0e46806de6f1cc952241154e0 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 04:22:15 +0530 Subject: [PATCH 15/89] SimpleChatTC: Saner/Robust AssistantResponse content_equiv Previously if content was empty, it would have always sent the toolcall info related version even if there was no toolcall info in it. Fixed now to return empty string, if both content and toolname are empty. --- tools/server/public_simplechat/simplechat.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 7cce5b5f5f8da..8c9fa56698e7a 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -71,8 +71,10 @@ class AssistantResponse { content_equiv() { if (this.response.content !== "") { return this.response.content; - } else { + } else if (this.response.toolname !== "") { return `ToolCall:${this.response.toolname}:${this.response.toolargs}`; + } else { + return "" } } From 2a2769778f578e362e5225bb59b998459dcdf13c Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sat, 11 Oct 2025 23:27:08 +0530 Subject: [PATCH 16/89] SimpleChatTC:tooljs: Trap console.log and store in new result key The implementations of javascript and simple_calculator now use provided helpers to trap console.log messages when they execute the code / expression provided by GenAi and inturn store the captured log messages in the newly added result key in tc_switch This should help trap the output generated by the provided code or expression as the case maybe and inturn return the same to the GenAi, for its further processing. --- tools/server/public_simplechat/tooljs.mjs | 48 +++++++++++++++++++++-- tools/server/public_simplechat/tools.mjs | 2 +- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 1796bfaa2b1f0..628563ebd1cd6 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -7,6 +7,40 @@ // +let gConsoleStr = "" +/** + * @type { {(...data: any[]): void} | null} + */ +let gOrigConsoleLog = null + + +/** + * @param {any[]} args + */ +function console_trapped(...args) { + let res = args.map((arg)=>{ + if (typeof arg == 'object') { + return JSON.stringify(arg); + } else { + return String(arg); + } + }).join(' '); + gConsoleStr += res; +} + +function console_redir() { + gOrigConsoleLog = console.log + console.log = console_trapped + gConsoleStr = "" +} + +function console_revert() { + if (gOrigConsoleLog !== null) { + console.log = gOrigConsoleLog + } +} + + let js_meta = { "type": "function", "function": { @@ -32,8 +66,11 @@ let js_meta = { * @param {any} obj */ function js_run(obj) { + console_redir() let func = new Function(obj["code"]) func() + console_revert() + tc_switch["javascript"]["result"] = gConsoleStr } @@ -62,22 +99,27 @@ let calc_meta = { * @param {any} obj */ function calc_run(obj) { + console_redir() let func = new Function(obj["arithexpr"]) func() + console_revert() + tc_switch["simple_calculator"]["result"] = gConsoleStr } /** - * @type {Object} + * @type {Object>} */ export let tc_switch = { "javascript": { "handler": js_run, - "meta": js_meta + "meta": js_meta, + "result": "" }, "simple_calculator": { "handler": calc_run, - "meta": calc_meta + "meta": calc_meta, + "result": "" } } diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index ba80e30a91347..d249a3f5433cd 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -9,7 +9,7 @@ import * as tjs from './tooljs.mjs' /** - * @type {Object} + * @type {Object>} */ let tc_switch = {} From 92b82ae5c7388331fe0957a4d964a3d8cc48d10e Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 01:20:53 +0530 Subject: [PATCH 17/89] SimpleChatTC: Implement a simple toolcall handling flow Checks for toolname to be defined or not in the GenAi's response If toolname is set, then check if a corresponding tool/func exists, and if so call the same by passing it the GenAi provided toolargs as a object. Inturn the text generated by the tool/func is captured and put into the user input entry text box, with tool_response tag around it. --- tools/server/public_simplechat/simplechat.js | 24 ++++++++++++++++++++ tools/server/public_simplechat/tools.mjs | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 8c9fa56698e7a..f0156e007ffb9 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -190,6 +190,7 @@ class SimpleChat { /** * Add an entry into xchat + * Also update iLastSys system prompt index tracker * @param {string} role * @param {string|undefined|null} content */ @@ -398,6 +399,7 @@ class SimpleChat { /** * Allow setting of system prompt, at any time. + * Updates the system prompt, if one was never set or if the newly passed is different from the last set system prompt. * @param {string} sysPrompt * @param {string} msgTag */ @@ -523,6 +525,24 @@ class SimpleChat { return theResp; } + /** + * Call the requested tool/function and get its response + * @param {AssistantResponse} ar + */ + async handle_toolcall(ar) { + let toolname = ar.response.toolname.trim(); + if (toolname === "") { + return undefined + } + for (const fn in tools.tc_switch) { + if (fn == toolname) { + tools.tc_switch[fn]["handler"](JSON.parse(ar.response.toolargs)) + return tools.tc_switch[fn]["result"] + } + } + return `Unknown Tool/Function Call:${toolname}` + } + } @@ -694,6 +714,10 @@ class MultiChatUI { } else { console.debug(`DBUG:SimpleChat:MCUI:HandleUserSubmit:ChatId has changed:[${chatId}] [${this.curChatId}]`); } + let toolResult = await chat.handle_toolcall(theResp) + if (toolResult !== undefined) { + this.elInUser.value = `${toolResult}` + } this.ui_reset_userinput(); } diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index d249a3f5433cd..adf87fbdf432e 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -11,7 +11,7 @@ import * as tjs from './tooljs.mjs' /** * @type {Object>} */ -let tc_switch = {} +export let tc_switch = {} export function setup() { for (const key in tjs.tc_switch) { From d8b1b36bb3fe2567cec7e774d80780e8f3ac1272 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 01:36:43 +0530 Subject: [PATCH 18/89] SimpleChatTC: Cleanup initial/1st go toolcall flow As output generated by any tool/function call is currently placed into the TextArea provided for End user (for their queries), bcas the GenAi (engine/LLM) may be expecting the tool response to be sent as a user role data with tool_response tag surrounding the results from the tool call. So also now at the end of submit btn click handling, the end user input text area is not cleared, if there was a tool call handled, for above reasons. Also given that running a simple arithmatic expression in itself doesnt generate any output, so wrap them in a console.log, to help capture the result using the console.log trapping flow that is already setup. --- tools/server/public_simplechat/simplechat.js | 11 +++++++---- tools/server/public_simplechat/tooljs.mjs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index f0156e007ffb9..618bb95679fa1 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -586,12 +586,15 @@ class MultiChatUI { /** * Reset user input ui. - * * clear user input + * * clear user input (if requested, default true) * * enable user input * * set focus to user input + * @param {boolean} [bClearElInUser=true] */ - ui_reset_userinput() { - this.elInUser.value = ""; + ui_reset_userinput(bClearElInUser=true) { + if (bClearElInUser) { + this.elInUser.value = ""; + } this.elInUser.disabled = false; this.elInUser.focus(); } @@ -718,7 +721,7 @@ class MultiChatUI { if (toolResult !== undefined) { this.elInUser.value = `${toolResult}` } - this.ui_reset_userinput(); + this.ui_reset_userinput(toolResult === undefined); } /** diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 628563ebd1cd6..ae62fd017691d 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -100,7 +100,7 @@ let calc_meta = { */ function calc_run(obj) { console_redir() - let func = new Function(obj["arithexpr"]) + let func = new Function(`console.log(${obj["arithexpr"]})`) func() console_revert() tc_switch["simple_calculator"]["result"] = gConsoleStr From 7a2bcfb5590700e4143b114ad8a84c8fff28c2d4 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 02:53:08 +0530 Subject: [PATCH 19/89] SimpleChatTC: Trap any exception raised during tool call and inform the GenAi/LLM about the same --- tools/server/public_simplechat/simplechat.js | 8 ++++++-- tools/server/public_simplechat/test-tools-cmdline.sh | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 618bb95679fa1..36cd5a59372f3 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -536,8 +536,12 @@ class SimpleChat { } for (const fn in tools.tc_switch) { if (fn == toolname) { - tools.tc_switch[fn]["handler"](JSON.parse(ar.response.toolargs)) - return tools.tc_switch[fn]["result"] + try { + tools.tc_switch[fn]["handler"](JSON.parse(ar.response.toolargs)) + return tools.tc_switch[fn]["result"] + } catch (error) { + return `Tool/Function call raised an exception:${error.name}:${error.message}` + } } } return `Unknown Tool/Function Call:${toolname}` diff --git a/tools/server/public_simplechat/test-tools-cmdline.sh b/tools/server/public_simplechat/test-tools-cmdline.sh index adea59cb6c279..8fc62d2af9a48 100644 --- a/tools/server/public_simplechat/test-tools-cmdline.sh +++ b/tools/server/public_simplechat/test-tools-cmdline.sh @@ -73,10 +73,12 @@ exit "content": "what is your name." "content": "What and all tools you have access to" + "content": "do you have access to any tools" "content": "Print a hello world message with python." "content": "Print a hello world message with javascript." "content": "Calculate the sum of 5 and 27." "content": "Can you get me todays date." + "content": "Can you get me a summary of latest news from bbc world" "content": "Can you get todays date. And inturn add 10 to todays date" "content": "Who is known as father of the nation in India, also is there a similar figure for USA as well as UK" "content": "Who is known as father of the nation in India, Add 10 to double his year of birth and show me the results." From f10ab96c77a49eec1b7d5e18d1f9992c80bbca51 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 12:58:37 +0530 Subject: [PATCH 20/89] SimpleChatTC: More clearer description of toolcalls execution env Should hopeful ensure that the GenAi/LLM will generate appropriate code/expression as the argument to pass to these tool calls, to some extent. --- tools/server/public_simplechat/tooljs.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index ae62fd017691d..c2bbc0c43c3a1 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -44,14 +44,14 @@ function console_revert() { let js_meta = { "type": "function", "function": { - "name": "javascript", - "description": "Runs code in an javascript interpreter and returns the result of the execution after few seconds", + "name": "run_javascript_function_code", + "description": "Runs given code as a function in a browser's javascript environment and returns the console.log outputs of the execution after few seconds", "parameters": { "type": "object", "properties": { "code": { "type": "string", - "description": "The code to run in the javascript interpreter." + "description": "The code belonging to a function to run in the browser's javascript interpreter." } }, "required": ["code"] @@ -78,13 +78,13 @@ let calc_meta = { "type": "function", "function": { "name": "simple_calculator", - "description": "Calculates the provided arithmatic expression using javascript interpreter and returns the result of the execution after few seconds", + "description": "Calculates the provided arithmatic expression using console.log of a browser's javascript interpreter and returns the output of the execution once it is done in few seconds", "parameters": { "type": "object", "properties": { "arithexpr":{ "type":"string", - "description":"The arithmatic expression that will be calculated using javascript interpreter." + "description":"The arithmatic expression that will be calculated by passing it to console.log of a browser's javascript interpreter." } }, "required": ["arithexpr"] @@ -111,7 +111,7 @@ function calc_run(obj) { * @type {Object>} */ export let tc_switch = { - "javascript": { + "run_javascript_function_code": { "handler": js_run, "meta": js_meta, "result": "" From a1f177656d7381ed5f1adae5e0e423a8e94944a2 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 13:01:59 +0530 Subject: [PATCH 21/89] SimpleChatTC: Clarify some type definitions to avoid warnings ie in vs code with ts-check --- tools/server/public_simplechat/simplechat.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 36cd5a59372f3..e39d9245ed2cc 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -18,6 +18,7 @@ class ApiEP { Chat: "chat", Completion: "completion", } + /** @type {Object} */ static UrlSuffix = { 'chat': `/chat/completions`, 'completion': `/completions`, @@ -40,6 +41,7 @@ class ApiEP { class AssistantResponse { constructor(content="", toolname="", toolargs="", trimmedContent="") { + /** @type {Object} */ this.response = { content: content, toolname: toolname, toolargs: toolargs, trimmedContent: trimmedContent }; } @@ -539,7 +541,7 @@ class SimpleChat { try { tools.tc_switch[fn]["handler"](JSON.parse(ar.response.toolargs)) return tools.tc_switch[fn]["result"] - } catch (error) { + } catch (/** @type {any} */error) { return `Tool/Function call raised an exception:${error.name}:${error.message}` } } @@ -824,11 +826,15 @@ class Me { "Last4": 5, }; this.apiEP = ApiEP.Type.Chat; + /** @type {Object} */ this.headers = { "Content-Type": "application/json", "Authorization": "", // Authorization: Bearer OPENAI_API_KEY } - // Add needed fields wrt json object to be sent wrt LLM web services completions endpoint. + /** + * Add needed fields wrt json object to be sent wrt LLM web services completions endpoint. + * @type {Object} + */ this.apiRequestOptions = { "model": "gpt-3.5-turbo", "temperature": 0.7, From 4ac6f0a59bbd1b533e6267d0f03afe7a6ecec80e Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 16:08:11 +0530 Subject: [PATCH 22/89] SimpleChatTC: Move tool calling to tools, try trap async failures Move tool calling logic into tools module. Try trap async promise failures by awaiting results of tool calling and putting full thing in an outer try catch. Have forgotten the nitty gritties of JS flow, this might help, need to check. --- tools/server/public_simplechat/simplechat.js | 14 ++++---------- tools/server/public_simplechat/tools.mjs | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index e39d9245ed2cc..4e243db2f02c6 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -536,17 +536,11 @@ class SimpleChat { if (toolname === "") { return undefined } - for (const fn in tools.tc_switch) { - if (fn == toolname) { - try { - tools.tc_switch[fn]["handler"](JSON.parse(ar.response.toolargs)) - return tools.tc_switch[fn]["result"] - } catch (/** @type {any} */error) { - return `Tool/Function call raised an exception:${error.name}:${error.message}` - } - } + try { + return await tools.tool_call(toolname, ar.response.toolargs) + } catch (/** @type {any} */error) { + return `Tool/Function call raised an exception:${error.name}:${error.message}` } - return `Unknown Tool/Function Call:${toolname}` } } diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index adf87fbdf432e..686d47a241b23 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -27,3 +27,22 @@ export function meta() { return tools } + +/** + * Try call the specified tool/function call and return its response + * @param {string} toolname + * @param {string} toolargs + */ +export async function tool_call(toolname, toolargs) { + for (const fn in tc_switch) { + if (fn == toolname) { + try { + tc_switch[fn]["handler"](JSON.parse(toolargs)) + return tc_switch[fn]["result"] + } catch (/** @type {any} */error) { + return `Tool/Function call raised an exception:${error.name}:${error.message}` + } + } + } + return `Unknown Tool/Function Call:${toolname}` +} From 379630622545ff0d1388c82e93c98ac2727c38c9 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 17:19:51 +0530 Subject: [PATCH 23/89] SimpleChatTC: Pass toolname to the tool handler So that when tool handler writes the result to the tc_switch, it can make use of the same, to write to the right location. NOTE: This also fixes the issue with I forgetting to rename the key in js_run wrt writing of result. --- tools/server/public_simplechat/tooljs.mjs | 10 ++++++---- tools/server/public_simplechat/tools.mjs | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index c2bbc0c43c3a1..dfaab850099bf 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -63,14 +63,15 @@ let js_meta = { /** * Implementation of the javascript interpretor logic. Minimal skeleton for now. * ALERT: Has access to the javascript environment and can mess with it and beyond + * @param {string} toolname * @param {any} obj */ -function js_run(obj) { +function js_run(toolname, obj) { console_redir() let func = new Function(obj["code"]) func() console_revert() - tc_switch["javascript"]["result"] = gConsoleStr + tc_switch[toolname]["result"] = gConsoleStr } @@ -96,14 +97,15 @@ let calc_meta = { /** * Implementation of the simple calculator logic. Minimal skeleton for now. * ALERT: Has access to the javascript environment and can mess with it and beyond + * @param {string} toolname * @param {any} obj */ -function calc_run(obj) { +function calc_run(toolname, obj) { console_redir() let func = new Function(`console.log(${obj["arithexpr"]})`) func() console_revert() - tc_switch["simple_calculator"]["result"] = gConsoleStr + tc_switch[toolname]["result"] = gConsoleStr } diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index 686d47a241b23..d75d3eb7bf73d 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -37,7 +37,7 @@ export async function tool_call(toolname, toolargs) { for (const fn in tc_switch) { if (fn == toolname) { try { - tc_switch[fn]["handler"](JSON.parse(toolargs)) + tc_switch[fn]["handler"](fn, JSON.parse(toolargs)) return tc_switch[fn]["result"] } catch (/** @type {any} */error) { return `Tool/Function call raised an exception:${error.name}:${error.message}` From 0ed8329a84791da9842caa69b379c317535aae5e Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 21:21:00 +0530 Subject: [PATCH 24/89] SimpleChatTC: Cleanup the function description a bit to better describe how it will be run, so that genai/llm while creating the code to run, will hopefully take care of any naunces required. --- tools/server/public_simplechat/tooljs.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index dfaab850099bf..b79b79dd821de 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -45,13 +45,13 @@ let js_meta = { "type": "function", "function": { "name": "run_javascript_function_code", - "description": "Runs given code as a function in a browser's javascript environment and returns the console.log outputs of the execution after few seconds", + "description": "Runs given code using function constructor mechanism in a browser's javascript environment and returns the console.log outputs of the execution after few seconds", "parameters": { "type": "object", "properties": { "code": { "type": "string", - "description": "The code belonging to a function to run in the browser's javascript interpreter." + "description": "The code belonging to the dynamic function to run in the browser's javascript interpreter environment." } }, "required": ["code"] From aa81f519332c2f46a5215578c9e44c072ca15ac3 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 22:16:34 +0530 Subject: [PATCH 25/89] SimpleChatTC: Update the readme.md wrt tool calling a bit --- tools/server/public_simplechat/readme.md | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index 24e026d455b03..388202156e935 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -78,6 +78,7 @@ Once inside * try trim garbage in response or not * amount of chat history in the context sent to server/ai-model * oneshot or streamed mode. + * use built in tool calling or not * In completion mode * one normally doesnt use a system prompt in completion mode. @@ -116,6 +117,13 @@ Once inside * the user input box will be disabled and a working message will be shown in it. * if trim garbage is enabled, the logic will try to trim repeating text kind of garbage to some extent. +* tool calling flow + * if tool calling is enabled and the user query results in need for one of the builtin tools to be + called, then the response will include request for tool call. + * the SimpleChat client will call the requested tool and inturn place the returned result into user + entry text area with generated result + * if user is ok with the tool response, they can click submit to send the same to the GenAi/LLM. + * just refresh the page, to reset wrt the chat history and or system prompt and start afresh. * Using NewChat one can start independent chat sessions. @@ -158,6 +166,15 @@ It is attached to the document object. Some of these can also be updated using t inturn the machine goes into power saving mode or so, the platform may stop network connection, leading to exception. + bTools - control whether tool calling is enabled or not + + remember to enable this only for GenAi/LLM models which support tool/function calling. + + the builtin tools meta data is sent to the ai model in the requests sent to it. + + inturn if the ai model requests a tool call to be made, the same will be done and the response + sent back to the ai model, under user control. + apiEP - select between /completions and /chat/completions endpoint provided by the server/ai-model. bCompletionFreshChatAlways - whether Completion mode collates complete/sliding-window history when @@ -281,6 +298,27 @@ NOTE: Not tested, as there is no free tier api testing available. However logica work. +### Tool Calling + +Provide a descriptive meta data explaining the tool / function being provided for tool calling. + +Provide a handler which should implement the specified tool / function call. It should place +the result to be sent back to the ai model in the result key of the tc_switch entry for the +corresponding tool. + +Update the tc_switch to include a object entry for the tool, which inturn icnludes +* the meta data as well as +* a reference to the handler and also +* the result key + + +### Debuging the handshake + +When working with llama.cpp server based GenAi/LLM running locally + +sudo tcpdump -i lo -s 0 -vvv -A host 127.0.0.1 and port 8080 | tee /tmp/td.log + + ## At the end Also a thank you to all open source and open model developers, who strive for the common good. From 5ed2bc3f481d55100f3fc8199074aad31fce3105 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Sun, 12 Oct 2025 23:50:35 +0530 Subject: [PATCH 26/89] SimpleChatTC: ToolCall hs info in normal assistant-user chat flow Also as part of same, wrap the request details in the assistant block using a similar tagging format as the tool_response in user block. --- tools/server/public_simplechat/readme.md | 19 +++++++++++++++++++ tools/server/public_simplechat/simplechat.js | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index 388202156e935..19e04520692e7 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -300,6 +300,8 @@ work. ### Tool Calling +#### Extending wiht new tools + Provide a descriptive meta data explaining the tool / function being provided for tool calling. Provide a handler which should implement the specified tool / function call. It should place @@ -311,6 +313,23 @@ Update the tc_switch to include a object entry for the tool, which inturn icnlud * a reference to the handler and also * the result key +#### Mapping tool calls and responses to normal assistant - user chat flow + +Instead of maintaining tool_call request and resultant response in logically seperate parallel +channel used for requesting tool_calls by the assistant and the resulstant tool role response, +the SimpleChatTC pushes it into the normal assistant - user chat flow itself, by including the +tool call and response as a pair of tagged request with details in the assistant block and inturn +tagged response in the subsequent user block. + +This allows the GenAi/LLM to be aware of the tool calls it made as well as the responses it got, +so that it can incorporate the results of the same in the subsequent chat / interactions. + +NOTE: This flow tested to be ok enough with Gemma-3N-E4B-it-Q8_0 LLM ai model for now. + +TODO: Need to think later, whether to continue this simple flow, or atleast use tool role wrt +the tool call responses or even go further and have the logically seperate tool_call request +structures also. + ### Debuging the handshake diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 4e243db2f02c6..83911fde42662 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -74,7 +74,7 @@ class AssistantResponse { if (this.response.content !== "") { return this.response.content; } else if (this.response.toolname !== "") { - return `ToolCall:${this.response.toolname}:${this.response.toolargs}`; + return `\n${this.response.toolname}\n${this.response.toolargs}\n`; } else { return "" } From 619d64d30ebcf42394067c5af8ee5270fdfc1945 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 00:41:19 +0530 Subject: [PATCH 27/89] SimpleChatTC: Add ui elements for tool call verify and trigger Instead of automatically calling the requested tool with supplied arguments, rather allow user to verify things before triggering the tool. NOTE: User already provided control over tool_response before submitting it to the ai assistant. --- tools/server/public_simplechat/index.html | 9 +++++++++ tools/server/public_simplechat/simplechat.js | 14 +++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/index.html b/tools/server/public_simplechat/index.html index f6413016fcc53..c9b508eecfe93 100644 --- a/tools/server/public_simplechat/index.html +++ b/tools/server/public_simplechat/index.html @@ -40,6 +40,15 @@

You need to have javascript enabled.

+
+
+
+ + +
+ +
+
diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 83911fde42662..73e1cafc80f6c 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -91,7 +91,11 @@ let gUsageMsg = `
  • Completion mode - no system prompt normally.
  • Use shift+enter for inserting enter/newline.
  • -
  • Enter your query to ai assistant below.
  • +
  • Enter your query to ai assistant in textarea provided below.
  • +
  • If ai assistant requests a tool call, varify same before triggering it.
  • +
      +
    • submit tool response placed into user query textarea
    • +
  • Default ContextWindow = [System, Last Query+Resp, Cur Query].
    • ChatHistInCtxt, MaxTokens, ModelCtxt window to expand
    • @@ -562,6 +566,10 @@ class MultiChatUI { this.elDivHeading = /** @type{HTMLSelectElement} */(document.getElementById("heading")); this.elDivSessions = /** @type{HTMLDivElement} */(document.getElementById("sessions-div")); this.elBtnSettings = /** @type{HTMLButtonElement} */(document.getElementById("settings")); + this.elDivTool = /** @type{HTMLDivElement} */(document.getElementById("tool-div")); + this.elBtnTool = /** @type{HTMLButtonElement} */(document.getElementById("tool-btn")); + this.elInToolName = /** @type{HTMLInputElement} */(document.getElementById("toolname-in")); + this.elInToolArgs = /** @type{HTMLInputElement} */(document.getElementById("toolargs-in")); this.validate_element(this.elInSystem, "system-in"); this.validate_element(this.elDivChat, "chat-div"); @@ -569,6 +577,10 @@ class MultiChatUI { this.validate_element(this.elDivHeading, "heading"); this.validate_element(this.elDivChat, "sessions-div"); this.validate_element(this.elBtnSettings, "settings"); + this.validate_element(this.elDivTool, "tool-div"); + this.validate_element(this.elInToolName, "toolname-in"); + this.validate_element(this.elInToolArgs, "toolargs-in"); + this.validate_element(this.elBtnTool, "tool-btn"); } /** From 226aa7d4aa55510f63acb10092ff5977238507ab Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 01:19:37 +0530 Subject: [PATCH 28/89] SimpleChatTC: Let user trigger tool call, instead of automatic Instead of automatically calling any requested tool by the GenAi / llm, that is from the tail end of the handle user submit btn click, Now if the GenAi/LLM has requested any tool to be called, then enable the Tool Run related UI elements and fill them with the tool name and tool args. In turn the user can verify if they are ok with the tool being called and the arguments being passed to it. Rather they can even fix any errors in the tool usage like the arithmatic expr to calculate that is being passed to simple_calculator or the javascript code being passed to run_javascript_function_code If user is ok with the tool call being requested, then trigger the same. The results if any will be automatically placed into the user query text area. User can cross verify if they are ok with the result and or modify it suitabley if required and inturn submit the same to the GenAi/LLM. --- tools/server/public_simplechat/simplechat.js | 55 +++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 73e1cafc80f6c..9b4cc58a15687 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -70,10 +70,17 @@ class AssistantResponse { this.response[resp.key] += resp.value; } + has_toolcall() { + if (this.response.toolname.trim() == "") { + return false + } + return true + } + content_equiv() { if (this.response.content !== "") { return this.response.content; - } else if (this.response.toolname !== "") { + } else if (this.has_toolcall()) { return `\n${this.response.toolname}\n${this.response.toolargs}\n`; } else { return "" @@ -533,15 +540,15 @@ class SimpleChat { /** * Call the requested tool/function and get its response - * @param {AssistantResponse} ar + * @param {string} toolname + * @param {string} toolargs */ - async handle_toolcall(ar) { - let toolname = ar.response.toolname.trim(); + async handle_toolcall(toolname, toolargs) { if (toolname === "") { return undefined } try { - return await tools.tool_call(toolname, ar.response.toolargs) + return await tools.tool_call(toolname, toolargs) } catch (/** @type {any} */error) { return `Tool/Function call raised an exception:${error.name}:${error.message}` } @@ -596,6 +603,24 @@ class MultiChatUI { } } + /** + * Reset/Setup Tool Call UI parts as needed + * @param {AssistantResponse} ar + */ + ui_reset_toolcall_as_needed(ar) { + if (ar.has_toolcall()) { + this.elDivTool.hidden = false + this.elInToolName.value = ar.response.toolname + this.elInToolArgs.value = ar.response.toolargs + this.elBtnTool.disabled = false + } else { + this.elDivTool.hidden = true + this.elInToolName.value = "" + this.elInToolArgs.value = "" + this.elBtnTool.disabled = true + } + } + /** * Reset user input ui. * * clear user input (if requested, default true) @@ -641,6 +666,13 @@ class MultiChatUI { }); }); + this.elBtnTool.addEventListener("click", (ev)=>{ + if (this.elDivTool.hidden) { + return; + } + this.handle_tool_run(this.curChatId); + }) + this.elInUser.addEventListener("keyup", (ev)=> { // allow user to insert enter into their message using shift+enter. // while just pressing enter key will lead to submitting. @@ -729,7 +761,18 @@ class MultiChatUI { } else { console.debug(`DBUG:SimpleChat:MCUI:HandleUserSubmit:ChatId has changed:[${chatId}] [${this.curChatId}]`); } - let toolResult = await chat.handle_toolcall(theResp) + this.ui_reset_toolcall_as_needed(theResp); + this.ui_reset_userinput(); + } + + /** + * @param {string} chatId + */ + async handle_tool_run(chatId) { + let chat = this.simpleChats[chatId]; + this.elInUser.value = "toolcall in progress..."; + this.elInUser.disabled = true; + let toolResult = await chat.handle_toolcall(this.elInToolName.value, this.elInToolArgs.value) if (toolResult !== undefined) { this.elInUser.value = `${toolResult}` } From 2aabca21357e2ec69803fbf0f46cf6231fd591ed Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 02:57:17 +0530 Subject: [PATCH 29/89] SimpleChatTC: Update readme with bit more details, Cleaner UI Also avoid showing Tool calling UI elements, when not needed to be shown. --- tools/server/public_simplechat/readme.md | 54 +++++++++++++++++--- tools/server/public_simplechat/simplechat.js | 5 ++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index 19e04520692e7..d2a1e22df63f2 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -33,6 +33,10 @@ Allows developer/end-user to control some of the behaviour by updating gMe membe console. Parallely some of the directly useful to end-user settings can also be changed using the provided settings ui. +For GenAi/LLM models supporting tool / function calling, allows one to interact with them and explore use of +ai driven augmenting of the knowledge used for generating answers by using the predefined tools/functions. +The end user is provided control over tool calling and response submitting. + NOTE: Current web service api doesnt expose the model context length directly, so client logic doesnt provide any adaptive culling of old messages nor of replacing them with summary of their content etal. However there is a optional sliding window based chat logic, which provides a simple minded culling of old messages from @@ -117,12 +121,15 @@ Once inside * the user input box will be disabled and a working message will be shown in it. * if trim garbage is enabled, the logic will try to trim repeating text kind of garbage to some extent. -* tool calling flow +* tool calling flow when working with ai models which support tool / function calling * if tool calling is enabled and the user query results in need for one of the builtin tools to be - called, then the response will include request for tool call. - * the SimpleChat client will call the requested tool and inturn place the returned result into user - entry text area with generated result + called, then the ai response might include request for tool call. + * the SimpleChat client will show details of the tool call (ie tool name and args passed) requested + and allow the user to trigger it as is or after modifying things as needed. + * inturn returned / generated result is placed into user query entry text area with approriate tags + ie generated result * if user is ok with the tool response, they can click submit to send the same to the GenAi/LLM. + User can even modify the response generated by the tool, if required, before submitting. * just refresh the page, to reset wrt the chat history and or system prompt and start afresh. @@ -170,11 +177,15 @@ It is attached to the document object. Some of these can also be updated using t remember to enable this only for GenAi/LLM models which support tool/function calling. - the builtin tools meta data is sent to the ai model in the requests sent to it. + the builtin tools' meta data is sent to the ai model in the requests sent to it. inturn if the ai model requests a tool call to be made, the same will be done and the response sent back to the ai model, under user control. + as tool calling will involve a bit of back and forth between ai assistant and end user, it is + recommended to set iRecentUserMsgCnt to 5 or more, so that enough context is retained during + chatting with ai models with tool support. + apiEP - select between /completions and /chat/completions endpoint provided by the server/ai-model. bCompletionFreshChatAlways - whether Completion mode collates complete/sliding-window history when @@ -300,9 +311,31 @@ work. ### Tool Calling +ALERT: Currently the way this is implemented, it is dangerous to use this, unless one verifies +all the tool calls requested and the responses generated manually to ensure everything is fine, +during interaction with ai modles with tools support. + +#### Builtin Tools + +The following tools/functions are currently provided by default +* simple_calculator - which can solve simple arithmatic expressions +* run_javascript_function_code - which can be used to run some javascript code in the browser + context. + +Currently the generated code / expression is run through a simple dynamic function mechanism. +May update things, in future, so that a WebWorker is used to avoid exposing browser global scope +to the generated code directly. Either way always remember to cross check the tool requests and +generated responses when using tool calling. + +May add +* web_fetch along with a corresponding simple local proxy server logic that can bypass the + CORS restrictions applied if trying to directly fetch from the browser js runtime environment. + + #### Extending wiht new tools -Provide a descriptive meta data explaining the tool / function being provided for tool calling. +Provide a descriptive meta data explaining the tool / function being provided for tool calling, +as well as its arguments. Provide a handler which should implement the specified tool / function call. It should place the result to be sent back to the ai model in the result key of the tc_switch entry for the @@ -330,6 +363,15 @@ TODO: Need to think later, whether to continue this simple flow, or atleast use the tool call responses or even go further and have the logically seperate tool_call request structures also. +#### ToDo + +Update to use web worker. + +Make the Tool Call related ui elements use up horizontal space properly. + +Try and trap promises based flows to ensure all generated results or errors if any are caught +before responding back to the ai model. + ### Debuging the handshake diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 9b4cc58a15687..599b147adcd59 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -649,6 +649,8 @@ class MultiChatUI { this.handle_session_switch(this.curChatId); } + this.ui_reset_toolcall_as_needed(new AssistantResponse()); + this.elBtnSettings.addEventListener("click", (ev)=>{ this.elDivChat.replaceChildren(); gMe.show_settings(this.elDivChat); @@ -729,6 +731,8 @@ class MultiChatUI { chat.clear(); } + this.ui_reset_toolcall_as_needed(new AssistantResponse()); + chat.add_system_anytime(this.elInSystem.value, chatId); let content = this.elInUser.value; @@ -766,6 +770,7 @@ class MultiChatUI { } /** + * Handle running of specified tool call if any, for the specified chat session. * @param {string} chatId */ async handle_tool_run(chatId) { From 90b2491223f21ee2d34639f3dde0ab5fe88458cf Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 03:18:09 +0530 Subject: [PATCH 30/89] SimpleChatTC: Tool Calling UI elements use up horizontal space --- tools/server/public_simplechat/index.html | 6 +++++- tools/server/public_simplechat/readme.md | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/index.html b/tools/server/public_simplechat/index.html index c9b508eecfe93..3cd840569c3a7 100644 --- a/tools/server/public_simplechat/index.html +++ b/tools/server/public_simplechat/index.html @@ -42,11 +42,15 @@
      +
      - +
      +
      +
      +

      diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index d2a1e22df63f2..29214369ec18d 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -328,8 +328,9 @@ to the generated code directly. Either way always remember to cross check the to generated responses when using tool calling. May add -* web_fetch along with a corresponding simple local proxy server logic that can bypass the - CORS restrictions applied if trying to directly fetch from the browser js runtime environment. +* web_fetch along with a corresponding simple local web proxy/caching server logic that can bypass + the CORS restrictions applied if trying to directly fetch from the browser js runtime environment. + Inturn maybe with a white list of allowed sites to access or so. #### Extending wiht new tools @@ -367,7 +368,7 @@ structures also. Update to use web worker. -Make the Tool Call related ui elements use up horizontal space properly. +WebFetch and Local web proxy/caching server Try and trap promises based flows to ensure all generated results or errors if any are caught before responding back to the ai model. From a8eadc429932e304beb7c19b5ac6fb38cc346764 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 03:32:58 +0530 Subject: [PATCH 31/89] SimpleChatTC: Update readme wrt --jinja argument and bit more --- tools/server/public_simplechat/readme.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index 29214369ec18d..6f9f986b01844 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -7,7 +7,7 @@ by Humans for All. To run from the build dir -bin/llama-server -m path/model.gguf --path ../tools/server/public_simplechat +bin/llama-server -m path/model.gguf --path ../tools/server/public_simplechat --jinja Continue reading for the details. @@ -68,6 +68,16 @@ next run this web front end in tools/server/public_simplechat * cd ../tools/server/public_simplechat * python3 -m http.server PORT +### for tool calling + +remember to + +* pass --jinja to llama-server to enable tool calling support from the server ai engine end. + +* enable bTools in the settings page of the client side gui. + +* use a GenAi/LLM model which supports tool calling. + ### using the front end Open this simple web front end from your local browser From 70bc1b42b91364037e293a41df6bc5a64cf377cf Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 22:11:25 +0530 Subject: [PATCH 32/89] SimpleChatTC: Move console.log trapping into its own module So that it can be used from different modules, if required. --- tools/server/public_simplechat/tooljs.mjs | 45 +++---------------- .../server/public_simplechat/toolsconsole.mjs | 38 ++++++++++++++++ 2 files changed, 45 insertions(+), 38 deletions(-) create mode 100644 tools/server/public_simplechat/toolsconsole.mjs diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index b79b79dd821de..23b340e514b17 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -7,38 +7,7 @@ // -let gConsoleStr = "" -/** - * @type { {(...data: any[]): void} | null} - */ -let gOrigConsoleLog = null - - -/** - * @param {any[]} args - */ -function console_trapped(...args) { - let res = args.map((arg)=>{ - if (typeof arg == 'object') { - return JSON.stringify(arg); - } else { - return String(arg); - } - }).join(' '); - gConsoleStr += res; -} - -function console_redir() { - gOrigConsoleLog = console.log - console.log = console_trapped - gConsoleStr = "" -} - -function console_revert() { - if (gOrigConsoleLog !== null) { - console.log = gOrigConsoleLog - } -} +import * as tconsole from "./toolsconsole.mjs" let js_meta = { @@ -67,11 +36,11 @@ let js_meta = { * @param {any} obj */ function js_run(toolname, obj) { - console_redir() + tconsole.console_redir() let func = new Function(obj["code"]) func() - console_revert() - tc_switch[toolname]["result"] = gConsoleStr + tconsole.console_revert() + tc_switch[toolname]["result"] = tconsole.gConsoleStr } @@ -101,11 +70,11 @@ let calc_meta = { * @param {any} obj */ function calc_run(toolname, obj) { - console_redir() + tconsole.console_redir() let func = new Function(`console.log(${obj["arithexpr"]})`) func() - console_revert() - tc_switch[toolname]["result"] = gConsoleStr + tconsole.console_revert() + tc_switch[toolname]["result"] = tconsole.gConsoleStr } diff --git a/tools/server/public_simplechat/toolsconsole.mjs b/tools/server/public_simplechat/toolsconsole.mjs new file mode 100644 index 0000000000000..0c7d436a0af02 --- /dev/null +++ b/tools/server/public_simplechat/toolsconsole.mjs @@ -0,0 +1,38 @@ +//@ts-check +// Helpers to handle tools/functions calling wrt console +// by Humans for All +// + + +export let gConsoleStr = "" +/** + * @type { {(...data: any[]): void} | null} + */ +export let gOrigConsoleLog = null + + +/** + * @param {any[]} args + */ +export function console_trapped(...args) { + let res = args.map((arg)=>{ + if (typeof arg == 'object') { + return JSON.stringify(arg); + } else { + return String(arg); + } + }).join(' '); + gConsoleStr += res; +} + +export function console_redir() { + gOrigConsoleLog = console.log + console.log = console_trapped + gConsoleStr = "" +} + +export function console_revert() { + if (gOrigConsoleLog !== null) { + console.log = gOrigConsoleLog + } +} From f8ebe8f8dd9738bd23ca0a846be31398cba1417b Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 22:31:31 +0530 Subject: [PATCH 33/89] SimpleChatTC:ToolsConsole:Cleanup a bit, add basic set of notes Try ensure as well as verify that original console.log is saved and not overwritten. Throw an exception if things seem off wrt same. Also ensure to add a newline at end of console.log messages --- .../server/public_simplechat/toolsconsole.mjs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tools/server/public_simplechat/toolsconsole.mjs b/tools/server/public_simplechat/toolsconsole.mjs index 0c7d436a0af02..b372dc74ef329 100644 --- a/tools/server/public_simplechat/toolsconsole.mjs +++ b/tools/server/public_simplechat/toolsconsole.mjs @@ -4,14 +4,17 @@ // +/** The redirected console.log's capture-data-space */ export let gConsoleStr = "" /** + * Maintain original console.log, when needed * @type { {(...data: any[]): void} | null} */ -export let gOrigConsoleLog = null +let gOrigConsoleLog = null /** + * The trapping console.log * @param {any[]} args */ export function console_trapped(...args) { @@ -22,17 +25,33 @@ export function console_trapped(...args) { return String(arg); } }).join(' '); - gConsoleStr += res; + gConsoleStr += `${res}\n`; } +/** + * Save the original console.log, if needed. + * Setup redir of console.log. + * Clear the redirected console.log's capture-data-space. + */ export function console_redir() { - gOrigConsoleLog = console.log + if (gOrigConsoleLog == null) { + if (console.log == console_trapped) { + throw new Error("ERRR:ToolsConsole:ReDir:Original Console.Log lost???"); + } + gOrigConsoleLog = console.log + } console.log = console_trapped gConsoleStr = "" } +/** + * Revert the redirected console.log to the original console.log, if possible. + */ export function console_revert() { if (gOrigConsoleLog !== null) { + if (gOrigConsoleLog == console_trapped) { + throw new Error("ERRR:ToolsConsole:Revert:Original Console.Log lost???"); + } console.log = gOrigConsoleLog } } From cc60600bb39f2ea64f22e30c02ba619d25e962e0 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Mon, 13 Oct 2025 23:07:44 +0530 Subject: [PATCH 34/89] SimpleChatTC: Initial skeleton of a simple toolsworker --- .../server/public_simplechat/toolsworker.mjs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tools/server/public_simplechat/toolsworker.mjs diff --git a/tools/server/public_simplechat/toolsworker.mjs b/tools/server/public_simplechat/toolsworker.mjs new file mode 100644 index 0000000000000..b17c5bb197cea --- /dev/null +++ b/tools/server/public_simplechat/toolsworker.mjs @@ -0,0 +1,19 @@ +//@ts-check +// STILL DANGER DANGER DANGER - Simple and Stupid - Use from a discardable VM only +// Helpers to handle tools/functions calling using web worker +// by Humans for All +// + +import * as tconsole from "./toolsconsole.mjs" + +tconsole.console_redir() + +onmessage = async (ev) => { + try { + eval(ev.data) + } catch (/** @type {any} */error) { + console.log(`\n\nTool/Function call raised an exception:${error.name}:${error.message}\n\n`) + } + tconsole.console_revert() + postMessage(tconsole.gConsoleStr) +} From 7ea9bf6bf632d21fab4de9c9691f6f5e34758636 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 00:54:21 +0530 Subject: [PATCH 35/89] SimpleChatTC: Pass around structured objects wrt tool worker The request for code to run as well as the resultant response data both need to follow a structured object convention, so that it is easy to map a request and the corresponding response to some extent. --- tools/server/public_simplechat/toolsworker.mjs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/toolsworker.mjs b/tools/server/public_simplechat/toolsworker.mjs index b17c5bb197cea..d17b772b4f66b 100644 --- a/tools/server/public_simplechat/toolsworker.mjs +++ b/tools/server/public_simplechat/toolsworker.mjs @@ -4,16 +4,23 @@ // by Humans for All // +/** + * Expects to get a message with identifier name and code to run + * Posts message with identifier name and data captured from console.log outputs + */ + + import * as tconsole from "./toolsconsole.mjs" + tconsole.console_redir() -onmessage = async (ev) => { +self.onmessage = function (ev) { try { - eval(ev.data) + eval(ev.data.code) } catch (/** @type {any} */error) { - console.log(`\n\nTool/Function call raised an exception:${error.name}:${error.message}\n\n`) + console.log(`\n\nTool/Function call "${ev.data.name}" raised an exception:${error.name}:${error.message}\n\n`) } tconsole.console_revert() - postMessage(tconsole.gConsoleStr) + self.postMessage({ name: ev.data.name, data: tconsole.gConsoleStr}) } From 466474822cd9fcaee32a81aac8a71d1861102e50 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 00:58:23 +0530 Subject: [PATCH 36/89] SimpleChatTC: Actual tool call implementations simplified These no longer need to worry about * setting up the console.log related redirection to capture the generated outputs, nor about * setting up a dynamic function for executing the needed tool call related code The web worker setup to help run tool calls in a relatively isolated environment independent of the main browser env, takes care of these. One needs to only worry about getting the handle to the web worker to use and inturn pass the need code wrt the tool call to it. --- tools/server/public_simplechat/tooljs.mjs | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 23b340e514b17..5ee8e83004b35 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -7,7 +7,7 @@ // -import * as tconsole from "./toolsconsole.mjs" +let gToolsWorker = /** @type{Worker} */(/** @type {unknown} */(null)); let js_meta = { @@ -31,16 +31,12 @@ let js_meta = { /** * Implementation of the javascript interpretor logic. Minimal skeleton for now. - * ALERT: Has access to the javascript environment and can mess with it and beyond + * ALERT: Has access to the javascript web worker environment and can mess with it and beyond * @param {string} toolname * @param {any} obj */ function js_run(toolname, obj) { - tconsole.console_redir() - let func = new Function(obj["code"]) - func() - tconsole.console_revert() - tc_switch[toolname]["result"] = tconsole.gConsoleStr + gToolsWorker.postMessage({ name: toolname, code: obj["code"]}) } @@ -65,16 +61,12 @@ let calc_meta = { /** * Implementation of the simple calculator logic. Minimal skeleton for now. - * ALERT: Has access to the javascript environment and can mess with it and beyond + * ALERT: Has access to the javascript web worker environment and can mess with it and beyond * @param {string} toolname * @param {any} obj */ function calc_run(toolname, obj) { - tconsole.console_redir() - let func = new Function(`console.log(${obj["arithexpr"]})`) - func() - tconsole.console_revert() - tc_switch[toolname]["result"] = tconsole.gConsoleStr + gToolsWorker.postMessage({ name: toolname, code: `console.log(${obj["arithexpr"]})`}) } @@ -94,3 +86,11 @@ export let tc_switch = { } } + +/** + * Used to get hold of the web worker to use for running tool/function call related code + * @param {Worker} toolsWorker + */ +export function init(toolsWorker) { + gToolsWorker = toolsWorker +} From 5933b2866966dc454327dcac2b5f7789923653b9 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 01:06:00 +0530 Subject: [PATCH 37/89] SimpleChatTC: Get ready for decoupled tool call response tools manager/module * setup the web worker that will help execute the tool call related codes in a js environment that is isolated from the browsers main js environment * pass the web worker to the tool call providers, for them to use * dont wait for the result from the tool call, as it will be got later asynchronously through a message * allow users of the tools manager to register a call back, which will be called when ever a message is got from the web worker containing response wrt previously requested tool call execution. simplechat * decouple toolcall response handling and toolcall requesting logic * setup a timeout to take back control if tool call takes up too much time. Inturn help alert the ai model, that the tool call took up too much time and so was aborted, by placing a approriate tagged tool response into user query area. * register a call back that will be called when response is got asynchronously wrt anye requested tool calls. In turn take care of updating the user query area with response got wrt the tool call, along with tool response tag around it. --- tools/server/public_simplechat/simplechat.js | 24 ++++++++++++++++---- tools/server/public_simplechat/tools.mjs | 21 ++++++++++++++--- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 599b147adcd59..925a181dbfa87 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -539,13 +539,15 @@ class SimpleChat { } /** - * Call the requested tool/function and get its response + * Call the requested tool/function. + * Returns undefined, if the call was placed successfully + * Else some appropriate error message will be returned. * @param {string} toolname * @param {string} toolargs */ async handle_toolcall(toolname, toolargs) { if (toolname === "") { - return undefined + return "Tool/Function call name not specified" } try { return await tools.tool_call(toolname, toolargs) @@ -675,6 +677,11 @@ class MultiChatUI { this.handle_tool_run(this.curChatId); }) + tools.setup((name, data)=>{ + this.elInUser.value = `${data}` + this.ui_reset_userinput(false) + }) + this.elInUser.addEventListener("keyup", (ev)=> { // allow user to insert enter into their message using shift+enter. // while just pressing enter key will lead to submitting. @@ -771,17 +778,24 @@ class MultiChatUI { /** * Handle running of specified tool call if any, for the specified chat session. + * Also sets up a timeout, so that user gets control back to interact with the ai model. * @param {string} chatId */ async handle_tool_run(chatId) { let chat = this.simpleChats[chatId]; this.elInUser.value = "toolcall in progress..."; this.elInUser.disabled = true; - let toolResult = await chat.handle_toolcall(this.elInToolName.value, this.elInToolArgs.value) + let toolname = this.elInToolName.value.trim() + let toolResult = await chat.handle_toolcall(toolname, this.elInToolArgs.value) if (toolResult !== undefined) { this.elInUser.value = `${toolResult}` + this.ui_reset_userinput(false) + } else { + setTimeout(() => { + this.elInUser.value = `Tool/Function call ${toolname} taking too much time, aborting...` + this.ui_reset_userinput(false) + }, 10000) } - this.ui_reset_userinput(toolResult === undefined); } /** @@ -1073,7 +1087,7 @@ function startme() { document["gMe"] = gMe; document["du"] = du; document["tools"] = tools; - tools.setup() + tools.init() for (let cid of gMe.defaultChatIds) { gMe.multiChat.new_chat_session(cid); } diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index d75d3eb7bf73d..4ece70ae565ed 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -8,12 +8,14 @@ import * as tjs from './tooljs.mjs' +let gToolsWorker = new Worker('./toolsworker.mjs'); /** * @type {Object>} */ export let tc_switch = {} -export function setup() { +export function init() { + tjs.init(gToolsWorker) for (const key in tjs.tc_switch) { tc_switch[key] = tjs.tc_switch[key] } @@ -27,9 +29,22 @@ export function meta() { return tools } +/** + * Setup the callback that will be called when ever message + * is recieved from the Tools Web Worker. + * @param {(name: string, data: string) => void} cb + */ +export function setup(cb) { + gToolsWorker.onmessage = function (ev) { + cb(ev.data.name, ev.data.data) + } +} + /** - * Try call the specified tool/function call and return its response + * Try call the specified tool/function call. + * Returns undefined, if the call was placed successfully + * Else some appropriate error message will be returned. * @param {string} toolname * @param {string} toolargs */ @@ -38,7 +53,7 @@ export async function tool_call(toolname, toolargs) { if (fn == toolname) { try { tc_switch[fn]["handler"](fn, JSON.parse(toolargs)) - return tc_switch[fn]["result"] + return undefined } catch (/** @type {any} */error) { return `Tool/Function call raised an exception:${error.name}:${error.message}` } From 50be171bccf24c28d810db02f5d5c7ae544fbbb5 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 01:34:22 +0530 Subject: [PATCH 38/89] SimpleChatTC: Web worker flow initial go cleanup Had forgotten to specify type as module wrt web worker, in order to allow it to import the toolsconsole module. Had forgotten to maintain the id of the timeout handler, which is needed to clear/stop the timeout handler from triggering, if tool call response is got well in time. As I am currently reverting the console redirection at end of handling a tool call code in the web worker message handler, I need to setup the redirection each time. Also I had forgotten to clear the console.log capture data space, before a new tool call code is executed, this is also fixed by this change. TODO: Need to abort the tool call code execution in the web worker if possible in future, if the client / browser side times out waiting for tool call response, ie if the tool call code is taking up too much time. --- tools/server/public_simplechat/simplechat.js | 3 ++- tools/server/public_simplechat/tools.mjs | 2 +- tools/server/public_simplechat/toolsworker.mjs | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 925a181dbfa87..6b897448c2f5c 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -678,6 +678,7 @@ class MultiChatUI { }) tools.setup((name, data)=>{ + clearTimeout(this.idTimeOut) this.elInUser.value = `${data}` this.ui_reset_userinput(false) }) @@ -791,7 +792,7 @@ class MultiChatUI { this.elInUser.value = `${toolResult}` this.ui_reset_userinput(false) } else { - setTimeout(() => { + this.idTimeOut = setTimeout(() => { this.elInUser.value = `Tool/Function call ${toolname} taking too much time, aborting...` this.ui_reset_userinput(false) }, 10000) diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index 4ece70ae565ed..75fe56e4f4e5f 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -8,7 +8,7 @@ import * as tjs from './tooljs.mjs' -let gToolsWorker = new Worker('./toolsworker.mjs'); +let gToolsWorker = new Worker('./toolsworker.mjs', { type: 'module' }); /** * @type {Object>} */ diff --git a/tools/server/public_simplechat/toolsworker.mjs b/tools/server/public_simplechat/toolsworker.mjs index d17b772b4f66b..e370fd0a9df34 100644 --- a/tools/server/public_simplechat/toolsworker.mjs +++ b/tools/server/public_simplechat/toolsworker.mjs @@ -13,9 +13,8 @@ import * as tconsole from "./toolsconsole.mjs" -tconsole.console_redir() - self.onmessage = function (ev) { + tconsole.console_redir() try { eval(ev.data.code) } catch (/** @type {any} */error) { From 44cfebcdd08ea25c27aa2b8ae7da6066333d5bc9 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 01:56:34 +0530 Subject: [PATCH 39/89] SimpleChatTC: Increase the sliding window context to Last4 QA As the tool calling, if enabled, will need access to last few user query and ai assistant responses (which will also include in them the tool call requests and the corresponding results), so that the model can build answers based on its tool call reqs and got responses, and also given that most of the models these days have sufficiently large context windows, so the sliding window context implemented by SimpleChat logic has been increased by default to include last 4 query and their responses roughlty. --- tools/server/public_simplechat/simplechat.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 6b897448c2f5c..4804c88ea0d28 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -103,7 +103,7 @@ let gUsageMsg = `
      • submit tool response placed into user query textarea
      -
    • Default ContextWindow = [System, Last Query+Resp, Cur Query].
    • +
    • Default ContextWindow = [System, Last4 Query+Resp, Cur Query].
      • ChatHistInCtxt, MaxTokens, ModelCtxt window to expand
      @@ -886,7 +886,7 @@ class Me { this.bCompletionFreshChatAlways = true; this.bCompletionInsertStandardRolePrefix = false; this.bTrimGarbage = true; - this.iRecentUserMsgCnt = 2; + this.iRecentUserMsgCnt = 5; this.sRecentUserMsgCnt = { "Full": -1, "Last0": 1, From 6f137f231186e0defb1a0275fc5c02256ada8219 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 02:28:42 +0530 Subject: [PATCH 40/89] SimpleChatTC: Update readme.md wrt latest updates. 2k maxtokens --- tools/server/public_simplechat/readme.md | 51 +++++++++++--------- tools/server/public_simplechat/simplechat.js | 4 +- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index 6f9f986b01844..c8cb786c3c6c8 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -239,10 +239,10 @@ It is attached to the document object. Some of these can also be updated using t be set if needed using the settings ui. iRecentUserMsgCnt - a simple minded SlidingWindow to limit context window load at Ai Model end. - This is disabled by default. However if enabled, then in addition to latest system message, only - the last/latest iRecentUserMsgCnt user messages after the latest system prompt and its responses - from the ai model will be sent to the ai-model, when querying for a new response. IE if enabled, - only user messages after the latest system message/prompt will be considered. + This is set to 5 by default. So in addition to latest system message, last/latest iRecentUserMsgCnt + user messages after the latest system prompt and its responses from the ai model will be sent + to the ai-model, when querying for a new response. Note that if enabled, only user messages after + the latest system message/prompt will be considered. This specified sliding window user message count also includes the latest user query. <0 : Send entire chat history to server @@ -282,9 +282,11 @@ full chat history. This way if there is any response with garbage/repeatation, i mess with things beyond the next question/request/query, in some ways. The trim garbage option also tries to help avoid issues with garbage in the context to an extent. -Set max_tokens to 1024, so that a relatively large previous reponse doesnt eat up the space -available wrt next query-response. However dont forget that the server when started should -also be started with a model context size of 1k or more, to be on safe side. +Set max_tokens to 2048, so that a relatively large previous reponse doesnt eat up the space +available wrt next query-response. While parallely allowing a good enough context size for +some amount of the chat history in the current session to influence future answers. However +dont forget that the server when started should also be started with a model context size of +2k or more, to be on safe side. The /completions endpoint of tools/server doesnt take max_tokens, instead it takes the internal n_predict, for now add the same here on the client side, maybe later add max_tokens @@ -321,9 +323,9 @@ work. ### Tool Calling -ALERT: Currently the way this is implemented, it is dangerous to use this, unless one verifies -all the tool calls requested and the responses generated manually to ensure everything is fine, -during interaction with ai modles with tools support. +ALERT: The simple minded way in which this is implemented, it can be dangerous in the worst case, +Always remember to verify all the tool calls requested and the responses generated manually to +ensure everything is fine, during interaction with ai modles with tools support. #### Builtin Tools @@ -332,10 +334,10 @@ The following tools/functions are currently provided by default * run_javascript_function_code - which can be used to run some javascript code in the browser context. -Currently the generated code / expression is run through a simple dynamic function mechanism. -May update things, in future, so that a WebWorker is used to avoid exposing browser global scope -to the generated code directly. Either way always remember to cross check the tool requests and -generated responses when using tool calling. +Currently the generated code / expression is run through a simple minded eval inside a web worker +mechanism. Use of WebWorker helps avoid exposing browser global scope to the generated code directly. +However any shared web worker scope isnt isolated. Either way always remember to cross check the tool +requests and generated responses when using tool calling. May add * web_fetch along with a corresponding simple local web proxy/caching server logic that can bypass @@ -343,19 +345,20 @@ May add Inturn maybe with a white list of allowed sites to access or so. -#### Extending wiht new tools +#### Extending with new tools Provide a descriptive meta data explaining the tool / function being provided for tool calling, as well as its arguments. -Provide a handler which should implement the specified tool / function call. It should place -the result to be sent back to the ai model in the result key of the tc_switch entry for the -corresponding tool. +Provide a handler which should implement the specified tool / function call or rather constructs +the code to be run to get the tool / function call job done, and inturn pass the same to the +provided web worker to get it executed. Remember to use console.log while generating any response +that should be sent back to the ai model, in your constructed code. -Update the tc_switch to include a object entry for the tool, which inturn icnludes +Update the tc_switch to include a object entry for the tool, which inturn includes * the meta data as well as * a reference to the handler and also -* the result key +* the result key (was used previously, may use in future, but for now left as is) #### Mapping tool calls and responses to normal assistant - user chat flow @@ -368,16 +371,16 @@ tagged response in the subsequent user block. This allows the GenAi/LLM to be aware of the tool calls it made as well as the responses it got, so that it can incorporate the results of the same in the subsequent chat / interactions. -NOTE: This flow tested to be ok enough with Gemma-3N-E4B-it-Q8_0 LLM ai model for now. +NOTE: This flow tested to be ok enough with Gemma-3N-E4B-it-Q8_0 LLM ai model for now. Logically +given the way current ai models work, most of them should understand things as needed, but need +to test this with other ai models later. TODO: Need to think later, whether to continue this simple flow, or atleast use tool role wrt -the tool call responses or even go further and have the logically seperate tool_call request +the tool call responses or even go further and have the logically seperate tool_calls request structures also. #### ToDo -Update to use web worker. - WebFetch and Local web proxy/caching server Try and trap promises based flows to ensure all generated results or errors if any are caught diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 4804c88ea0d28..9c791222188d5 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -907,8 +907,8 @@ class Me { this.apiRequestOptions = { "model": "gpt-3.5-turbo", "temperature": 0.7, - "max_tokens": 1024, - "n_predict": 1024, + "max_tokens": 2048, + "n_predict": 2048, "cache_prompt": false, //"frequency_penalty": 1.2, //"presence_penalty": 1.2, From dbf050c768abcbec5f0fff98d2cc3040d3323c6d Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 02:35:24 +0530 Subject: [PATCH 41/89] SimpleChatTC: update descs to indicate use of web workers ie wrt the tool calls provided. --- tools/server/public_simplechat/tooljs.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 5ee8e83004b35..6aea9a5ee4d21 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -14,13 +14,13 @@ let js_meta = { "type": "function", "function": { "name": "run_javascript_function_code", - "description": "Runs given code using function constructor mechanism in a browser's javascript environment and returns the console.log outputs of the execution after few seconds", + "description": "Runs given code using eval within a web worker context in a browser's javascript environment and returns the console.log outputs of the execution after few seconds", "parameters": { "type": "object", "properties": { "code": { "type": "string", - "description": "The code belonging to the dynamic function to run in the browser's javascript interpreter environment." + "description": "The code that will be run using eval within a web worker in the browser's javascript interpreter environment." } }, "required": ["code"] @@ -44,7 +44,7 @@ let calc_meta = { "type": "function", "function": { "name": "simple_calculator", - "description": "Calculates the provided arithmatic expression using console.log of a browser's javascript interpreter and returns the output of the execution once it is done in few seconds", + "description": "Calculates the provided arithmatic expression using console.log within a web worker of a browser's javascript interpreter environment and returns the output of the execution once it is done in few seconds", "parameters": { "type": "object", "properties": { From 39c1c01cc9fd42e6f416614f090f23bc1a0bdc88 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 16:48:40 +0530 Subject: [PATCH 42/89] SimpleChatTC:ChatMessage: AssistantResponse into chat message class Modify the constructor, newFrom and clear towards this goal. --- tools/server/public_simplechat/simplechat.js | 27 ++++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 9c791222188d5..4183ca87a25b9 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -38,23 +38,34 @@ class ApiEP { } -class AssistantResponse { +class ChatMessage { - constructor(content="", toolname="", toolargs="", trimmedContent="") { - /** @type {Object} */ - this.response = { content: content, toolname: toolname, toolargs: toolargs, trimmedContent: trimmedContent }; + /** + * Represent a Message in the Chat + * @param {string} role + * @param {string} content + * @param {Array} tool_calls + * @param {string} trimmedContent + */ + constructor(role = "", content="", tool_calls=[], trimmedContent="") { + /** @type {Object} */ + this.ns = { role: role, content: content, tool_calls: tool_calls } + this.trimmedContent = trimmedContent; } /** * Create a new instance from an existing instance - * @param {AssistantResponse} old + * @param {ChatMessage} old */ static newFrom(old) { - return new AssistantResponse(old.response.content, old.response.toolname, old.response.toolargs, old.response.trimmedContent) + return new ChatMessage(old.ns.role, old.ns.content, old.ns.tool_calls, old.trimmedContent) } clear() { - this.response = { content: "", toolname: "", toolargs: "", trimmedContent: "" }; + this.ns.role = ""; + this.ns.content = ""; + this.ns.tool_calls = []; + this.trimmedContent = ""; } /** @@ -71,7 +82,7 @@ class AssistantResponse { } has_toolcall() { - if (this.response.toolname.trim() == "") { + if (this.ns.tool_calls.trim() == "") { return false } return true From e9a7871f9388a4b48e8a3d32501ba7085cb140fc Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 17:59:06 +0530 Subject: [PATCH 43/89] SimpleChatTC:ChatMessageEx: UpdateStream logic Rename ChatMessage to ChatMessageEx. Add typedefs for NSToolCall and NSChatMessage, they represent the way the corresponding data is structured in network hs. Add logic to build the ChatMessageEx from data got over network in streaming mode. --- tools/server/public_simplechat/simplechat.js | 69 ++++++++++++++++---- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 4183ca87a25b9..608479c8623ea 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -38,7 +38,15 @@ class ApiEP { } -class ChatMessage { +/** + * @typedef {{id: string, type: string, function: {name: string, arguments: string}}} NSToolCalls + */ + +/** + * @typedef {{role: string, content: string, tool_calls: Array}} NSChatMessage + */ + +class ChatMessageEx { /** * Represent a Message in the Chat @@ -48,17 +56,17 @@ class ChatMessage { * @param {string} trimmedContent */ constructor(role = "", content="", tool_calls=[], trimmedContent="") { - /** @type {Object} */ + /** @type {NSChatMessage} */ this.ns = { role: role, content: content, tool_calls: tool_calls } this.trimmedContent = trimmedContent; } /** * Create a new instance from an existing instance - * @param {ChatMessage} old + * @param {ChatMessageEx} old */ static newFrom(old) { - return new ChatMessage(old.ns.role, old.ns.content, old.ns.tool_calls, old.trimmedContent) + return new ChatMessageEx(old.ns.role, old.ns.content, old.ns.tool_calls, old.trimmedContent) } clear() { @@ -69,16 +77,53 @@ class ChatMessage { } /** - * Helps collate the latest response from the server/ai-model, as it is becoming available. - * This is mainly useful for the stream mode. - * @param {{key: string, value: string}} resp + * Update based on the drip by drip data got from network in streaming mode + * @param {any} nwo + * @param {string} apiEP */ - append_response(resp) { - if (resp.value == null) { - return + update_stream(nwo, apiEP) { + if (apiEP == ApiEP.Type.Chat) { + if (nwo["choices"][0]["finish_reason"] === null) { + let content = nwo["choices"][0]["delta"]["content"]; + if (content !== undefined) { + if (content !== null) { + this.ns.content += content; + } + } else { + let toolCalls = nwo["choices"][0]["delta"]["tool_calls"]; + if ( toolCalls !== undefined) { + if (toolCalls[0]["function"]["name"] !== undefined) { + this.ns.tool_calls.push(toolCalls[0]) + /* + this.ns.tool_calls[0].function.name = toolCalls[0]["function"]["name"]; + this.ns.tool_calls[0].id = toolCalls[0]["id"]; + this.ns.tool_calls[0].type = toolCalls[0]["type"]; + this.ns.tool_calls[0].function.arguments = toolCalls[0]["function"]["arguments"] + */ + } else { + if (toolCalls[0]["function"]["arguments"] !== undefined) { + this.ns.tool_calls[0].function.arguments += toolCalls[0]["function"]["arguments"]; + } + } + } + } + } + } else { + try { + this.ns.content += nwo["choices"][0]["text"]; + } catch { + this.ns.content += nwo["content"]; + } } - console.debug(resp.key, resp.value) - this.response[resp.key] += resp.value; + } + + /** + * Update based on the data got from network in oneshot mode + * @param {any} nwo + * @param {string} apiEP + */ + update_oneshot(nwo, apiEP) { + } has_toolcall() { From 340ae0c6c1c657f145be5d31a2009a98b2374c27 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 18:28:49 +0530 Subject: [PATCH 44/89] SimpleChatTC:ChatMessageEx:cleanup, HasToolCalls, ContentEquiv Update HasToolCalls and ContentEquiv to work with new structure --- tools/server/public_simplechat/simplechat.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 608479c8623ea..993094fe424bb 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -77,7 +77,8 @@ class ChatMessageEx { } /** - * Update based on the drip by drip data got from network in streaming mode + * Update based on the drip by drip data got from network in streaming mode. + * Tries to support both Chat and Completion endpoints * @param {any} nwo * @param {string} apiEP */ @@ -88,12 +89,14 @@ class ChatMessageEx { if (content !== undefined) { if (content !== null) { this.ns.content += content; + } else { + this.ns.role = nwo["choices"][0]["delta"]["role"]; } } else { let toolCalls = nwo["choices"][0]["delta"]["tool_calls"]; - if ( toolCalls !== undefined) { + if (toolCalls !== undefined) { if (toolCalls[0]["function"]["name"] !== undefined) { - this.ns.tool_calls.push(toolCalls[0]) + this.ns.tool_calls.push(toolCalls[0]); /* this.ns.tool_calls[0].function.name = toolCalls[0]["function"]["name"]; this.ns.tool_calls[0].id = toolCalls[0]["id"]; @@ -127,17 +130,17 @@ class ChatMessageEx { } has_toolcall() { - if (this.ns.tool_calls.trim() == "") { + if (this.ns.tool_calls.length == 0) { return false } return true } content_equiv() { - if (this.response.content !== "") { - return this.response.content; + if (this.ns.content !== "") { + return this.ns.content; } else if (this.has_toolcall()) { - return `\n${this.response.toolname}\n${this.response.toolargs}\n`; + return `\n${this.ns.tool_calls[0].function.name}\n${this.ns.tool_calls[0].function.arguments}\n`; } else { return "" } @@ -184,7 +187,7 @@ class SimpleChat { */ this.xchat = []; this.iLastSys = -1; - this.latestResponse = new AssistantResponse(); + this.latestResponse = new ChatMessageEx(); } clear() { From 0629f79886e96e8203980f4d65e847e0b9652825 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 18:36:25 +0530 Subject: [PATCH 45/89] SimpleChatTC:ChatMessage: remove ResponseExtractStream Use the equivalent update_stream directly added to ChatMessageEx. update_stream is also more generic to some extent and also directly implemented by the ChatMessageEx class. --- tools/server/public_simplechat/simplechat.js | 42 ++------------------ 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 993094fe424bb..33c4e4b3f72df 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -83,6 +83,7 @@ class ChatMessageEx { * @param {string} apiEP */ update_stream(nwo, apiEP) { + console.debug(nwo, apiEP) if (apiEP == ApiEP.Type.Chat) { if (nwo["choices"][0]["finish_reason"] === null) { let content = nwo["choices"][0]["delta"]["content"]; @@ -408,43 +409,6 @@ class SimpleChat { return assistant; } - /** - * Extract the ai-model/assistant's response from the http response got in streaming mode. - * @param {any} respBody - * @param {string} apiEP - */ - response_extract_stream(respBody, apiEP) { - console.debug(respBody, apiEP) - let key = "content" - let assistant = ""; - if (apiEP == ApiEP.Type.Chat) { - if (respBody["choices"][0]["finish_reason"] === null) { - if (respBody["choices"][0]["delta"]["content"] !== undefined) { - assistant = respBody["choices"][0]["delta"]["content"]; - } else { - if (respBody["choices"][0]["delta"]["tool_calls"] !== undefined) { - if (respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["name"] !== undefined) { - key = "toolname"; - assistant = respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["name"]; - } else { - if (respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] !== undefined) { - key = "toolargs"; - assistant = respBody["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"]; - } - } - } - } - } - } else { - try { - assistant = respBody["choices"][0]["text"]; - } catch { - assistant = respBody["content"]; - } - } - return { key: key, value: assistant }; - } - /** * Allow setting of system prompt, but only at begining. * @param {string} sysPrompt @@ -541,7 +505,7 @@ class SimpleChat { } let curJson = JSON.parse(curLine); console.debug("DBUG:SC:PART:Json:", curJson); - this.latestResponse.append_response(this.response_extract_stream(curJson, apiEP)); + this.latestResponse.update_stream(curJson, apiEP); } elP.innerText = this.latestResponse.content_equiv() elP.scrollIntoView(false); @@ -550,7 +514,7 @@ class SimpleChat { } } console.debug("DBUG:SC:PART:Full:", this.latestResponse.content_equiv()); - return AssistantResponse.newFrom(this.latestResponse); + return ChatMessageEx.newFrom(this.latestResponse); } /** From bb25aa0ea2cb41a96874bf6dca016ba1a4b4c920 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 18:52:18 +0530 Subject: [PATCH 46/89] SimpleChatTC:ChatMessageEx: add update_oneshot response_extract logic moved directly into ChatMessageEx as update oneshot, with suitable adjustments. Inturn use the same directly. --- tools/server/public_simplechat/simplechat.js | 39 ++++++++------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 33c4e4b3f72df..9b29081a773cc 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -127,7 +127,15 @@ class ChatMessageEx { * @param {string} apiEP */ update_oneshot(nwo, apiEP) { - + if (apiEP == ApiEP.Type.Chat) { + this.ns.content = nwo["choices"][0]["message"]["content"]; + } else { + try { + this.ns.content = nwo["choices"][0]["text"]; + } catch { + this.ns.content = nwo["content"]; + } + } } has_toolcall() { @@ -389,25 +397,6 @@ class SimpleChat { } } - /** - * Extract the ai-model/assistant's response from the http response got. - * Optionally trim the message wrt any garbage at the end. - * @param {any} respBody - * @param {string} apiEP - */ - response_extract(respBody, apiEP) { - let assistant = new AssistantResponse(); - if (apiEP == ApiEP.Type.Chat) { - assistant.response.content = respBody["choices"][0]["message"]["content"]; - } else { - try { - assistant.response.content = respBody["choices"][0]["text"]; - } catch { - assistant.response.content = respBody["content"]; - } - } - return assistant; - } /** * Allow setting of system prompt, but only at begining. @@ -525,7 +514,9 @@ class SimpleChat { async handle_response_oneshot(resp, apiEP) { let respBody = await resp.json(); console.debug(`DBUG:SimpleChat:SC:${this.chatId}:HandleUserSubmit:RespBody:${JSON.stringify(respBody)}`); - return this.response_extract(respBody, apiEP); + let cm = new ChatMessageEx() + cm.update_oneshot(respBody, apiEP) + return cm } /** @@ -553,9 +544,9 @@ class SimpleChat { theResp = await this.handle_response_oneshot(resp, apiEP); } if (gMe.bTrimGarbage) { - let origMsg = theResp.response.content; - theResp.response.content = du.trim_garbage_at_end(origMsg); - theResp.response.trimmedContent = origMsg.substring(theResp.response.content.length); + let origMsg = theResp.ns.content; + theResp.ns.content = du.trim_garbage_at_end(origMsg); + theResp.trimmedContent = origMsg.substring(theResp.ns.content.length); } this.add(Roles.Assistant, theResp.content_equiv()); return theResp; From ae00cb2ef972a64e73856f73b6c0bddd99d6149f Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 19:03:48 +0530 Subject: [PATCH 47/89] SimpleChatTC:ChatMessageEx: ods load, system prompt related these have been updated to work with ChatMessageEx to an extent --- tools/server/public_simplechat/simplechat.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 9b29081a773cc..b7262369bc6e5 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -179,7 +179,7 @@ let gUsageMsg = ` `; -/** @typedef {{role: string, content: string}[]} ChatMessages */ +/** @typedef {ChatMessageEx[]} ChatMessages */ /** @typedef {{iLastSys: number, xchat: ChatMessages}} SimpleChatODS */ @@ -222,7 +222,11 @@ class SimpleChat { /** @type {SimpleChatODS} */ let ods = JSON.parse(sods); this.iLastSys = ods.iLastSys; - this.xchat = ods.xchat; + this.xchat = []; + for (const cur of ods.xchat) { + // TODO: May have to account for missing fields + this.xchat.push(new ChatMessageEx(cur.ns.role, cur.ns.content, cur.ns.tool_calls, cur.trimmedContent)) + } } /** @@ -410,10 +414,10 @@ class SimpleChat { } } else { if (sysPrompt.length > 0) { - if (this.xchat[0].role !== Roles.System) { + if (this.xchat[0].ns.role !== Roles.System) { console.error(`ERRR:SimpleChat:SC:${msgTag}:You need to specify system prompt before any user query, ignoring...`); } else { - if (this.xchat[0].content !== sysPrompt) { + if (this.xchat[0].ns.content !== sysPrompt) { console.error(`ERRR:SimpleChat:SC:${msgTag}:You cant change system prompt, mid way through, ignoring...`); } } @@ -437,7 +441,7 @@ class SimpleChat { return this.add(Roles.System, sysPrompt); } - let lastSys = this.xchat[this.iLastSys].content; + let lastSys = this.xchat[this.iLastSys].ns.content; if (lastSys !== sysPrompt) { return this.add(Roles.System, sysPrompt); } @@ -451,7 +455,7 @@ class SimpleChat { if (this.iLastSys == -1) { return ""; } - let sysPrompt = this.xchat[this.iLastSys].content; + let sysPrompt = this.xchat[this.iLastSys].ns.content; return sysPrompt; } From e1e1d4297bf44ccd914bfb61d9743294efa72f6e Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 19:33:24 +0530 Subject: [PATCH 48/89] SimpleChatTC:ChatMessageEx: RecentChat, GetSystemLatest GetSystemLatest and its users updated wrt ChatMessageEx. RecentChat updated wrt ChatMessageEx. Also now irrespective of whether full history is being retrieved or only a subset, both cases refer to the ChatMessageEx instances in SimpleChat.xchat without creating new instances of anything. --- tools/server/public_simplechat/simplechat.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index b7262369bc6e5..d2d8be73a0517 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -248,8 +248,8 @@ class SimpleChat { /** @type{ChatMessages} */ let rchat = []; let sysMsg = this.get_system_latest(); - if (sysMsg.length != 0) { - rchat.push({role: Roles.System, content: sysMsg}); + if (sysMsg.ns.content.length != 0) { + rchat.push(sysMsg) } let iUserCnt = 0; let iStart = this.xchat.length; @@ -258,17 +258,17 @@ class SimpleChat { break; } let msg = this.xchat[i]; - if (msg.role == Roles.User) { + if (msg.ns.role == Roles.User) { iStart = i; iUserCnt += 1; } } for(let i = iStart; i < this.xchat.length; i++) { let msg = this.xchat[i]; - if (msg.role == Roles.System) { + if (msg.ns.role == Roles.System) { continue; } - rchat.push({role: msg.role, content: msg.content}); + rchat.push(msg) } return rchat; } @@ -453,10 +453,9 @@ class SimpleChat { */ get_system_latest() { if (this.iLastSys == -1) { - return ""; + return new ChatMessageEx(Roles.System); } - let sysPrompt = this.xchat[this.iLastSys].ns.content; - return sysPrompt; + return this.xchat[this.iLastSys]; } @@ -882,7 +881,7 @@ class MultiChatUI { console.error(`ERRR:SimpleChat:MCUI:HandleSessionSwitch:${chatId} missing...`); return; } - this.elInSystem.value = chat.get_system_latest(); + this.elInSystem.value = chat.get_system_latest().ns.content; this.elInUser.value = ""; chat.show(this.elDivChat); this.elInUser.focus(); @@ -959,7 +958,7 @@ class Me { chat.load(); queueMicrotask(()=>{ chat.show(div); - this.multiChat.elInSystem.value = chat.get_system_latest(); + this.multiChat.elInSystem.value = chat.get_system_latest().ns.content; }); }); div.appendChild(btn); From 3b73b00397405d2887da1cc5d88d6b2938c7ddd3 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 21:02:22 +0530 Subject: [PATCH 49/89] SimpleChatTC:ChatMessageEx: Upd Add, rm sysPromptAtBeginOnly hlpr Simplify Add semantic by expecting any validation of stuff before adding to be done by the callers of Add and not by add itself. Also update it to expect ChatMessageEx object Update all users of add to follow the new syntax and semantic. Remove the old and ununsed AddSysPromptOnlyAtBegin helper --- tools/server/public_simplechat/simplechat.js | 61 ++++++-------------- 1 file changed, 19 insertions(+), 42 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index d2d8be73a0517..3c50d4f62694a 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -274,17 +274,14 @@ class SimpleChat { } /** - * Add an entry into xchat + * Add an entry into xchat. + * NOTE: A new copy is created and added into xchat. * Also update iLastSys system prompt index tracker - * @param {string} role - * @param {string|undefined|null} content + * @param {ChatMessageEx} chatMsg */ - add(role, content) { - if ((content == undefined) || (content == null) || (content == "")) { - return false; - } - this.xchat.push( {role: role, content: content} ); - if (role == Roles.System) { + add(chatMsg) { + this.xchat.push(ChatMessageEx.newFrom(chatMsg)); + if (chatMsg.ns.role == Roles.System) { this.iLastSys = this.xchat.length - 1; } this.save(); @@ -402,30 +399,6 @@ class SimpleChat { } - /** - * Allow setting of system prompt, but only at begining. - * @param {string} sysPrompt - * @param {string} msgTag - */ - add_system_begin(sysPrompt, msgTag) { - if (this.xchat.length == 0) { - if (sysPrompt.length > 0) { - return this.add(Roles.System, sysPrompt); - } - } else { - if (sysPrompt.length > 0) { - if (this.xchat[0].ns.role !== Roles.System) { - console.error(`ERRR:SimpleChat:SC:${msgTag}:You need to specify system prompt before any user query, ignoring...`); - } else { - if (this.xchat[0].ns.content !== sysPrompt) { - console.error(`ERRR:SimpleChat:SC:${msgTag}:You cant change system prompt, mid way through, ignoring...`); - } - } - } - } - return false; - } - /** * Allow setting of system prompt, at any time. * Updates the system prompt, if one was never set or if the newly passed is different from the last set system prompt. @@ -438,12 +411,12 @@ class SimpleChat { } if (this.iLastSys < 0) { - return this.add(Roles.System, sysPrompt); + return this.add(new ChatMessageEx(Roles.System, sysPrompt)); } let lastSys = this.xchat[this.iLastSys].ns.content; if (lastSys !== sysPrompt) { - return this.add(Roles.System, sysPrompt); + return this.add(new ChatMessageEx(Roles.System, sysPrompt)); } return false; } @@ -473,6 +446,7 @@ class SimpleChat { let tdUtf8 = new TextDecoder("utf-8"); let rr = resp.body.getReader(); this.latestResponse.clear() + this.latestResponse.ns.role = Roles.Assistant let xLines = new du.NewLines(); while(true) { let { value: cur, done: done } = await rr.read(); @@ -517,7 +491,7 @@ class SimpleChat { async handle_response_oneshot(resp, apiEP) { let respBody = await resp.json(); console.debug(`DBUG:SimpleChat:SC:${this.chatId}:HandleUserSubmit:RespBody:${JSON.stringify(respBody)}`); - let cm = new ChatMessageEx() + let cm = new ChatMessageEx(Roles.Assistant) cm.update_oneshot(respBody, apiEP) return cm } @@ -532,15 +506,16 @@ class SimpleChat { * @param {HTMLDivElement} elDiv */ async handle_response(resp, apiEP, elDiv) { - let theResp = null + let theResp = null; if (gMe.bStream) { try { theResp = await this.handle_response_multipart(resp, apiEP, elDiv); - this.latestResponse.clear() + this.latestResponse.clear(); } catch (error) { theResp = this.latestResponse; - this.add(Roles.Assistant, theResp.content_equiv()); - this.latestResponse.clear() + theResp.ns.role = Roles.Assistant; + this.add(theResp); + this.latestResponse.clear(); throw error; } } else { @@ -551,7 +526,8 @@ class SimpleChat { theResp.ns.content = du.trim_garbage_at_end(origMsg); theResp.trimmedContent = origMsg.substring(theResp.ns.content.length); } - this.add(Roles.Assistant, theResp.content_equiv()); + theResp.ns.role = Roles.Assistant; + this.add(theResp); return theResp; } @@ -761,10 +737,11 @@ class MultiChatUI { chat.add_system_anytime(this.elInSystem.value, chatId); let content = this.elInUser.value; - if (!chat.add(Roles.User, content)) { + if (content.trim() == "") { console.debug(`WARN:SimpleChat:MCUI:${chatId}:HandleUserSubmit:Ignoring empty user input...`); return; } + chat.add(new ChatMessageEx(Roles.User, content)) chat.show(this.elDivChat); let theUrl = ApiEP.Url(gMe.baseURL, apiEP); From aa80bf0117940140005d8950e8645fc9d7257007 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 21:36:11 +0530 Subject: [PATCH 50/89] SimpleChatTC:ChatMessageEx: Recent chat users upd Users of recent_chat updated to work with ChatMessageEx As part of same recent_chat_ns also added, for the case where the array of chat messages can be passed as is ie in the chat mode, provided it has only the network handshake representation of the messages. --- tools/server/public_simplechat/simplechat.js | 29 +++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 3c50d4f62694a..8bd8a285db380 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -273,6 +273,21 @@ class SimpleChat { return rchat; } + + /** + * Return recent chat messages in the format, + * which can be directly sent to the ai server. + * @param {number} iRecentUserMsgCnt - look at recent_chat for semantic + */ + recent_chat_ns(iRecentUserMsgCnt) { + let xchat = this.recent_chat(iRecentUserMsgCnt); + let chat = [] + for (const msg of xchat) { + chat.push(msg.ns) + } + return chat + } + /** * Add an entry into xchat. * NOTE: A new copy is created and added into xchat. @@ -299,8 +314,8 @@ class SimpleChat { } let last = undefined; for(const x of this.recent_chat(gMe.iRecentUserMsgCnt)) { - let entry = ui.el_create_append_p(`${x.role}: ${x.content}`, div); - entry.className = `role-${x.role}`; + let entry = ui.el_create_append_p(`${x.ns.role}: ${x.content_equiv()}`, div); + entry.className = `role-${x.ns.role}`; last = entry; } if (last !== undefined) { @@ -338,7 +353,7 @@ class SimpleChat { * The needed fields/options are picked from a global object. * Add optional stream flag, if required. * Convert the json into string. - * @param {Object} obj + * @param {Object} obj */ request_jsonstr_extend(obj) { for(let k in gMe.apiRequestOptions) { @@ -358,7 +373,7 @@ class SimpleChat { */ request_messages_jsonstr() { let req = { - messages: this.recent_chat(gMe.iRecentUserMsgCnt), + messages: this.recent_chat_ns(gMe.iRecentUserMsgCnt), } return this.request_jsonstr_extend(req); } @@ -370,15 +385,15 @@ class SimpleChat { request_prompt_jsonstr(bInsertStandardRolePrefix) { let prompt = ""; let iCnt = 0; - for(const chat of this.recent_chat(gMe.iRecentUserMsgCnt)) { + for(const msg of this.recent_chat(gMe.iRecentUserMsgCnt)) { iCnt += 1; if (iCnt > 1) { prompt += "\n"; } if (bInsertStandardRolePrefix) { - prompt += `${chat.role}: `; + prompt += `${msg.ns.role}: `; } - prompt += `${chat.content}`; + prompt += `${msg.ns.content}`; } let req = { prompt: prompt, From 755505e8ce6bcf4587310f961b3921a2f8cd931f Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 22:26:48 +0530 Subject: [PATCH 51/89] SimpleChatTC:ChatMessageEx: Cleanup remaining stuff wrt ChatMessageEx related required flow as well as avoid warnings --- tools/server/public_simplechat/simplechat.js | 27 ++++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 8bd8a285db380..0d338ebaa1421 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -437,7 +437,7 @@ class SimpleChat { } /** - * Retrieve the latest system prompt. + * Retrieve the latest system prompt related chat message entry. */ get_system_latest() { if (this.iLastSys == -1) { @@ -609,19 +609,20 @@ class MultiChatUI { if (el == null) { throw Error(`ERRR:SimpleChat:MCUI:${msgTag} element missing in html...`); } else { + // @ts-ignore console.debug(`INFO:SimpleChat:MCUI:${msgTag} Id[${el.id}] Name[${el["name"]}]`); } } /** * Reset/Setup Tool Call UI parts as needed - * @param {AssistantResponse} ar + * @param {ChatMessageEx} ar */ ui_reset_toolcall_as_needed(ar) { if (ar.has_toolcall()) { this.elDivTool.hidden = false - this.elInToolName.value = ar.response.toolname - this.elInToolArgs.value = ar.response.toolargs + this.elInToolName.value = ar.ns.tool_calls[0].function.name + this.elInToolArgs.value = ar.ns.tool_calls[0].function.arguments this.elBtnTool.disabled = false } else { this.elDivTool.hidden = true @@ -659,7 +660,7 @@ class MultiChatUI { this.handle_session_switch(this.curChatId); } - this.ui_reset_toolcall_as_needed(new AssistantResponse()); + this.ui_reset_toolcall_as_needed(new ChatMessageEx()); this.elBtnSettings.addEventListener("click", (ev)=>{ this.elDivChat.replaceChildren(); @@ -747,7 +748,7 @@ class MultiChatUI { chat.clear(); } - this.ui_reset_toolcall_as_needed(new AssistantResponse()); + this.ui_reset_toolcall_as_needed(new ChatMessageEx()); chat.add_system_anytime(this.elInSystem.value, chatId); @@ -775,8 +776,8 @@ class MultiChatUI { let theResp = await chat.handle_response(resp, apiEP, this.elDivChat); if (chatId == this.curChatId) { chat.show(this.elDivChat); - if (theResp.response.trimmedContent.length > 0) { - let p = ui.el_create_append_p(`TRIMMED:${theResp.response.trimmedContent}`, this.elDivChat); + if (theResp.trimmedContent.length > 0) { + let p = ui.el_create_append_p(`TRIMMED:${theResp.trimmedContent}`, this.elDivChat); p.className="role-trim"; } } else { @@ -847,6 +848,11 @@ class MultiChatUI { } } + /** + * Create session button and append to specified Div element. + * @param {HTMLDivElement} elDiv + * @param {string} cid + */ create_session_btn(elDiv, cid) { let btn = ui.el_create_button(cid, (ev)=>{ let target = /** @type{HTMLButtonElement} */(ev.target); @@ -896,6 +902,7 @@ class Me { this.bCompletionInsertStandardRolePrefix = false; this.bTrimGarbage = true; this.iRecentUserMsgCnt = 5; + /** @type {Object} */ this.sRecentUserMsgCnt = { "Full": -1, "Last0": 1, @@ -1063,6 +1070,7 @@ class Me { this.show_settings_apirequestoptions(elDiv); let sel = ui.el_creatediv_select("SetApiEP", "ApiEndPoint", ApiEP.Type, this.apiEP, (val)=>{ + // @ts-ignore this.apiEP = ApiEP.Type[val]; }); elDiv.appendChild(sel.div); @@ -1094,8 +1102,11 @@ function startme() { console.log("INFO:SimpleChat:StartMe:Starting..."); gMe = new Me(); gMe.debug_disable(); + // @ts-ignore document["gMe"] = gMe; + // @ts-ignore document["du"] = du; + // @ts-ignore document["tools"] = tools; tools.init() for (let cid of gMe.defaultChatIds) { From 7fb5526fb8901129ca4a2c5084f9c91523b43812 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Tue, 14 Oct 2025 23:27:03 +0530 Subject: [PATCH 52/89] SimpleChatTC:Load allows old and new ChatMessage(Ex) formats --- tools/server/public_simplechat/simplechat.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 0d338ebaa1421..ce09e764dcc43 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -224,8 +224,13 @@ class SimpleChat { this.iLastSys = ods.iLastSys; this.xchat = []; for (const cur of ods.xchat) { - // TODO: May have to account for missing fields - this.xchat.push(new ChatMessageEx(cur.ns.role, cur.ns.content, cur.ns.tool_calls, cur.trimmedContent)) + if (cur.ns == undefined) { + /** @typedef {{role: string, content: string}} OldChatMessage */ + let tcur = /** @type {OldChatMessage} */(/** @type {unknown} */(cur)); + this.xchat.push(new ChatMessageEx(tcur.role, tcur.content)) + } else { + this.xchat.push(new ChatMessageEx(cur.ns.role, cur.ns.content, cur.ns.tool_calls, cur.trimmedContent)) + } } } From 69cbc81f97e33df1f879f07498630c39331d4e42 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 00:04:29 +0530 Subject: [PATCH 53/89] SimpleChatTC:ChatMessageEx: send tool_calls, only if needed --- tools/server/public_simplechat/simplechat.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index ce09e764dcc43..2b82a4648da85 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -11,6 +11,7 @@ class Roles { static System = "system"; static User = "user"; static Assistant = "assistant"; + static Tool = "tool"; } class ApiEP { @@ -286,9 +287,14 @@ class SimpleChat { */ recent_chat_ns(iRecentUserMsgCnt) { let xchat = this.recent_chat(iRecentUserMsgCnt); - let chat = [] + let chat = []; for (const msg of xchat) { - chat.push(msg.ns) + let tmsg = ChatMessageEx.newFrom(msg); + if (!tmsg.has_toolcall()) { + // @ts-ignore + delete(tmsg.ns.tool_calls) + } + chat.push(tmsg.ns); } return chat } From f379e651fafbf66e21daae8b77bef8bced4e2898 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 01:20:27 +0530 Subject: [PATCH 54/89] SimpleChatTC:Propogate toolcall id through tool call chain Use HTMLElement's dataset to maintain tool call id along with the element which maintains the toolname. Pass it along to the tools manager and inturn the actual tool calls and through them to the web worker handling the tool call related code and inturn returning it back as part of the obj which is used to return the tool call result. Embed the tool call id, function name and function result into the content field of chat message in terms of a xml structure Also make use of tool role to send back the tool call result. Do note that currently the id, name and content are all embedded into the content field of the tool role message sent to the ai engine on the server. NOTE: Use the user query entry area for showing tool call result in the above mentioned xml form, as well as for user to enter their own queries. Based on presence of the xml format data at beginning the logic will treat it has a tool result and if not then as a normal user query. The css has been updated to help show tool results/msgs in a lightyellow background --- tools/server/public_simplechat/simplechat.css | 3 ++ tools/server/public_simplechat/simplechat.js | 46 +++++++++++++++---- tools/server/public_simplechat/tooljs.mjs | 10 ++-- tools/server/public_simplechat/tools.mjs | 9 ++-- .../server/public_simplechat/toolsworker.mjs | 2 +- 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.css b/tools/server/public_simplechat/simplechat.css index 13bfb80b48be8..d4755074b77c5 100644 --- a/tools/server/public_simplechat/simplechat.css +++ b/tools/server/public_simplechat/simplechat.css @@ -21,6 +21,9 @@ .role-user { background-color: lightgray; } +.role-tool { + background-color: lightyellow; +} .role-trim { background-color: lightpink; } diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 2b82a4648da85..281b2c15b958c 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -77,6 +77,17 @@ class ChatMessageEx { this.trimmedContent = ""; } + /** + * Create a all in one tool call result string + * @param {string} toolCallId + * @param {string} toolName + * @param {string} toolResult + */ + static createToolCallResultAllInOne(toolCallId, toolName, toolResult) { + return ` ${toolCallId} ${toolName} ${toolResult} `; + } + + /** * Update based on the drip by drip data got from network in streaming mode. * Tries to support both Chat and Completion endpoints @@ -561,15 +572,16 @@ class SimpleChat { * Call the requested tool/function. * Returns undefined, if the call was placed successfully * Else some appropriate error message will be returned. + * @param {string} toolcallid * @param {string} toolname * @param {string} toolargs */ - async handle_toolcall(toolname, toolargs) { + async handle_toolcall(toolcallid, toolname, toolargs) { if (toolname === "") { return "Tool/Function call name not specified" } try { - return await tools.tool_call(toolname, toolargs) + return await tools.tool_call(toolcallid, toolname, toolargs) } catch (/** @type {any} */error) { return `Tool/Function call raised an exception:${error.name}:${error.message}` } @@ -633,11 +645,13 @@ class MultiChatUI { if (ar.has_toolcall()) { this.elDivTool.hidden = false this.elInToolName.value = ar.ns.tool_calls[0].function.name + this.elInToolName.dataset.tool_call_id = ar.ns.tool_calls[0].id this.elInToolArgs.value = ar.ns.tool_calls[0].function.arguments this.elBtnTool.disabled = false } else { this.elDivTool.hidden = true this.elInToolName.value = "" + this.elInToolName.dataset.tool_call_id = "" this.elInToolArgs.value = "" this.elBtnTool.disabled = true } @@ -697,9 +711,9 @@ class MultiChatUI { this.handle_tool_run(this.curChatId); }) - tools.setup((name, data)=>{ + tools.setup((id, name, data)=>{ clearTimeout(this.idTimeOut) - this.elInUser.value = `${data}` + this.elInUser.value = ChatMessageEx.createToolCallResultAllInOne(id, name, data); this.ui_reset_userinput(false) }) @@ -744,6 +758,14 @@ class MultiChatUI { /** * Handle user query submit request, wrt specified chat session. + * NOTE: Currently the user query entry area is used for + * * showing and allowing edits by user wrt tool call results + * in a predfined simple xml format, + * ie before they submit tool result to ai engine on server + * * as well as for user to enter their own queries. + * Based on presence of the predefined xml format data at beginning + * the logic will treat it has a tool result and if not then as a + * normal user query. * @param {string} chatId * @param {string} apiEP */ @@ -768,7 +790,11 @@ class MultiChatUI { console.debug(`WARN:SimpleChat:MCUI:${chatId}:HandleUserSubmit:Ignoring empty user input...`); return; } - chat.add(new ChatMessageEx(Roles.User, content)) + if (content.startsWith("")) { + chat.add(new ChatMessageEx(Roles.Tool, content)) + } else { + chat.add(new ChatMessageEx(Roles.User, content)) + } chat.show(this.elDivChat); let theUrl = ApiEP.Url(gMe.baseURL, apiEP); @@ -808,13 +834,17 @@ class MultiChatUI { this.elInUser.value = "toolcall in progress..."; this.elInUser.disabled = true; let toolname = this.elInToolName.value.trim() - let toolResult = await chat.handle_toolcall(toolname, this.elInToolArgs.value) + let toolCallId = this.elInToolName.dataset.tool_call_id; + if (toolCallId === undefined) { + toolCallId = "??? ToolCallId Missing ???" + } + let toolResult = await chat.handle_toolcall(toolCallId, toolname, this.elInToolArgs.value) if (toolResult !== undefined) { - this.elInUser.value = `${toolResult}` + this.elInUser.value = ChatMessageEx.createToolCallResultAllInOne(toolCallId, toolname, toolResult); this.ui_reset_userinput(false) } else { this.idTimeOut = setTimeout(() => { - this.elInUser.value = `Tool/Function call ${toolname} taking too much time, aborting...` + this.elInUser.value = ChatMessageEx.createToolCallResultAllInOne(toolCallId, toolname, `Tool/Function call ${toolname} taking too much time, aborting...`); this.ui_reset_userinput(false) }, 10000) } diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 6aea9a5ee4d21..a44333ca1b3e7 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -32,11 +32,12 @@ let js_meta = { /** * Implementation of the javascript interpretor logic. Minimal skeleton for now. * ALERT: Has access to the javascript web worker environment and can mess with it and beyond + * @param {string} toolcallid * @param {string} toolname * @param {any} obj */ -function js_run(toolname, obj) { - gToolsWorker.postMessage({ name: toolname, code: obj["code"]}) +function js_run(toolcallid, toolname, obj) { + gToolsWorker.postMessage({ id: toolcallid, name: toolname, code: obj["code"]}) } @@ -62,11 +63,12 @@ let calc_meta = { /** * Implementation of the simple calculator logic. Minimal skeleton for now. * ALERT: Has access to the javascript web worker environment and can mess with it and beyond + * @param {string} toolcallid * @param {string} toolname * @param {any} obj */ -function calc_run(toolname, obj) { - gToolsWorker.postMessage({ name: toolname, code: `console.log(${obj["arithexpr"]})`}) +function calc_run(toolcallid, toolname, obj) { + gToolsWorker.postMessage({ id: toolcallid, name: toolname, code: `console.log(${obj["arithexpr"]})`}) } diff --git a/tools/server/public_simplechat/tools.mjs b/tools/server/public_simplechat/tools.mjs index 75fe56e4f4e5f..8c89e965258b4 100644 --- a/tools/server/public_simplechat/tools.mjs +++ b/tools/server/public_simplechat/tools.mjs @@ -32,11 +32,11 @@ export function meta() { /** * Setup the callback that will be called when ever message * is recieved from the Tools Web Worker. - * @param {(name: string, data: string) => void} cb + * @param {(id: string, name: string, data: string) => void} cb */ export function setup(cb) { gToolsWorker.onmessage = function (ev) { - cb(ev.data.name, ev.data.data) + cb(ev.data.id, ev.data.name, ev.data.data) } } @@ -45,14 +45,15 @@ export function setup(cb) { * Try call the specified tool/function call. * Returns undefined, if the call was placed successfully * Else some appropriate error message will be returned. + * @param {string} toolcallid * @param {string} toolname * @param {string} toolargs */ -export async function tool_call(toolname, toolargs) { +export async function tool_call(toolcallid, toolname, toolargs) { for (const fn in tc_switch) { if (fn == toolname) { try { - tc_switch[fn]["handler"](fn, JSON.parse(toolargs)) + tc_switch[fn]["handler"](toolcallid, fn, JSON.parse(toolargs)) return undefined } catch (/** @type {any} */error) { return `Tool/Function call raised an exception:${error.name}:${error.message}` diff --git a/tools/server/public_simplechat/toolsworker.mjs b/tools/server/public_simplechat/toolsworker.mjs index e370fd0a9df34..590c45234be7b 100644 --- a/tools/server/public_simplechat/toolsworker.mjs +++ b/tools/server/public_simplechat/toolsworker.mjs @@ -21,5 +21,5 @@ self.onmessage = function (ev) { console.log(`\n\nTool/Function call "${ev.data.name}" raised an exception:${error.name}:${error.message}\n\n`) } tconsole.console_revert() - self.postMessage({ name: ev.data.name, data: tconsole.gConsoleStr}) + self.postMessage({ id: ev.data.id, name: ev.data.name, data: tconsole.gConsoleStr}) } From 4efa2327efb79f1f5a6b93839a807838b4bd00a3 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 03:28:48 +0530 Subject: [PATCH 55/89] SimpleChatTC:ChatMessageEx: Build tool role result fully Expand the xml format id, name and content in content field of tool result into apropriate fields in the tool result message sent to the genai/llm engine on the server. --- tools/server/public_simplechat/simplechat.js | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 281b2c15b958c..847115cb8e7d3 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -87,6 +87,33 @@ class ChatMessageEx { return ` ${toolCallId} ${toolName} ${toolResult} `; } + /** + * Extract the elements of the all in one tool call result string + * @param {string} allInOne + */ + static extractToolCallResultAllInOne(allInOne) { + const regex = /\s*(.*?)<\/id>\s*(.*?)<\/name>\s*([\s\S]*?)<\/content>\s*<\/tool_response>/si; + const caught = allInOne.match(regex) + let data = { tool_call_id: "Error", name: "Error", content: "Error" } + if (caught) { + data = { + tool_call_id: caught[1].trim(), + name: caught[2].trim(), + content: caught[3].trim() + } + } + return data + } + + /** + * Set extra members into the ns object + * @param {string | number} key + * @param {any} value + */ + ns_set_extra(key, value) { + // @ts-ignore + this.ns[key] = value + } /** * Update based on the drip by drip data got from network in streaming mode. @@ -305,6 +332,12 @@ class SimpleChat { // @ts-ignore delete(tmsg.ns.tool_calls) } + if (tmsg.ns.role == Roles.Tool) { + let res = ChatMessageEx.extractToolCallResultAllInOne(tmsg.ns.content) + tmsg.ns.content = res.content + tmsg.ns_set_extra("tool_call_id", res.tool_call_id) + tmsg.ns_set_extra("name", res.name) + } chat.push(tmsg.ns); } return chat From a644cb3e1c514575aa102fa75637be9d0cdc12f3 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 03:35:55 +0530 Subject: [PATCH 56/89] SimpleChatTC:ChatMessageEx:While at it also ns_delete these common helpers avoid needing ignore tagging to ts-check, in places where valid constructs have been used which go beyond strict structured js handling that is tried to be achieved using it, but are still valid and legal. --- tools/server/public_simplechat/simplechat.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 847115cb8e7d3..bc4403516228d 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -115,6 +115,15 @@ class ChatMessageEx { this.ns[key] = value } + /** + * Remove specified key and its value from ns object + * @param {string | number} key + */ + ns_delete(key) { + // @ts-ignore + delete(this.ns[key]) + } + /** * Update based on the drip by drip data got from network in streaming mode. * Tries to support both Chat and Completion endpoints @@ -329,8 +338,7 @@ class SimpleChat { for (const msg of xchat) { let tmsg = ChatMessageEx.newFrom(msg); if (!tmsg.has_toolcall()) { - // @ts-ignore - delete(tmsg.ns.tool_calls) + tmsg.ns_delete("tool_calls") } if (tmsg.ns.role == Roles.Tool) { let res = ChatMessageEx.extractToolCallResultAllInOne(tmsg.ns.content) From 61c231487f590eccdc6ecbed86819d13224c62dc Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 04:13:34 +0530 Subject: [PATCH 57/89] SimpleChatTC:Readme: Updated wrt new relativelyProper toolCallsHS Also update the sliding window context size to last 9 chat messages so that there is a sufficiently large context for multi turn tool calls based adjusting by ai and user, without needing to go full hog, which has the issue of overflowing the currently set context window wrt the loaded ai model. --- tools/server/public_simplechat/readme.md | 20 ++++++++++++++------ tools/server/public_simplechat/simplechat.js | 5 +++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index c8cb786c3c6c8..d50588cce5e78 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -136,8 +136,9 @@ Once inside called, then the ai response might include request for tool call. * the SimpleChat client will show details of the tool call (ie tool name and args passed) requested and allow the user to trigger it as is or after modifying things as needed. + NOTE: Tool sees the original tool call only, for now * inturn returned / generated result is placed into user query entry text area with approriate tags - ie generated result + ie generated result with meta data * if user is ok with the tool response, they can click submit to send the same to the GenAi/LLM. User can even modify the response generated by the tool, if required, before submitting. @@ -193,7 +194,7 @@ It is attached to the document object. Some of these can also be updated using t sent back to the ai model, under user control. as tool calling will involve a bit of back and forth between ai assistant and end user, it is - recommended to set iRecentUserMsgCnt to 5 or more, so that enough context is retained during + recommended to set iRecentUserMsgCnt to 10 or more, so that enough context is retained during chatting with ai models with tool support. apiEP - select between /completions and /chat/completions endpoint provided by the server/ai-model. @@ -239,7 +240,7 @@ It is attached to the document object. Some of these can also be updated using t be set if needed using the settings ui. iRecentUserMsgCnt - a simple minded SlidingWindow to limit context window load at Ai Model end. - This is set to 5 by default. So in addition to latest system message, last/latest iRecentUserMsgCnt + This is set to 10 by default. So in addition to latest system message, last/latest iRecentUserMsgCnt user messages after the latest system prompt and its responses from the ai model will be sent to the ai-model, when querying for a new response. Note that if enabled, only user messages after the latest system message/prompt will be considered. @@ -325,7 +326,7 @@ work. ALERT: The simple minded way in which this is implemented, it can be dangerous in the worst case, Always remember to verify all the tool calls requested and the responses generated manually to -ensure everything is fine, during interaction with ai modles with tools support. +ensure everything is fine, during interaction with ai models with tools support. #### Builtin Tools @@ -358,9 +359,11 @@ that should be sent back to the ai model, in your constructed code. Update the tc_switch to include a object entry for the tool, which inturn includes * the meta data as well as * a reference to the handler and also + the handler should take toolCallId, toolName and toolArgs and pass these along to + web worker as needed. * the result key (was used previously, may use in future, but for now left as is) -#### Mapping tool calls and responses to normal assistant - user chat flow +#### OLD: Mapping tool calls and responses to normal assistant - user chat flow Instead of maintaining tool_call request and resultant response in logically seperate parallel channel used for requesting tool_calls by the assistant and the resulstant tool role response, @@ -375,10 +378,14 @@ NOTE: This flow tested to be ok enough with Gemma-3N-E4B-it-Q8_0 LLM ai model fo given the way current ai models work, most of them should understand things as needed, but need to test this with other ai models later. -TODO: Need to think later, whether to continue this simple flow, or atleast use tool role wrt +TODO:OLD: Need to think later, whether to continue this simple flow, or atleast use tool role wrt the tool call responses or even go further and have the logically seperate tool_calls request structures also. +DONE: rather both tool_calls structure wrt assistant messages and tool role based tool call +result messages are generated as needed. + + #### ToDo WebFetch and Local web proxy/caching server @@ -386,6 +393,7 @@ WebFetch and Local web proxy/caching server Try and trap promises based flows to ensure all generated results or errors if any are caught before responding back to the ai model. +Trap error responses. ### Debuging the handshake diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index bc4403516228d..9363063123930 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -219,7 +219,7 @@ let gUsageMsg = `
      • submit tool response placed into user query textarea
      -
    • Default ContextWindow = [System, Last4 Query+Resp, Cur Query].
    • +
    • Default ContextWindow = [System, Last9 Query+Resp, Cur Query].
      • ChatHistInCtxt, MaxTokens, ModelCtxt window to expand
      @@ -983,7 +983,7 @@ class Me { this.bCompletionFreshChatAlways = true; this.bCompletionInsertStandardRolePrefix = false; this.bTrimGarbage = true; - this.iRecentUserMsgCnt = 5; + this.iRecentUserMsgCnt = 10; /** @type {Object} */ this.sRecentUserMsgCnt = { "Full": -1, @@ -991,6 +991,7 @@ class Me { "Last1": 2, "Last2": 3, "Last4": 5, + "Last9": 10, }; this.apiEP = ApiEP.Type.Chat; /** @type {Object} */ From 75a63e3ef1e7a6c0c8408d6f430cfbf6cc7cf5a7 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 05:31:08 +0530 Subject: [PATCH 58/89] SimpleChatTC:ChatMessageEx: Better tool result extractor --- tools/server/public_simplechat/simplechat.js | 25 +++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/simplechat.js b/tools/server/public_simplechat/simplechat.js index 9363063123930..c7259fb950b1c 100644 --- a/tools/server/public_simplechat/simplechat.js +++ b/tools/server/public_simplechat/simplechat.js @@ -91,7 +91,7 @@ class ChatMessageEx { * Extract the elements of the all in one tool call result string * @param {string} allInOne */ - static extractToolCallResultAllInOne(allInOne) { + static extractToolCallResultAllInOneSimpleMinded(allInOne) { const regex = /\s*(.*?)<\/id>\s*(.*?)<\/name>\s*([\s\S]*?)<\/content>\s*<\/tool_response>/si; const caught = allInOne.match(regex) let data = { tool_call_id: "Error", name: "Error", content: "Error" } @@ -105,6 +105,29 @@ class ChatMessageEx { return data } + /** + * Extract the elements of the all in one tool call result string + * This should potentially account for content tag having xml content within to an extent. + * @param {string} allInOne + */ + static extractToolCallResultAllInOne(allInOne) { + const dParser = new DOMParser(); + const got = dParser.parseFromString(allInOne, 'text/xml'); + const parseErrors = got.querySelector('parseerror') + if (parseErrors) { + console.debug("WARN:ChatMessageEx:ExtractToolCallResultAllInOne:", parseErrors.textContent.trim()) + } + const id = got.querySelector('id')?.textContent.trim(); + const name = got.querySelector('name')?.textContent.trim(); + const content = got.querySelector('content')?.textContent.trim(); + let data = { + tool_call_id: id? id : "Error", + name: name? name : "Error", + content: content? content : "Error" + } + return data + } + /** * Set extra members into the ns object * @param {string | number} key From 99b04ec4388577323fe45f27c3c86fcb05e551ec Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 18:20:20 +0530 Subject: [PATCH 59/89] SimpleChatTC:ChatMessageEx: 1st go at trying to track promises --- .../server/public_simplechat/toolsworker.mjs | 9 ++-- tools/server/public_simplechat/xpromise.mjs | 46 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 tools/server/public_simplechat/xpromise.mjs diff --git a/tools/server/public_simplechat/toolsworker.mjs b/tools/server/public_simplechat/toolsworker.mjs index 590c45234be7b..d1c7a2e42b4b0 100644 --- a/tools/server/public_simplechat/toolsworker.mjs +++ b/tools/server/public_simplechat/toolsworker.mjs @@ -5,18 +5,19 @@ // /** - * Expects to get a message with identifier name and code to run - * Posts message with identifier name and data captured from console.log outputs + * Expects to get a message with id, name and code to run + * Posts message with id, name and data captured from console.log outputs */ import * as tconsole from "./toolsconsole.mjs" +import * as xpromise from "./xpromise.mjs" -self.onmessage = function (ev) { +self.onmessage = async function (ev) { tconsole.console_redir() try { - eval(ev.data.code) + await xpromise.evalWithPromiseTracking(ev.data.code); } catch (/** @type {any} */error) { console.log(`\n\nTool/Function call "${ev.data.name}" raised an exception:${error.name}:${error.message}\n\n`) } diff --git a/tools/server/public_simplechat/xpromise.mjs b/tools/server/public_simplechat/xpromise.mjs new file mode 100644 index 0000000000000..de3612d7649c3 --- /dev/null +++ b/tools/server/public_simplechat/xpromise.mjs @@ -0,0 +1,46 @@ +//@ts-check +// Helpers for a tracked promise land +// by Humans for All +// + + +/** + * @typedef {(resolve: (value: any) => void, reject: (reason?: any) => void) => void} PromiseExecutor + */ + + +/** + * Eval which allows promises generated by the evald code to be tracked. + * @param {string} codeToEval + */ +export function evalWithPromiseTracking(codeToEval) { + const _Promise = globalThis.Promise; + /** @type {any[]} */ + const trackedPromises = []; + + const Promise = function ( /** @type {PromiseExecutor} */ executor) { + const promise = new _Promise(executor); + trackedPromises.push(promise); + + promise.then = function (...args) { + const newPromise = _Promise.prototype.then.apply(this, args); + trackedPromises.push(newPromise); + return newPromise; + }; + + promise.catch = function (...args) { + const newPromise = _Promise.prototype.catch.apply(this, args); + trackedPromises.push(newPromise); + return newPromise; + }; + + return promise; + }; + + Promise.prototype = _Promise.prototype; + Object.assign(Promise, _Promise); + + eval(codeToEval); + + return _Promise.all(trackedPromises); +} From 7dc999f619fc1cd803a7647cade4ba46d608cb8f Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 21:48:18 +0530 Subject: [PATCH 60/89] SimpleChatTC:TrapPromise: log the trapping also possible refinement wrt trapping, if needed, added as comment all or allSettled to use or not is the question. whether to wait for a round trip through the related event loop or not is also a question. --- tools/server/public_simplechat/toolsworker.mjs | 2 ++ tools/server/public_simplechat/xpromise.mjs | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/toolsworker.mjs b/tools/server/public_simplechat/toolsworker.mjs index d1c7a2e42b4b0..b85b83b33b327 100644 --- a/tools/server/public_simplechat/toolsworker.mjs +++ b/tools/server/public_simplechat/toolsworker.mjs @@ -15,6 +15,7 @@ import * as xpromise from "./xpromise.mjs" self.onmessage = async function (ev) { + console.info("DBUG:WW:OnMessage started...") tconsole.console_redir() try { await xpromise.evalWithPromiseTracking(ev.data.code); @@ -23,4 +24,5 @@ self.onmessage = async function (ev) { } tconsole.console_revert() self.postMessage({ id: ev.data.id, name: ev.data.name, data: tconsole.gConsoleStr}) + console.info("DBUG:WW:OnMessage done") } diff --git a/tools/server/public_simplechat/xpromise.mjs b/tools/server/public_simplechat/xpromise.mjs index de3612d7649c3..7134293c0b4fa 100644 --- a/tools/server/public_simplechat/xpromise.mjs +++ b/tools/server/public_simplechat/xpromise.mjs @@ -13,22 +13,25 @@ * Eval which allows promises generated by the evald code to be tracked. * @param {string} codeToEval */ -export function evalWithPromiseTracking(codeToEval) { +export async function evalWithPromiseTracking(codeToEval) { const _Promise = globalThis.Promise; /** @type {any[]} */ const trackedPromises = []; const Promise = function ( /** @type {PromiseExecutor} */ executor) { + console.info("WW:PT:Promise") const promise = new _Promise(executor); trackedPromises.push(promise); promise.then = function (...args) { + console.info("WW:PT:Then") const newPromise = _Promise.prototype.then.apply(this, args); trackedPromises.push(newPromise); return newPromise; }; promise.catch = function (...args) { + console.info("WW:PT:Catch") const newPromise = _Promise.prototype.catch.apply(this, args); trackedPromises.push(newPromise); return newPromise; @@ -42,5 +45,7 @@ export function evalWithPromiseTracking(codeToEval) { eval(codeToEval); + //await Promise(resolve=>setTimeout(resolve, 0)); + //return _Promise.allSettled(trackedPromises); return _Promise.all(trackedPromises); } From 5d764933a99bd0f1144d7598edf1d9b2b0803124 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 22:10:27 +0530 Subject: [PATCH 61/89] SimpleChatTC:Promises: trap normal fetch (dont care await or not) --- tools/server/public_simplechat/xpromise.mjs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/server/public_simplechat/xpromise.mjs b/tools/server/public_simplechat/xpromise.mjs index 7134293c0b4fa..94d33ac3538f5 100644 --- a/tools/server/public_simplechat/xpromise.mjs +++ b/tools/server/public_simplechat/xpromise.mjs @@ -15,6 +15,8 @@ */ export async function evalWithPromiseTracking(codeToEval) { const _Promise = globalThis.Promise; + const _fetch = globalThis.fetch + /** @type {any[]} */ const trackedPromises = []; @@ -43,6 +45,13 @@ export async function evalWithPromiseTracking(codeToEval) { Promise.prototype = _Promise.prototype; Object.assign(Promise, _Promise); + const fetch = function(/** @type {any[]} */ ...args) { + console.info("WW:PT:Fetch") + const fpromise = _fetch(args); + trackedPromises.push(fpromise) + return fpromise; + } + eval(codeToEval); //await Promise(resolve=>setTimeout(resolve, 0)); From 788a9e489262e3823b5da0dc5b975a4c29307b8d Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 22:52:07 +0530 Subject: [PATCH 62/89] SimpleChatTC: Allow await in generated code that will be evald --- tools/server/public_simplechat/xpromise.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/xpromise.mjs b/tools/server/public_simplechat/xpromise.mjs index 94d33ac3538f5..801ef7adc6c48 100644 --- a/tools/server/public_simplechat/xpromise.mjs +++ b/tools/server/public_simplechat/xpromise.mjs @@ -25,6 +25,7 @@ export async function evalWithPromiseTracking(codeToEval) { const promise = new _Promise(executor); trackedPromises.push(promise); + // @ts-ignore promise.then = function (...args) { console.info("WW:PT:Then") const newPromise = _Promise.prototype.then.apply(this, args); @@ -47,12 +48,15 @@ export async function evalWithPromiseTracking(codeToEval) { const fetch = function(/** @type {any[]} */ ...args) { console.info("WW:PT:Fetch") + // @ts-ignore const fpromise = _fetch(args); trackedPromises.push(fpromise) return fpromise; } - eval(codeToEval); + //let tf = new Function(codeToEval); + //await tf() + await eval(`(async () => { ${codeToEval} })()`); //await Promise(resolve=>setTimeout(resolve, 0)); //return _Promise.allSettled(trackedPromises); From c155bd313cd74a34a965d0aefc9c5a38b932e9f1 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 23:13:02 +0530 Subject: [PATCH 63/89] SimpleChatTC:Ensure fetch's promise chain is also trapped Dont forget to map members of got entity from fetch to things from saved original promise, bcas remember what is got is a promise. also add some comments around certain decisions and needed exploration --- tools/server/public_simplechat/xpromise.mjs | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/xpromise.mjs b/tools/server/public_simplechat/xpromise.mjs index 801ef7adc6c48..6f001ef9de6e7 100644 --- a/tools/server/public_simplechat/xpromise.mjs +++ b/tools/server/public_simplechat/xpromise.mjs @@ -1,5 +1,6 @@ //@ts-check // Helpers for a tracked promise land +// Traps regular promise as well as promise by fetch // by Humans for All // @@ -51,14 +52,36 @@ export async function evalWithPromiseTracking(codeToEval) { // @ts-ignore const fpromise = _fetch(args); trackedPromises.push(fpromise) + + // @ts-ignore + fpromise.then = function (...args) { + console.info("WW:PT:FThen") + const newPromise = _Promise.prototype.then.apply(this, args); + trackedPromises.push(newPromise); + return newPromise; + }; + + fpromise.catch = function (...args) { + console.info("WW:PT:FCatch") + const newPromise = _Promise.prototype.catch.apply(this, args); + trackedPromises.push(newPromise); + return newPromise; + }; + return fpromise; } + fetch.prototype = _fetch.prototype; + Object.assign(fetch, _fetch); + //let tf = new Function(codeToEval); //await tf() await eval(`(async () => { ${codeToEval} })()`); + // Should I allow things to go back to related event loop once //await Promise(resolve=>setTimeout(resolve, 0)); - //return _Promise.allSettled(trackedPromises); + + // Need and prefer promise failures to be trapped using reject/catch logic + // so using all instead of allSettled. return _Promise.all(trackedPromises); } From 1e1bb9a712405a13840aaa4af2fbec821a57e37a Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Wed, 15 Oct 2025 23:31:07 +0530 Subject: [PATCH 64/89] SimpleChatTC: update readme wrt promise related trapping --- tools/server/public_simplechat/readme.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index d50588cce5e78..0ae62a3a7b079 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -385,16 +385,22 @@ structures also. DONE: rather both tool_calls structure wrt assistant messages and tool role based tool call result messages are generated as needed. +#### Related stuff + +Promise as well as users of promise (for now fetch) have been trapped wrt their then and catch flow, +so that any scheduled asynchronous code or related async error handling using promise mechanism also +gets executed, before tool calling returns and thus data / error generated by those async code also +get incorporated in result sent to ai engine on the server side. #### ToDo WebFetch and Local web proxy/caching server -Try and trap promises based flows to ensure all generated results or errors if any are caught -before responding back to the ai model. +Is the promise land trap deep enough, need to think through and explore around this once later. Trap error responses. + ### Debuging the handshake When working with llama.cpp server based GenAi/LLM running locally From 9b1ed1fbf71153d62a4992b74a3cf1165c564d57 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 11:52:21 +0530 Subject: [PATCH 65/89] SimpleChatTC:WebFetchThroughProxy:Initial go creating request --- tools/server/public_simplechat/tooljs.mjs | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index a44333ca1b3e7..9591aeef4b8a3 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -72,6 +72,48 @@ function calc_run(toolcallid, toolname, obj) { } +let weburlfetch_meta = { + "type": "function", + "function": { + "name": "web_url_fetch", + "description": "Fetch the requested web url through a proxy server in few seconds", + "parameters": { + "type": "object", + "properties": { + "url":{ + "type":"string", + "description":"the url of the page / content to fetch from the internet" + } + }, + "required": ["url"] + } + } + } + + +/** + * Implementation of the web url logic. Dumb initial go. + * Expects a simple minded proxy server to be running locally + * * listening on port 3128 + * * expecting http requests + * * with a query token named path which gives the actual url to fetch + * ALERT: Has access to the javascript web worker environment and can mess with it and beyond + * @param {string} toolcallid + * @param {string} toolname + * @param {any} obj + */ +function weburlfetch_run(toolcallid, toolname, obj) { + if (gToolsWorker.onmessage != null) { + let newUrl = `http://127.0.0.1:3128/?path=${obj.url}` + fetch(newUrl).then(resp=>resp.text()).then(data => { + gToolsWorker.onmessage(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) + }).catch((err)=>{ + gToolsWorker.onmessage(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: `Error:${err}`}})) + }) + } +} + + /** * @type {Object>} */ @@ -85,6 +127,11 @@ export let tc_switch = { "handler": calc_run, "meta": calc_meta, "result": "" + }, + "web_url_fetch": { + "handler": weburlfetch_run, + "meta": weburlfetch_meta, + "result": "" } } From 1b361ee9eb9426fb9b4fde3d5b37324bcc0dbc3b Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 17:45:33 +0530 Subject: [PATCH 66/89] SimpleChatTC:SimpleProxy:Process args --port --- .../local.tools/simpleproxy.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tools/server/public_simplechat/local.tools/simpleproxy.py diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py new file mode 100644 index 0000000000000..85940e60d86d3 --- /dev/null +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -0,0 +1,37 @@ +# A simple proxy server +# by Humans for All +# +# Listens on the specified port (defaults to squids 3128) +# * if a url query is got (http://localhost:3128/?url=http://site.of.interest/path/of/interest) +# fetches the contents of the specified url and returns the same to the requester +# + +import sys + +gMe = {} + +def process_args(args: list[str]): + global gMe + gMe['INTERNAL.ProcessArgs.Malformed'] = [] + gMe['INTERNAL.ProcessArgs.Unknown'] = [] + iArg = 1 + while iArg < len(args): + cArg = args[iArg] + if (not cArg.startswith("--")): + gMe['INTERNAL.ProcessArgs.Malformed'].append(cArg) + print(f"WARN:ProcessArgs:{iArg}:IgnoringMalformedCommandOr???:{cArg}") + iArg += 1 + continue + match cArg: + case '--port': + iArg += 1 + gMe[cArg] = args[iArg] + iArg += 1 + case _: + gMe['INTERNAL.ProcessArgs.Unknown'].append(cArg) + print(f"WARN:ProcessArgs:{iArg}:IgnoringUnknownCommand:{cArg}") + iArg += 1 + + +if __name__ == "__main__": + process_args(sys.argv) From 9ec3d4514119e0d6da3cafbc1b01812ca97736eb Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 18:06:56 +0530 Subject: [PATCH 67/89] SimpleChatTC:SimpleProxy: Start server, Show requested path --- .../local.tools/simpleproxy.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 85940e60d86d3..c9d7264b7ae89 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -6,10 +6,20 @@ # fetches the contents of the specified url and returns the same to the requester # + import sys +import http.server + gMe = {} + +class ProxyHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + print(self.path) + + + def process_args(args: list[str]): global gMe gMe['INTERNAL.ProcessArgs.Malformed'] = [] @@ -33,5 +43,15 @@ def process_args(args: list[str]): iArg += 1 +def run(): + gMe['server'] = http.server.HTTPServer(('',gMe['--port']), ProxyHandler) + try: + gMe['server'].serve_forever() + except KeyboardInterrupt: + print("INFO:Run:Shuting down...") + gMe['server'].server_close() + sys.exit(0) + if __name__ == "__main__": process_args(sys.argv) + run() From 620da4533b174724d5936ff57f6640cc5a8682c8 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 18:18:45 +0530 Subject: [PATCH 68/89] SimpleChatTC:SimpleProxy: Cleanup for basic run --- .../local.tools/simpleproxy.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index c9d7264b7ae89..ce8c0c5718d37 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -11,7 +11,10 @@ import http.server -gMe = {} +gMe = { + '--port': 3128, + 'server': None +} class ProxyHandler(http.server.BaseHTTPRequestHandler): @@ -35,7 +38,7 @@ def process_args(args: list[str]): match cArg: case '--port': iArg += 1 - gMe[cArg] = args[iArg] + gMe[cArg] = int(args[iArg]) iArg += 1 case _: gMe['INTERNAL.ProcessArgs.Unknown'].append(cArg) @@ -44,13 +47,20 @@ def process_args(args: list[str]): def run(): - gMe['server'] = http.server.HTTPServer(('',gMe['--port']), ProxyHandler) try: + gMe['server'] = http.server.HTTPServer(('',gMe['--port']), ProxyHandler) gMe['server'].serve_forever() except KeyboardInterrupt: print("INFO:Run:Shuting down...") - gMe['server'].server_close() + if (gMe['server']): + gMe['server'].server_close() sys.exit(0) + except Exception as exc: + print(f"ERRR:Run:Exception:{exc}") + if (gMe['server']): + gMe['server'].server_close() + sys.exit(1) + if __name__ == "__main__": process_args(sys.argv) From bdd054bbb7bda54fa4303b5fc68ec5668539c447 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 18:39:02 +0530 Subject: [PATCH 69/89] SimpleChatTC:SimpleProxy: Extract and check path, route to handlers --- .../local.tools/simpleproxy.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index ce8c0c5718d37..105ba794db0b6 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -9,6 +9,7 @@ import sys import http.server +import urllib.parse gMe = { @@ -19,8 +20,29 @@ class ProxyHandler(http.server.BaseHTTPRequestHandler): def do_GET(self): - print(self.path) + print(f"DBUG:ProxyHandler:{self.path}") + pr = urllib.parse.urlparse(self.path) + print(f"DBUG:ProxyHandler:{pr}") + match pr.path: + case '/urlraw': + handle_urlraw(self, pr) + case '/urltext': + handle_urltext(self, pr) + case _: + print(f"WARN:ProxyHandler:UnknownPath{pr.path}") + self.send_response(400) + self.send_header('Content-Type', 'plain/text') + self.end_headers() + self.wfile.write(f"WARN:UnknownPath:{pr.path}") + + +def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): + print(f"DBUG:HandleUrlRaw:{pr}") + queryParams = urllib.parse.parse_qs(pr.query) + +def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): + print(f"DBUG:HandleUrlText:{pr}") def process_args(args: list[str]): From 5ac60a87d4c75ab5964db0e3533c920144abcdeb Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 18:54:20 +0530 Subject: [PATCH 70/89] SimpleChatTC:SimpleProxy:implement handle_urlraw A basic go at it --- .../local.tools/simpleproxy.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 105ba794db0b6..5b41b37f0c9d7 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -19,6 +19,7 @@ class ProxyHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): print(f"DBUG:ProxyHandler:{self.path}") pr = urllib.parse.urlparse(self.path) @@ -30,19 +31,36 @@ def do_GET(self): handle_urltext(self, pr) case _: print(f"WARN:ProxyHandler:UnknownPath{pr.path}") - self.send_response(400) - self.send_header('Content-Type', 'plain/text') - self.end_headers() - self.wfile.write(f"WARN:UnknownPath:{pr.path}") + self.send_error(400, f"WARN:UnknownPath:{pr.path}") def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): print(f"DBUG:HandleUrlRaw:{pr}") queryParams = urllib.parse.parse_qs(pr.query) + url = queryParams['url'] + if (not url) or (len(url) == 0): + ph.send_error(400, "WARN:UrlRaw:MissingUrl") + return + try: + # Get requested url + with urllib.request.urlopen(url, timeout=10) as response: + contentData = response.read() + statusCode = response.status or 200 + contentType = response.getheader('Content-Type') or 'text/html' + # Send back to client + ph.send_response(statusCode) + ph.send_header('Content-Type', contentType) + # Add CORS for browser fetch, just inc ase + ph.send_header('Access-Control-Allow-Origin', '*') + ph.end_headers() + ph.wfile.write(contentData) + except Exception as exc: + ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): print(f"DBUG:HandleUrlText:{pr}") + ph.send_error(400, "WARN:UrlText:Not implemented") def process_args(args: list[str]): From ac448c34225efcb824b4c385bad7e57f7211f649 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 18:58:06 +0530 Subject: [PATCH 71/89] SimpleChatTC:SimpleProxy:UrlRaw: Fixup basic oversight wrt 1st go --- tools/server/public_simplechat/local.tools/simpleproxy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 5b41b37f0c9d7..d1b229f5cf24a 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -10,6 +10,7 @@ import sys import http.server import urllib.parse +import urllib.request gMe = { @@ -38,6 +39,8 @@ def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): print(f"DBUG:HandleUrlRaw:{pr}") queryParams = urllib.parse.parse_qs(pr.query) url = queryParams['url'] + print(f"DBUG:HandleUrlRaw:Url:{url}") + url = url[0] if (not url) or (len(url) == 0): ph.send_error(400, "WARN:UrlRaw:MissingUrl") return From 2ca2b980540f35ba818716f87a50581273ed7296 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 19:02:55 +0530 Subject: [PATCH 72/89] SimpleChatTC:WebFetch: Update to use internal SimpleProxy.py --- tools/server/public_simplechat/tooljs.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 9591aeef4b8a3..9fb306ccb65e8 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -104,7 +104,7 @@ let weburlfetch_meta = { */ function weburlfetch_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { - let newUrl = `http://127.0.0.1:3128/?path=${obj.url}` + let newUrl = `http://127.0.0.1:3128/urlraw?url=${obj.url}` fetch(newUrl).then(resp=>resp.text()).then(data => { gToolsWorker.onmessage(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) }).catch((err)=>{ From 04a5f95ed3a9a5d310c0d35fde6227dc80c66c63 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Thu, 16 Oct 2025 22:47:21 +0530 Subject: [PATCH 73/89] SimpleChatTC:SimpleProxy: Cleanup few messages --- tools/server/public_simplechat/local.tools/simpleproxy.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index d1b229f5cf24a..f1322913d745f 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -53,7 +53,7 @@ def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): # Send back to client ph.send_response(statusCode) ph.send_header('Content-Type', contentType) - # Add CORS for browser fetch, just inc ase + # Add CORS for browser fetch, just in case ph.send_header('Access-Control-Allow-Origin', '*') ph.end_headers() ph.wfile.write(contentData) @@ -91,7 +91,9 @@ def process_args(args: list[str]): def run(): try: - gMe['server'] = http.server.HTTPServer(('',gMe['--port']), ProxyHandler) + gMe['serverAddr'] = ('', gMe['--port']) + gMe['server'] = http.server.HTTPServer(gMe['serverAddr'], ProxyHandler) + print(f"INFO:Run:Starting on {gMe['serverAddr']}") gMe['server'].serve_forever() except KeyboardInterrupt: print("INFO:Run:Shuting down...") @@ -99,7 +101,7 @@ def run(): gMe['server'].server_close() sys.exit(0) except Exception as exc: - print(f"ERRR:Run:Exception:{exc}") + print(f"ERRR:Run:Exiting:Exception:{exc}") if (gMe['server']): gMe['server'].server_close() sys.exit(1) From 295893cf8dad553e8bfe190016a8e96626aef9c9 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 00:11:01 +0530 Subject: [PATCH 74/89] SimpleChatTC:SimpleProxy:Common UrlReq helper for UrlRaw & UrlText Declare the result of UrlReq as a DataClass, so that one doesnt goof up wrt updating and accessing members. Duplicate UrlRaw into UrlText, need to add Text extracting from html next for UrlText --- .../local.tools/simpleproxy.py | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index f1322913d745f..d1f4cb3ec594a 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -11,6 +11,7 @@ import http.server import urllib.parse import urllib.request +from dataclasses import dataclass gMe = { @@ -35,35 +36,69 @@ def do_GET(self): self.send_error(400, f"WARN:UnknownPath:{pr.path}") -def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): - print(f"DBUG:HandleUrlRaw:{pr}") +@dataclass(frozen=True) +class UrlReqResp: + callOk: bool + httpStatus: int + httpStatusMsg: str = "" + contentType: str = "" + contentData: urllib.request._UrlopenRet = "" + + +def handle_urlreq(pr: urllib.parse.ParseResult, tag: str): + print(f"DBUG:{tag}:{pr}") queryParams = urllib.parse.parse_qs(pr.query) url = queryParams['url'] - print(f"DBUG:HandleUrlRaw:Url:{url}") + print(f"DBUG:{tag}:Url:{url}") url = url[0] if (not url) or (len(url) == 0): - ph.send_error(400, "WARN:UrlRaw:MissingUrl") - return + return UrlReqResp(False, 400, f"WARN:{tag}:MissingUrl") try: # Get requested url with urllib.request.urlopen(url, timeout=10) as response: contentData = response.read() statusCode = response.status or 200 contentType = response.getheader('Content-Type') or 'text/html' + return UrlReqResp(True, statusCode, "", contentType, contentData) + except Exception as exc: + return UrlReqResp(False, 502, f"WARN:UrlFetchFailed:{exc}") + + +def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): + try: + # Get requested url + got = handle_urlreq(pr, "HandleUrlRaw") + if not got.callOk: + ph.send_error(got.httpStatus, got.httpStatusMsg) + return # Send back to client - ph.send_response(statusCode) - ph.send_header('Content-Type', contentType) + ph.send_response(got.httpStatus) + ph.send_header('Content-Type', got.contentType) # Add CORS for browser fetch, just in case ph.send_header('Access-Control-Allow-Origin', '*') ph.end_headers() - ph.wfile.write(contentData) + ph.wfile.write(got.contentData) except Exception as exc: ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): - print(f"DBUG:HandleUrlText:{pr}") - ph.send_error(400, "WARN:UrlText:Not implemented") + try: + # Get requested url + got = handle_urlreq(pr, "HandleUrlText") + if not got.callOk: + ph.send_error(got.httpStatus, got.httpStatusMsg) + return + # Extract Text + # Send back to client + ph.send_response(got.httpStatus) + ph.send_header('Content-Type', got.contentType) + # Add CORS for browser fetch, just in case + ph.send_header('Access-Control-Allow-Origin', '*') + ph.end_headers() + ph.wfile.write(got.contentData) + except Exception as exc: + ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") def process_args(args: list[str]): From 469453759cc1f07c5731f467b3216afdcc4b7153 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 00:42:04 +0530 Subject: [PATCH 75/89] SimpleChatTC:SimpleProxy: ElementTree, No _UrlopenRet As _UrlopenRet not exposed for use outside urllib, so decode and encode the data. Add skeleton to try get the html/xml tree top elements --- .../public_simplechat/local.tools/simpleproxy.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index d1f4cb3ec594a..0014b0219b995 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -12,6 +12,7 @@ import urllib.parse import urllib.request from dataclasses import dataclass +import xml.etree.ElementTree as xmlET gMe = { @@ -42,7 +43,7 @@ class UrlReqResp: httpStatus: int httpStatusMsg: str = "" contentType: str = "" - contentData: urllib.request._UrlopenRet = "" + contentData: str = "" def handle_urlreq(pr: urllib.parse.ParseResult, tag: str): @@ -56,7 +57,7 @@ def handle_urlreq(pr: urllib.parse.ParseResult, tag: str): try: # Get requested url with urllib.request.urlopen(url, timeout=10) as response: - contentData = response.read() + contentData = response.read().decode('utf-8') statusCode = response.status or 200 contentType = response.getheader('Content-Type') or 'text/html' return UrlReqResp(True, statusCode, "", contentType, contentData) @@ -77,7 +78,7 @@ def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): # Add CORS for browser fetch, just in case ph.send_header('Access-Control-Allow-Origin', '*') ph.end_headers() - ph.wfile.write(got.contentData) + ph.wfile.write(got.contentData.encode('utf-8')) except Exception as exc: ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") @@ -90,13 +91,16 @@ def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): ph.send_error(got.httpStatus, got.httpStatusMsg) return # Extract Text + html = xmlET.fromstring(got.contentData) + for el in html.iter(): + print(el) # Send back to client ph.send_response(got.httpStatus) ph.send_header('Content-Type', got.contentType) # Add CORS for browser fetch, just in case ph.send_header('Access-Control-Allow-Origin', '*') ph.end_headers() - ph.wfile.write(got.contentData) + ph.wfile.write(got.contentData.encode('utf-8')) except Exception as exc: ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") From 696d5604ea52e77030ba9e61411528a70ca72439 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 01:30:46 +0530 Subject: [PATCH 76/89] SimpleChatTC:SimpleProxy: Switch to html.parser As html can be malformed, xml ElementTree XMLParser cant handle the same properly, so switch to the HtmlParser helper class that is provided by python and try extend it. Currently a minimal skeleton to just start it out, which captures only the body contents. --- .../local.tools/simpleproxy.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 0014b0219b995..4ac26b6b22d89 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -12,7 +12,7 @@ import urllib.parse import urllib.request from dataclasses import dataclass -import xml.etree.ElementTree as xmlET +import html.parser gMe = { @@ -83,6 +83,26 @@ def handle_urlraw(ph: ProxyHandler, pr: urllib.parse.ParseResult): ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") +class TextHtmlParser(html.parser.HTMLParser): + + def __init__(self): + super().__init__() + self.bBody = False + self.text = "" + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]): + if tag == 'body': + self.bBody = True + + def handle_endtag(self, tag: str): + if tag == 'body': + self.bBody = False + + def handle_data(self, data: str): + if self.bBody: + self.text += f"{data}\n" + + def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): try: # Get requested url @@ -91,16 +111,15 @@ def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): ph.send_error(got.httpStatus, got.httpStatusMsg) return # Extract Text - html = xmlET.fromstring(got.contentData) - for el in html.iter(): - print(el) + textHtml = TextHtmlParser() + textHtml.feed(got.contentData) # Send back to client ph.send_response(got.httpStatus) ph.send_header('Content-Type', got.contentType) # Add CORS for browser fetch, just in case ph.send_header('Access-Control-Allow-Origin', '*') ph.end_headers() - ph.wfile.write(got.contentData.encode('utf-8')) + ph.wfile.write(textHtml.text.encode('utf-8')) except Exception as exc: ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") From 2e6a386eb6c87d8ecfb7e9641ff47f30dddc7b8b Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 01:39:35 +0530 Subject: [PATCH 77/89] SimpleChatTC:SimpleProxy:UrlText: Capture body except for scripts --- .../server/public_simplechat/local.tools/simpleproxy.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 4ac26b6b22d89..242e644648d98 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -88,18 +88,25 @@ class TextHtmlParser(html.parser.HTMLParser): def __init__(self): super().__init__() self.bBody = False + self.bCapture = False self.text = "" def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]): if tag == 'body': self.bBody = True + self.bCapture = True + if tag == 'script': + self.bCapture = False def handle_endtag(self, tag: str): if tag == 'body': self.bBody = False + if tag == 'script': + if self.bBody: + self.bCapture = True def handle_data(self, data: str): - if self.bBody: + if self.bCapture: self.text += f"{data}\n" From 0a3b4c236d5198892ae452951c721d3e6b919a99 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 01:50:26 +0530 Subject: [PATCH 78/89] SimpleChatTC:SimpleProxy:UrlText: Avoid style blocks also --- tools/server/public_simplechat/local.tools/simpleproxy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 242e644648d98..3076c0d4aeb80 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -97,11 +97,13 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]): self.bCapture = True if tag == 'script': self.bCapture = False + if tag == 'style': + self.bCapture = False def handle_endtag(self, tag: str): if tag == 'body': self.bBody = False - if tag == 'script': + if tag == 'script' or tag == 'style': if self.bBody: self.bCapture = True From 61b937104c108e96c882db8eba7994ed921d0a46 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 02:08:38 +0530 Subject: [PATCH 79/89] SimpleChatTC:WebUrl FetchStrip through simple proxy --- tools/server/public_simplechat/tooljs.mjs | 67 +++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 9fb306ccb65e8..e9dc3c6f90a8c 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -72,6 +72,16 @@ function calc_run(toolcallid, toolname, obj) { } +/** + * Send a message to Tools WebWorker's monitor in main thread directly + * @param {MessageEvent} mev + */ +function message_toolsworker(mev) { + // @ts-ignore + gToolsWorker.onmessage(mev) +} + + let weburlfetch_meta = { "type": "function", "function": { @@ -96,8 +106,8 @@ let weburlfetch_meta = { * Expects a simple minded proxy server to be running locally * * listening on port 3128 * * expecting http requests - * * with a query token named path which gives the actual url to fetch - * ALERT: Has access to the javascript web worker environment and can mess with it and beyond + * * with a query token named url which gives the actual url to fetch + * ALERT: Accesses a external web proxy/caching server be aware and careful * @param {string} toolcallid * @param {string} toolname * @param {any} obj @@ -106,9 +116,53 @@ function weburlfetch_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { let newUrl = `http://127.0.0.1:3128/urlraw?url=${obj.url}` fetch(newUrl).then(resp=>resp.text()).then(data => { - gToolsWorker.onmessage(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) + message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) + }).catch((err)=>{ + message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: `Error:${err}`}})) + }) + } +} + + +let weburlfetchstrip_meta = { + "type": "function", + "function": { + "name": "web_url_fetch_strip", + "description": "Fetch the requested web url through a proxy server and strip away head, script, styles blocks before sending remaining body in few seconds", + "parameters": { + "type": "object", + "properties": { + "url":{ + "type":"string", + "description":"the url of the page that will be fetched from the internet and inturn contents stripped to some extent" + } + }, + "required": ["url"] + } + } + } + + +/** + * Implementation of the web url logic. Dumb initial go. + * Expects a simple minded proxy server to be running locally + * * listening on port 3128 + * * expecting http requests + * * with a query token named url which gives the actual url to fetch + * * strips out head as well as any script and style blocks in body + * before returning remaining body contents. + * ALERT: Accesses a external web proxy/caching server be aware and careful + * @param {string} toolcallid + * @param {string} toolname + * @param {any} obj + */ +function weburlfetchstrip_run(toolcallid, toolname, obj) { + if (gToolsWorker.onmessage != null) { + let newUrl = `http://127.0.0.1:3128/urltext?url=${obj.url}` + fetch(newUrl).then(resp=>resp.text()).then(data => { + message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) }).catch((err)=>{ - gToolsWorker.onmessage(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: `Error:${err}`}})) + message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: `Error:${err}`}})) }) } } @@ -132,6 +186,11 @@ export let tc_switch = { "handler": weburlfetch_run, "meta": weburlfetch_meta, "result": "" + }, + "web_url_fetch_strip": { + "handler": weburlfetchstrip_run, + "meta": weburlfetchstrip_meta, + "result": "" } } From 378bde5fc82df9b05f32d4c18ee00c25e05d9ec9 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 02:52:37 +0530 Subject: [PATCH 80/89] SimpleChatTC:SimpleProxy:UrlText: Try strip empty lines some what --- .../public_simplechat/local.tools/simpleproxy.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 3076c0d4aeb80..ad21cb3dc4df1 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -111,6 +111,16 @@ def handle_data(self, data: str): if self.bCapture: self.text += f"{data}\n" + def get_stripped_text(self): + oldLen = -99 + newLen = len(self.text) + aStripped = self.text; + while oldLen != newLen: + oldLen = newLen + aStripped = aStripped.replace("\n\n\n","\n") + newLen = len(aStripped) + return aStripped + def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): try: @@ -128,7 +138,7 @@ def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): # Add CORS for browser fetch, just in case ph.send_header('Access-Control-Allow-Origin', '*') ph.end_headers() - ph.wfile.write(textHtml.text.encode('utf-8')) + ph.wfile.write(textHtml.get_stripped_text().encode('utf-8')) except Exception as exc: ph.send_error(502, f"WARN:UrlFetchFailed:{exc}") From e7277cbb6c1c05b23a7c8be18d00d3b3b2624b44 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 03:02:53 +0530 Subject: [PATCH 81/89] SimpleChatTC:SimpleProxy:UrlText: Slightly better trimming First identify lines which have only whitespace and replace them with lines with only newline char in them. Next strip out adjacent lines, if they have only newlines --- .../local.tools/simpleproxy.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index ad21cb3dc4df1..ad85b1b809425 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -90,6 +90,7 @@ def __init__(self): self.bBody = False self.bCapture = False self.text = "" + self.textStripped = "" def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]): if tag == 'body': @@ -111,15 +112,33 @@ def handle_data(self, data: str): if self.bCapture: self.text += f"{data}\n" - def get_stripped_text(self): + def syncup(self): + self.textStripped = self.text + + def strip_adjacent_newlines(self): oldLen = -99 - newLen = len(self.text) - aStripped = self.text; + newLen = len(self.textStripped) + aStripped = self.textStripped; while oldLen != newLen: oldLen = newLen aStripped = aStripped.replace("\n\n\n","\n") newLen = len(aStripped) - return aStripped + self.textStripped = aStripped + + def strip_whitespace_lines(self): + aLines = self.textStripped.splitlines() + self.textStripped = "" + for line in aLines: + if (len(line.strip())==0): + self.textStripped += "\n" + continue + self.textStripped += f"{line}\n" + + def get_stripped_text(self): + self.syncup() + self.strip_whitespace_lines() + self.strip_adjacent_newlines() + return self.textStripped def handle_urltext(ph: ProxyHandler, pr: urllib.parse.ParseResult): From 9db2f278c0844aedfcb9ed126d97465017ef458e Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 03:56:35 +0530 Subject: [PATCH 82/89] SimpleChatTC:WebUrlText:Update name and desc to see if prefered --- tools/server/public_simplechat/tooljs.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index e9dc3c6f90a8c..cb75dc019cf29 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -127,14 +127,14 @@ function weburlfetch_run(toolcallid, toolname, obj) { let weburlfetchstrip_meta = { "type": "function", "function": { - "name": "web_url_fetch_strip", - "description": "Fetch the requested web url through a proxy server and strip away head, script, styles blocks before sending remaining body in few seconds", + "name": "web_url_fetch_strip_htmltags_and_some_useless", + "description": "Fetch the requested web url through a proxy server and strip away html tags as well as head, script, styles blocks before returning the remaining body in few seconds", "parameters": { "type": "object", "properties": { "url":{ "type":"string", - "description":"the url of the page that will be fetched from the internet and inturn contents stripped to some extent" + "description":"the url of the page that will be fetched from the internet and inturn unwanted stuff stripped from its contents to some extent" } }, "required": ["url"] @@ -187,7 +187,7 @@ export let tc_switch = { "meta": weburlfetch_meta, "result": "" }, - "web_url_fetch_strip": { + "web_url_fetch_strip_htmltags_and_some_useless": { "handler": weburlfetchstrip_run, "meta": weburlfetchstrip_meta, "result": "" From c5a602f25b1fa1e9ce217606e3c7fab30e48486b Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 04:13:46 +0530 Subject: [PATCH 83/89] SimpleChatTC:SimpleProxy:Options just in case --- .../public_simplechat/local.tools/simpleproxy.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index ad85b1b809425..4e69c1cf61550 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -23,19 +23,29 @@ class ProxyHandler(http.server.BaseHTTPRequestHandler): + # Handle GET requests def do_GET(self): - print(f"DBUG:ProxyHandler:{self.path}") + print(f"DBUG:ProxyHandler:GET:{self.path}") pr = urllib.parse.urlparse(self.path) - print(f"DBUG:ProxyHandler:{pr}") + print(f"DBUG:ProxyHandler:GET:{pr}") match pr.path: case '/urlraw': handle_urlraw(self, pr) case '/urltext': handle_urltext(self, pr) case _: - print(f"WARN:ProxyHandler:UnknownPath{pr.path}") + print(f"WARN:ProxyHandler:GET:UnknownPath{pr.path}") self.send_error(400, f"WARN:UnknownPath:{pr.path}") + # Handle OPTIONS for CORS preflights (just in case from browser) + def do_OPTIONS(self): + print(f"DBUG:ProxyHandler:OPTIONS:{self.path}") + self.send_response(200) + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS') + self.send_header('Access-Control-Allow-Headers', '*') + self.end_headers() + @dataclass(frozen=True) class UrlReqResp: From 6efbd126060816ab7d7f820e833b55ad291f8ce2 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 05:22:13 +0530 Subject: [PATCH 84/89] SimpleChatTC:WebFetch:UrlEnc url2fetch b4Passing toProxy asQuery Ensures that if the url being requested as any query strings in them then things dont get messed up, when the url to get inc its query is extracted from the proxy request's query string --- tools/server/public_simplechat/tooljs.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index cb75dc019cf29..5b7979f0ab5a0 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -114,7 +114,7 @@ let weburlfetch_meta = { */ function weburlfetch_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { - let newUrl = `http://127.0.0.1:3128/urlraw?url=${obj.url}` + let newUrl = `http://127.0.0.1:3128/urlraw?url=${encodeURIComponent(obj.url)}` fetch(newUrl).then(resp=>resp.text()).then(data => { message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) }).catch((err)=>{ @@ -158,7 +158,7 @@ let weburlfetchstrip_meta = { */ function weburlfetchstrip_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { - let newUrl = `http://127.0.0.1:3128/urltext?url=${obj.url}` + let newUrl = `http://127.0.0.1:3128/urltext?url=${encodeURIComponent(obj.url)}` fetch(newUrl).then(resp=>resp.text()).then(data => { message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) }).catch((err)=>{ From 8f0700a462b83699b01ee5dc7218d315e7eed193 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 05:36:43 +0530 Subject: [PATCH 85/89] SimpleChatTC: Update readme wrt web fetch and related simple proxy --- tools/server/public_simplechat/readme.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index 0ae62a3a7b079..d90261ed380c9 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -334,16 +334,21 @@ The following tools/functions are currently provided by default * simple_calculator - which can solve simple arithmatic expressions * run_javascript_function_code - which can be used to run some javascript code in the browser context. +* web_url_fetch - fetch requested url through a proxy server +* web_url_fetch_strip_htmltags_and_some_useless - fetch requested url through a proxy server + and also try strip the html respose of html tags and also head, script & style blocks. Currently the generated code / expression is run through a simple minded eval inside a web worker mechanism. Use of WebWorker helps avoid exposing browser global scope to the generated code directly. However any shared web worker scope isnt isolated. Either way always remember to cross check the tool requests and generated responses when using tool calling. -May add -* web_fetch along with a corresponding simple local web proxy/caching server logic that can bypass - the CORS restrictions applied if trying to directly fetch from the browser js runtime environment. - Inturn maybe with a white list of allowed sites to access or so. +web_url_fetch and family works along with a corresponding simple local web proxy/caching server logic +that can bypass the CORS restrictions applied if trying to directly fetch from the browser js runtime +environment. Depending on the path specified on the proxy server, if urltext, it additionally tries +to convert html content into equivalent text to some extent. May add support for white list of allowed +sites to access or so. +* tools/server/public_simplechat/local.tools/simpleproxy.py #### Extending with new tools @@ -394,8 +399,6 @@ get incorporated in result sent to ai engine on the server side. #### ToDo -WebFetch and Local web proxy/caching server - Is the promise land trap deep enough, need to think through and explore around this once later. Trap error responses. From 514cec05bda92cadc8403814693ca6b5f2f7b6d5 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 18:01:25 +0530 Subject: [PATCH 86/89] SimpleChatTC:SimpleProxy:HtmlParser more generic and flexible also now track header, footer and nav so that they arent captured --- .../local.tools/simpleproxy.py | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 4e69c1cf61550..78cf6d6767ee2 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -97,29 +97,34 @@ class TextHtmlParser(html.parser.HTMLParser): def __init__(self): super().__init__() - self.bBody = False + self.inside = { + 'body': False, + 'script': False, + 'style': False, + 'header': False, + 'footer': False, + 'nav': False + } + self.monitored = [ 'body', 'script', 'style', 'header', 'footer', 'nav' ] self.bCapture = False self.text = "" self.textStripped = "" + def do_capture(self): + if self.inside['body'] and not (self.inside['script'] or self.inside['style'] or self.inside['header'] or self.inside['footer'] or self.inside['nav']): + return True + return False + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]): - if tag == 'body': - self.bBody = True - self.bCapture = True - if tag == 'script': - self.bCapture = False - if tag == 'style': - self.bCapture = False + if tag in self.monitored: + self.inside[tag] = True def handle_endtag(self, tag: str): - if tag == 'body': - self.bBody = False - if tag == 'script' or tag == 'style': - if self.bBody: - self.bCapture = True + if tag in self.monitored: + self.inside[tag] = False def handle_data(self, data: str): - if self.bCapture: + if self.do_capture(): self.text += f"{data}\n" def syncup(self): From e12431786b194669f98ea694dbdceafc9bbb6748 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 18:25:50 +0530 Subject: [PATCH 87/89] SimpleChatTC:WebFetch: Cleanup the names and descriptions a bit --- tools/server/public_simplechat/readme.md | 11 +++--- tools/server/public_simplechat/tooljs.mjs | 48 ++++++++++++----------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/tools/server/public_simplechat/readme.md b/tools/server/public_simplechat/readme.md index d90261ed380c9..6c1d55132b624 100644 --- a/tools/server/public_simplechat/readme.md +++ b/tools/server/public_simplechat/readme.md @@ -343,11 +343,12 @@ mechanism. Use of WebWorker helps avoid exposing browser global scope to the gen However any shared web worker scope isnt isolated. Either way always remember to cross check the tool requests and generated responses when using tool calling. -web_url_fetch and family works along with a corresponding simple local web proxy/caching server logic -that can bypass the CORS restrictions applied if trying to directly fetch from the browser js runtime -environment. Depending on the path specified on the proxy server, if urltext, it additionally tries -to convert html content into equivalent text to some extent. May add support for white list of allowed -sites to access or so. +fetch_web_url_raw/text and family works along with a corresponding simple local web proxy/caching +server logic, this helps bypass the CORS restrictions applied if trying to directly fetch from the +browser js runtime environment. Depending on the path specified wrt the proxy server, if urltext +(and not urlraw), it additionally tries to convert html content into equivalent text to some extent +in a simple minded manner by dropping head block as well as all scripts/styles/footers/headers/nav. +May add support for white list of allowed sites to access or so. The simple proxy can be found at * tools/server/public_simplechat/local.tools/simpleproxy.py diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index 5b7979f0ab5a0..da09d9013a06c 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -82,17 +82,17 @@ function message_toolsworker(mev) { } -let weburlfetch_meta = { +let fetchweburlraw_meta = { "type": "function", "function": { - "name": "web_url_fetch", - "description": "Fetch the requested web url through a proxy server in few seconds", + "name": "fetch_web_url_raw", + "description": "Fetch the requested web url through a proxy server and return the got content as is, in few seconds", "parameters": { "type": "object", "properties": { "url":{ "type":"string", - "description":"the url of the page / content to fetch from the internet" + "description":"url of the web page to fetch from the internet" } }, "required": ["url"] @@ -102,17 +102,18 @@ let weburlfetch_meta = { /** - * Implementation of the web url logic. Dumb initial go. + * Implementation of the fetch web url raw logic. Dumb initial go. * Expects a simple minded proxy server to be running locally * * listening on port 3128 * * expecting http requests - * * with a query token named url which gives the actual url to fetch - * ALERT: Accesses a external web proxy/caching server be aware and careful + * * with a query token named url wrt the path urlraw + * which gives the actual url to fetch + * ALERT: Accesses a seperate/external web proxy/caching server, be aware and careful * @param {string} toolcallid * @param {string} toolname * @param {any} obj */ -function weburlfetch_run(toolcallid, toolname, obj) { +function fetchweburlraw_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { let newUrl = `http://127.0.0.1:3128/urlraw?url=${encodeURIComponent(obj.url)}` fetch(newUrl).then(resp=>resp.text()).then(data => { @@ -124,17 +125,17 @@ function weburlfetch_run(toolcallid, toolname, obj) { } -let weburlfetchstrip_meta = { +let fetchweburltext_meta = { "type": "function", "function": { - "name": "web_url_fetch_strip_htmltags_and_some_useless", - "description": "Fetch the requested web url through a proxy server and strip away html tags as well as head, script, styles blocks before returning the remaining body in few seconds", + "name": "fetch_web_url_text", + "description": "Fetch the requested web url through a proxy server and return its text content after stripping away the html tags as well as head, script, style, header, footer, nav blocks, in few seconds", "parameters": { "type": "object", "properties": { "url":{ "type":"string", - "description":"the url of the page that will be fetched from the internet and inturn unwanted stuff stripped from its contents to some extent" + "description":"url of the page that will be fetched from the internet and inturn unwanted stuff stripped from its contents to some extent" } }, "required": ["url"] @@ -144,19 +145,20 @@ let weburlfetchstrip_meta = { /** - * Implementation of the web url logic. Dumb initial go. + * Implementation of the fetch web url text logic. Dumb initial go. * Expects a simple minded proxy server to be running locally * * listening on port 3128 * * expecting http requests - * * with a query token named url which gives the actual url to fetch - * * strips out head as well as any script and style blocks in body + * * with a query token named url wrt urltext path, + * which gives the actual url to fetch + * * strips out head as well as any script, style, header, footer, nav and so blocks in body * before returning remaining body contents. - * ALERT: Accesses a external web proxy/caching server be aware and careful + * ALERT: Accesses a seperate/external web proxy/caching server, be aware and careful * @param {string} toolcallid * @param {string} toolname * @param {any} obj */ -function weburlfetchstrip_run(toolcallid, toolname, obj) { +function fetchweburltext_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { let newUrl = `http://127.0.0.1:3128/urltext?url=${encodeURIComponent(obj.url)}` fetch(newUrl).then(resp=>resp.text()).then(data => { @@ -182,14 +184,14 @@ export let tc_switch = { "meta": calc_meta, "result": "" }, - "web_url_fetch": { - "handler": weburlfetch_run, - "meta": weburlfetch_meta, + "fetch_web_url_raw": { + "handler": fetchweburlraw_run, + "meta": fetchweburlraw_meta, "result": "" }, - "web_url_fetch_strip_htmltags_and_some_useless": { - "handler": weburlfetchstrip_run, - "meta": weburlfetchstrip_meta, + "fetch_web_url_text": { + "handler": fetchweburltext_run, + "meta": fetchweburltext_meta, "result": "" } } From 910995ef1b3640d439a331ce1ec0633b949d7464 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 22:16:30 +0530 Subject: [PATCH 88/89] SimpleChatTC:SimpleProxy: Ensure CORS related headers sent always Add a new send headers common helper and use the same wrt the overridden send_error as well as do_OPTIONS This ensures that if there is any error during proxy opertions, the send_error propogates to the fetch from any browser properly without browser intercepting it with a CORS error --- .../local.tools/simpleproxy.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/server/public_simplechat/local.tools/simpleproxy.py b/tools/server/public_simplechat/local.tools/simpleproxy.py index 78cf6d6767ee2..ce638dfa15cda 100644 --- a/tools/server/public_simplechat/local.tools/simpleproxy.py +++ b/tools/server/public_simplechat/local.tools/simpleproxy.py @@ -23,6 +23,20 @@ class ProxyHandler(http.server.BaseHTTPRequestHandler): + # Common headers to include in responses from this server + def send_headers_common(self): + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS') + self.send_header('Access-Control-Allow-Headers', '*') + self.end_headers() + + # overrides the SendError helper + # so that the common headers mentioned above can get added to them + # else CORS failure will be triggered by the browser on fetch from browser. + def send_error(self, code: int, message: str | None = None, explain: str | None = None) -> None: + self.send_response(code, message) + self.send_headers_common() + # Handle GET requests def do_GET(self): print(f"DBUG:ProxyHandler:GET:{self.path}") @@ -41,10 +55,7 @@ def do_GET(self): def do_OPTIONS(self): print(f"DBUG:ProxyHandler:OPTIONS:{self.path}") self.send_response(200) - self.send_header('Access-Control-Allow-Origin', '*') - self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS') - self.send_header('Access-Control-Allow-Headers', '*') - self.end_headers() + self.send_headers_common() @dataclass(frozen=True) From 4ffed9476f3b7e8e196f8b65b3f0479be33c51c4 Mon Sep 17 00:00:00 2001 From: hanishkvc Date: Fri, 17 Oct 2025 22:46:08 +0530 Subject: [PATCH 89/89] SimpleChatTC:WebFetch:Trap Non Ok status and raise error So that the same error path is used for logical error wrt http req also, without needing a different path for it. Dont forget to return the resp text/json/..., so that the contents are passed along the promise then chain --- tools/server/public_simplechat/tooljs.mjs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/server/public_simplechat/tooljs.mjs b/tools/server/public_simplechat/tooljs.mjs index da09d9013a06c..a60f283e7f4e8 100644 --- a/tools/server/public_simplechat/tooljs.mjs +++ b/tools/server/public_simplechat/tooljs.mjs @@ -116,7 +116,12 @@ let fetchweburlraw_meta = { function fetchweburlraw_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { let newUrl = `http://127.0.0.1:3128/urlraw?url=${encodeURIComponent(obj.url)}` - fetch(newUrl).then(resp=>resp.text()).then(data => { + fetch(newUrl).then(resp => { + if (!resp.ok) { + throw new Error(`${resp.status}:${resp.statusText}`); + } + return resp.text() + }).then(data => { message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) }).catch((err)=>{ message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: `Error:${err}`}})) @@ -161,7 +166,12 @@ let fetchweburltext_meta = { function fetchweburltext_run(toolcallid, toolname, obj) { if (gToolsWorker.onmessage != null) { let newUrl = `http://127.0.0.1:3128/urltext?url=${encodeURIComponent(obj.url)}` - fetch(newUrl).then(resp=>resp.text()).then(data => { + fetch(newUrl).then(resp => { + if (!resp.ok) { + throw new Error(`${resp.status}:${resp.statusText}`); + } + return resp.text() + }).then(data => { message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: data}})) }).catch((err)=>{ message_toolsworker(new MessageEvent('message', {data: {id: toolcallid, name: toolname, data: `Error:${err}`}}))