From e6dc16a1b4a4c6ad5c8db32fc0b6562f8d10254e Mon Sep 17 00:00:00 2001
From: Arvin Xu <arvinx@foxmail.com>
Date: Tue, 9 Jul 2024 12:21:27 +0800
Subject: [PATCH] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20groq=20tools=20cal?=
 =?UTF-8?q?ling=20(#3171)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add groq

* add groq

* 📝 docs: add groq tools calling

* add groq
---
 docs/usage/tools-calling/groq.zh-CN.mdx       | 176 +++++++++++++++++-
 src/libs/agent-runtime/utils/debugStream.ts   |   5 +
 .../utils/openaiCompatibleFactory/index.ts    |   5 +-
 3 files changed, 179 insertions(+), 7 deletions(-)

diff --git a/docs/usage/tools-calling/groq.zh-CN.mdx b/docs/usage/tools-calling/groq.zh-CN.mdx
index 061b5ae4dfe8..7649eea9dd9e 100644
--- a/docs/usage/tools-calling/groq.zh-CN.mdx
+++ b/docs/usage/tools-calling/groq.zh-CN.mdx
@@ -2,14 +2,182 @@
 title: Groq Tool Calling
 ---
 
-# Groq Tool Calling
+# Groq 平台模型 Tools Calling 评测（Llama 3/Mistral）
 
-## LLAMA3-70B
+<Callout type={'info'}>
+  由于 Groq 本身不支持 stream，因此 Tools Calling 的调用是普通请求。
+</Callout>
 
-TODO
+Groq 平台的模型 Tools Calling 能力一览：
+
+| 模型         | 支持 Tools Calling | 流式 （Stream） | 并发（Parallel） | 简单指令得分 | 复杂指令 |
+| ------------ | ------------------ | --------------- | ---------------- | ------------ | -------- |
+| LLAMA3 70B   | ✅                 | ❌              | ✅               | 🌟🌟         | 🌟🌟     |
+| LLAMA3 8B    | ✅                 | ❌              | ✅               | 🌟🌟         | 🌟       |
+| Mixtral-8x7B | ✅                 | ❌              | ✅               | ⛔           | 🌟🌟     |
+
+
+## LLAMA3 70B
+
+### 简单调用指令：天气查询
+
+测试指令：指令 ①
+
+<Video src="https://github.com/lobehub/lobe-chat/assets/28616219/a509d3f4-20b5-40a4-a2dc-adf11e1789e8" />
+
+从上述视频中可以看到 LLAMA3 70B 支持并发 Tools Calling，可以同时调用多次天气查询。
+
+<Image
+  alt="LLAMA3 70B 简单指令的 Tools Calling"
+  src="https://github.com/lobehub/lobe-chat/assets/28616219/f9cd0d4d-0e44-4bff-98cc-560f11259a60"
+/>
+
+<details>
+  <summary>Tools Calling 原始输出：</summary>
+
+```yml
+[no stream response] 2024-7-8 15:50:40.166
+
+{"id":"chatcmpl-ec4b6c0b-1078-4f50-a39c-e58b3b1f9c31","object":"chat.completion","created":1720425030,"model":"llama3-70b-8192","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_v89g","type":"function","function":{"name":"realtime-weather____fetchCurrentWeather","arguments":"{\"city\":\"杭州\"}"}},{"id":"call_jxwk","type":"function","function":{"name":"realtime-weather____fetchCurrentWeather","arguments":"{\"city\":\"北京}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":969,"prompt_time":0.224209489,"completion_tokens":68,"completion_time":0.194285714,"total_tokens":1037,"total_time":0.418495203},"system_fingerprint":"fp_87cbfbbc4d","x_groq":{"id":"req_01j28n57x9e78a6bfbn9sdn139"}}
+
+```
+
+</details>
+
+### 复杂调用指令：文生图
+
+测试指令：指令 ②
+
+<Video src="https://github.com/lobehub/lobe-chat/assets/28616219/b790f5d8-e797-4325-a974-436505b16fba" />
+
+<Image
+  alt="LLAMA3 70B 复杂指令的 Tools Calling"
+  src="https://github.com/lobehub/lobe-chat/assets/28616219/082170fa-6d09-4602-8e23-eb678024305f"
+/>
+
+<details>
+<summary>Tools Calling 原始输出：</summary>
+
+```yml
+[no stream response] 2024-7-8 18:0:34.811
+
+{"id":"chatcmpl-e3b59ca9-1172-4ae2-96c7-3d6997a1f8a8","object":"chat.completion","created":1720432834,"model":"llama3-70b-8192","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_azm9","type":"function","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\":[\"A small, fluffy, and playful golden retriever puppy with a white patch on its forehead, sitting on a green grass field with a bright blue sky in the background, photo.\",\"A cute, little, brown and white Dalmatian puppy with a red collar, running around in a park with a sunny day, illustration.\",\"A tiny, grey and white Poodle puppy with a pink ribbon, sitting on a white couch with a few toys surrounding it, watercolor painting.\",\"A sweet, small, black and white Chihuahua puppy with a pink bow, lying on a soft, white blanket with a few stuffed animals nearby, oil painting.\"],\"quality\":\"standard\",\"seeds\":[],\"size\":\"1024x1024\",\"style\":\"vivid\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":2305,"prompt_time":3.027052298,"completion_tokens":246,"completion_time":0.702857143,"total_tokens":2551,"total_time":3.729909441},"system_fingerprint":"fp_7ab5f7e105","x_groq":{"id":"req_01j28wk2q0efvs22qatw7rd0ds"}}
+
+POST /api/chat/groq 200 in 17462ms
+```
+
+</details>
 
 ## LLAMA3-8B
 
-TODO
+### 简单调用指令：天气查询
+
+测试指令：指令 ①
+
+<Video src="https://github.com/lobehub/lobe-chat/assets/28616219/8077f432-b5f6-4e68-a311-2aac1d7ea892" />
+
+从上述视频中可以看到 LLAMA3-8B 对于天气插件可以正常调用，并获得正确的总结结果。但是它并没有完全 follow 我们的描述指令，没有回答「好的」。
+
+<Image
+  alt="LLAMA3-8B 简单指令的 Tools Calling"
+  src="https://github.com/lobehub/lobe-chat/assets/28616219/9d553f4e-0818-4f6c-be9d-70da289dd723"
+/>
+
+<details>
+  <summary>Tools Calling 原始输出：</summary>
+
+```yml
+[no stream response] 2024-7-9 11:33:16.920
+
+{"id":"chatcmpl-f3672d59-e91d-4253-af1b-bfc4e0912085","object":"chat.completion","created":1720495996,"model":"llama3-8b-8192","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_rjtk","type":"function","function":{"name":"realtime-weather____fetchCurrentWeather","arguments":"{\"city\":\"杭州市\"}"}},{"id":"call_7pqh","type":"functi,"function":{"name":"realtime-weather____fetchCurrentWeather","arguments":"{\"city\":\"北京市\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":969,"ppt_time":0.145428625,"completion_tokens":128,"completion_time":0.101364747,"total_tokens":1097,"total_time":0.246793372},"system_fingerprint":"fp_33d61fdfc3","x_groq":{"id":"req_01j2artze1exz82nettf2h9066"}}
+
+POST /api/chat/groq 200 in 1649ms
+```
+
+</details>
+
+### 复杂调用指令：文生图
+
+测试指令：指令 ②
+
+<Video src="https://github.com/lobehub/lobe-chat/assets/28616219/48dcfb0a-ba03-4eae-951e-37f3645eb9c7" />
+
+LLAMA3 8B 在 DallE 的输出场景下，只会输出 1 张图片，而不是像 LLAMA3 70B 一样输出 4 张，意味着在复杂 Tools 指令层面，能力和 GPT 3.5 Turbo 接近，不如 GPT 4。
+
+<Image
+  alt="LLAMA3 8B 复杂指令的 Tools Calling"
+  src="https://github.com/lobehub/lobe-chat/assets/28616219/b0180793-5934-4296-a569-60fbc70e8262"
+/>
+
+<details>
+<summary>Tools Calling 原始输出：</summary>
+
+```yml
+[no stream response] 2024-7-9 11:58:27.40
+
+{"id":"chatcmpl-3c38f4d2-3424-416c-9fb0-0969d2683959","object":"chat.completion","created":1720497506,"model":"llama3-8b-8192","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_k6xj","type":"function","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\":[\"Create a watercolor painting of a small white dog with a pink nose, wearing a red collar and sitting on a green grass. The dog's ears should be floppy and its fur should be curly.\"],\"quality\":\"standard\",\"seeds\":[],\"size\":\"1024x1024\",\"style\":\"natural\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":2282,"prompt_time":0.342335558,"completion_tokens":148,"completion_time":0.118023813,"total_tokens":2430,"total_time":0.460359371},"system_fingerprint":"fp_179b0f92c9","x_groq":{"id":"req_01j2at921tec8aymdq48czcw1y"}}
+
+POST /api/chat/groq 200 in 2517ms
+```
+
+  </details>
 
 ## Mixtral-8x7B
+
+### 简单调用指令：天气查询
+
+测试指令：指令 ①
+
+<Video src="https://github.com/lobehub/lobe-chat/assets/28616219/70b887fa-0e16-4ed2-892e-db52dcecf71d" />
+
+从上述视频中可以看到 Mixtral-8x7B 对于天气插件的查询输出的参数有问题，导致无法正常调用插件。
+
+<Image
+  alt="Mixtral-8x7B 简单指令的 Tools Calling"
+  src="https://github.com/lobehub/lobe-chat/assets/28616219/1b7bf28e-fa31-4838-ae92-9100a4bbb71c"
+/>
+
+<details>
+  <summary>Tools Calling 原始输出：</summary>
+
+```yml
+
+[no stream response] 2024-7-8 22:18:19.682
+
+{"id":"chatcmpl-9f89d669-5642-48be-b5cd-7a29756800c0","object":"chat.completion","created":1720448299,"model":"mixtral-8x7b-32768","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_719t","type":"function","function":{"name":"realtime-weather____fetchCurrentWeather","arguments":"{\"city\":\"Hangzhou,Beijing\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":1262,"prompt_time":0.116684046,"completion_tokens":102,"completion_time":0.163113006,"total_tokens":1364,"total_time":0.279797052},"system_fingerprint":"fp_c5f20b5bb1","x_groq":{"id":"req_01j29bbc8xen2s3thp9qen5bys"}}
+
+POST /api/chat/groq 200 in 4860ms
+```
+
+</details>
+
+### 复杂调用指令：文生图
+
+测试指令：指令 ②
+
+<Video src="https://github.com/lobehub/lobe-chat/assets/28616219/86c4ce08-c634-4a45-8782-c74245eee05b" />
+
+非常意外地，Mixtral 8x7B 在文生图的 Tools Calling 的场景下，居然可以正常出图，而且出图的能力基本和 LLAMA3 70B 持平。
+
+<Image
+  alt="Mixtral-8x7B 复杂指令的 Tools Calling"
+  src="https://github.com/lobehub/lobe-chat/assets/28616219/6e7bc3fa-3bc7-4c23-8f88-f38bb889c594"
+/>
+<Image
+  alt="Mixtral-8x7B 复杂指令的 Tools Calling"
+  src="https://github.com/lobehub/lobe-chat/assets/28616219/815b2792-adaf-4315-9014-f20531a3978b"
+/>
+
+<details>
+  <summary>Tools Calling 原始输出：</summary>
+
+```yml
+[no stream response] 2024-7-9 12:6:38.11
+
+{"id":"chatcmpl-a90069f3-b119-41b9-b8d7-c7f97373529e","object":"chat.completion","created":1720497998,"model":"mixtral-8x7b-32768","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_hw8t","type":"function","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\":[\"A colorful and playful illustration of a friendly small dog, looking directly at the viewer with a wagging tail and bright, expressive eyes. The dog's fur is glossy and well-groomed, with a mix of black, white, and brown colors. The background is a lush, green park with a clear blue sky and a few fluffy clouds.\",\"An oil painting of a small, energetic dog in a sunlit park, chasing a frisbee. The dog is a mix of black and white fur, with a distinct brown spot above its left eye. The park features tall trees, providing dappled shade across the grassy landscape. The frisbee is bright orange and stands out against the natural surroundings.\",\"A realistic watercolor painting of a small, fluffy white dog curled up next to a warm fireplace during a cozy winter evening. The dog's eyes are closed in contentment, and a single red bow is tied around its neck. The background includes a plush armchair, a stack of books, and a softly lit room.\",\"A fun and engaging cartoon of a small dog sitting at a café table, enjoying a cup of coffee and a croissant. The dog has a expressive face and a blue scarf around its neck. The café has a vintage, 1920's style and a red awning, with a bustling city background.\"],\"quality\":\"standard\",\"size\":\"1024x1024\",\"style\":\"vivid\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":2920,"prompt_time":0.228639219,"completion_tokens":465,"completion_time":0.755757988,"total_tokens":3385,"total_time":0.984397207},"system_fingerprint":"fp_c5f20b5bb1","x_groq":{"id":"req_01j2atr155f0nv8rmfk448e2at"}}
+
+POST /api/chat/groq 200 in 6216ms
+
+```
+
+</details>
diff --git a/src/libs/agent-runtime/utils/debugStream.ts b/src/libs/agent-runtime/utils/debugStream.ts
index 661ee8cde980..11a15b46cc19 100644
--- a/src/libs/agent-runtime/utils/debugStream.ts
+++ b/src/libs/agent-runtime/utils/debugStream.ts
@@ -46,3 +46,8 @@ export const debugStream = async (stream: ReadableStream) => {
     }
   }
 };
+
+export const debugResponse = (response: any) => {
+  console.log(`\n[no stream response] ${getTime()}\n`);
+  console.log(JSON.stringify(response) + '\n');
+};
diff --git a/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts b/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts
index da5e94349b01..e3c1d19c2196 100644
--- a/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts
+++ b/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts
@@ -8,7 +8,7 @@ import { LobeRuntimeAI } from '../../BaseAI';
 import { AgentRuntimeErrorType, ILobeAgentRuntimeErrorType } from '../../error';
 import { ChatCompetitionOptions, ChatCompletionErrorPayload, ChatStreamPayload } from '../../types';
 import { AgentRuntimeError } from '../createError';
-import { debugStream } from '../debugStream';
+import { debugResponse, debugStream } from '../debugStream';
 import { desensitizeUrl } from '../desensitizeUrl';
 import { handleOpenAIError } from '../handleOpenAIError';
 import { StreamingResponse } from '../response';
@@ -106,8 +106,7 @@ export const LobeOpenAICompatibleFactory = ({
         }
 
         if (debug?.chatCompletion?.()) {
-          console.log('\n[no stream response]\n');
-          console.log(JSON.stringify(response) + '\n');
+          debugResponse(response);
         }
 
         const stream = this.transformResponseToStream(response as unknown as OpenAI.ChatCompletion);