From 199cc7f5a423f324ef0c0cb942ed8db92335cde2 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 16:49:51 +0800 Subject: [PATCH 01/15] :art: fix: Update messages list in request.py - Add SystemMessage to messages list - Fix indentation in check_stop function - Update indentation in extract function --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7adbfd252..ec4646baa 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,9 @@ The model adheres to the Openai Schema, other models are not supported. Please a ### 🍔 Login Modes -- `Login via url`: Use `/login token#https://provider.com` to Login. The program posts the token to the interface to +- `Login via url`: Use `/login token$https://provider.com` to Login. The program posts the token to the interface to retrieve configuration information -- `Login`: Use `/login https://api.com/v1#key#model` to login +- `Login`: Use `/login https://api.com/v1$key$model` to login ### 🧀 Plugin Previews From 1a137ccad48d9dc311b887e30de7788667dcd705 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 16:51:53 +0800 Subject: [PATCH 02/15] :art: fix: Update messages list in request.py - Add SystemMessage to messages list - Fix indentation in check_stop function - Update indentation in extract function --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b82b8cc6f..793a522f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,8 +65,8 @@ dependencies = [ ] requires-python = ">=3.9,<3.12" readme = "README.md" -license = { text = "GPL3" } -keywords = ["llmbot", "llmkira", "openai", "chatgpt", "telegram", "bot"] +license = { text = "Apache-2.0", file = "LICENSE" } +keywords = ["llmbot", "llmkira", "openai", "chatgpt", "llm"] classifiers = ["Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9"] [project.urls] From ecbb5b7014ea5bfffc83e01bc4f64833735bef4c Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 16:53:42 +0800 Subject: [PATCH 03/15] :art: fix: Update messages list in request.py - Add SystemMessage to messages list - Fix indentation in check_stop function - Update indentation in extract function --- app/tutorial.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/tutorial.py b/app/tutorial.py index 4b9aea801..48c0b9450 100644 --- a/app/tutorial.py +++ b/app/tutorial.py @@ -9,7 +9,9 @@ import elara from rich.console import Console -elara_client = elara.exe(path=pathlib.Path(__file__).parent / "elara.db", commitdb=True) +elara_client = elara.exe( + path=pathlib.Path(__file__).parent / ".tutorial.db", commitdb=True +) tutorial = [ { From 9ca531016a0712ff07ad2684f5140247ee24fbce Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 17:07:27 +0800 Subject: [PATCH 04/15] add trigger... --- app/sender/discord/__init__.py | 10 ++++++---- app/sender/kook/__init__.py | 9 +++++---- app/sender/slack/__init__.py | 7 ++++--- app/sender/telegram/__init__.py | 9 +++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/app/sender/discord/__init__.py b/app/sender/discord/__init__.py index 2111b8591..149d38acd 100644 --- a/app/sender/discord/__init__.py +++ b/app/sender/discord/__init__.py @@ -401,19 +401,21 @@ async def on_guild_create(event_: hikari.GuildMessageCreateEvent): "discord_hikari:ignore a empty message,do you turn on the MESSAGE_CONTENT setting?" ) return - """ + # 扳机 trigger = await get_trigger_loop( platform_name=__sender__, message=event_.content, - uid=uid_make(__sender__, event_.message.author.id) + uid=uid_make(__sender__, event_.message.author.id), ) if trigger: if trigger.action == "allow": - return await create_task(event_.message, disable_tool_action=trigger.function_enable) + return await create_task( + event_.message, disable_tool_action=trigger.function_enable + ) if trigger.action == "deny": return await event_.message.respond(content=trigger.message) - """ + # 命令 # Bot may cant read message if is_command(text=event_.content, command=f"{BotSetting.prefix}chat"): diff --git a/app/sender/kook/__init__.py b/app/sender/kook/__init__.py index 844b2d37a..ce2e5314e 100644 --- a/app/sender/kook/__init__.py +++ b/app/sender/kook/__init__.py @@ -426,19 +426,20 @@ async def on_guild_create(msg: PublicMessage): # 追溯回复 async def on_dm_create(msg: PrivateMessage): - """ # 扳机 trigger = await get_trigger_loop( platform_name=__sender__, message=msg.content, - uid=uid_make(__sender__, msg.author_id) + uid=uid_make(__sender__, msg.author_id), ) if trigger: if trigger.action == "allow": - return await create_task(msg, disable_tool_action=trigger.function_enable) + return await create_task( + msg, disable_tool_action=trigger.function_enable + ) if trigger.action == "deny": return await msg.reply(content=trigger.message) - """ + if is_command(text=msg.content, command="/task"): return await create_task(msg, disable_tool_action=False) if is_command(text=msg.content, command="/ask"): diff --git a/app/sender/slack/__init__.py b/app/sender/slack/__init__.py index e93e17220..4723d5281 100644 --- a/app/sender/slack/__init__.py +++ b/app/sender/slack/__init__.py @@ -29,6 +29,7 @@ from llmkira.kv_manager.env import EnvManager from llmkira.kv_manager.file import File from llmkira.memory import global_message_runtime +from llmkira.openapi.trigger import get_trigger_loop from llmkira.sdk.tools import ToolRegister from llmkira.task import Task, TaskHeader from llmkira.task.schema import Sign, EventMessage @@ -392,7 +393,7 @@ async def deal_group(event_: SlackMessageEvent): if not await validate_join(event_=event_): return None _text = event_.text - """ + # 扳机 trigger = await get_trigger_loop( platform_name=__sender__, @@ -402,7 +403,7 @@ async def deal_group(event_: SlackMessageEvent): if trigger: if trigger.action == "allow": return await create_task( - event_, funtion_enable=trigger.function_enable + event_, disable_tool_action=trigger.function_enable ) if trigger.action == "deny": return await bot.client.chat_postMessage( @@ -410,7 +411,7 @@ async def deal_group(event_: SlackMessageEvent): text=trigger.message, thread_ts=event_.thread_ts, ) - """ + # 默认指令 if is_command(text=_text, command="!chat"): return await create_task( diff --git a/app/sender/telegram/__init__.py b/app/sender/telegram/__init__.py index 2c9cf137e..f790758c9 100644 --- a/app/sender/telegram/__init__.py +++ b/app/sender/telegram/__init__.py @@ -26,6 +26,7 @@ from llmkira.kv_manager.env import EnvManager from llmkira.kv_manager.file import File, CacheDatabaseError from llmkira.memory import global_message_runtime +from llmkira.openapi.trigger import get_trigger_loop from llmkira.sdk.tools.register import ToolRegister from llmkira.task import Task, TaskHeader from llmkira.task.schema import Sign, EventMessage @@ -357,7 +358,7 @@ async def handle_private_msg(message: types.Message): message.text = message.text if message.text else message.caption if not message.text: return None - """ + trigger = await get_trigger_loop( platform_name=__sender__, message=message.text, @@ -373,7 +374,7 @@ async def handle_private_msg(message: types.Message): return await create_task( message, disable_tool_action=__default_disable_tool_action__ ) - """ + if is_command(text=message.text, command="/task"): return await create_task(message, disable_tool_action=True) if is_command(text=message.text, command="/ask"): @@ -393,7 +394,7 @@ async def handle_group_msg(message: types.Message): message.text = message.text if message.text else message.caption if not message.text: return None - """ + # 扳机 trigger = await get_trigger_loop( platform_name=__sender__, @@ -407,7 +408,7 @@ async def handle_group_msg(message: types.Message): ) if trigger.action == "deny": return await bot.reply_to(message, text=trigger.message) - """ + # 响应 if is_command( text=message.text, From c0b668867247210cc3eea7d30ad98182a5dd3254 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 17:07:33 +0800 Subject: [PATCH 05/15] add trigger... --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b1a02461f..7f6bece65 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,4 @@ config_dir/*.secret/ /.pdm-python /.montydb/ /.snapshot/ +/.tutorial.db From 91d6068acb91629af26c7ba06ce97b17202d158b Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 17:07:40 +0800 Subject: [PATCH 06/15] add trigger... --- llmkira/openapi/trigger/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llmkira/openapi/trigger/__init__.py b/llmkira/openapi/trigger/__init__.py index 2aca16611..420fafb8b 100644 --- a/llmkira/openapi/trigger/__init__.py +++ b/llmkira/openapi/trigger/__init__.py @@ -13,10 +13,14 @@ import inspect from functools import wraps from typing import Literal, List, Callable +from typing import TYPE_CHECKING from loguru import logger from pydantic import BaseModel, Field +if TYPE_CHECKING: + pass + class Trigger(BaseModel): on_func: Callable = None @@ -37,7 +41,7 @@ def update_func(self, func: callable): async def get_trigger_loop(platform_name: str, message: str, uid: str = None): """ receiver builder - message: "RawMessage" + message: Message Content :return 如果有触发，则返回触发的action，否则返回None 代表没有操作 """ sorted(__trigger_phrases__, key=lambda x: x.priority) From 32bfe8c04adbdc0179b5bc6d14e2247d87fc087f Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 17:13:40 +0800 Subject: [PATCH 07/15] add trigger... --- docs/dev_note/chat_start.md | 4 ++++ docs/dev_note/hook.md | 5 +++++ docs/note/app_auth.md | 17 ----------------- docs/note/design_rule.md | 15 --------------- docs/note/release.md | 5 ----- llmkira/openapi/hook/__init__.py | 1 + 6 files changed, 10 insertions(+), 37 deletions(-) create mode 100644 docs/dev_note/chat_start.md create mode 100644 docs/dev_note/hook.md delete mode 100644 docs/note/app_auth.md delete mode 100644 docs/note/design_rule.md delete mode 100644 docs/note/release.md create mode 100644 llmkira/openapi/hook/__init__.py diff --git a/docs/dev_note/chat_start.md b/docs/dev_note/chat_start.md new file mode 100644 index 000000000..b88972c26 --- /dev/null +++ b/docs/dev_note/chat_start.md @@ -0,0 +1,4 @@ + +## 时钟中断 + +一问多答已经实现了。那么更仿真的对话方式是在循环中时钟中断，每次中断时，投入多组消息。这样就可以实现多组消息的对话了。 diff --git a/docs/dev_note/hook.md b/docs/dev_note/hook.md new file mode 100644 index 000000000..2cd89b284 --- /dev/null +++ b/docs/dev_note/hook.md @@ -0,0 +1,5 @@ +## 钩子 + +由于现阶段的 v1 OpenAI API 没有媒体支持，所以我们需要自己实现一个钩子来渲染媒体数据。 + +当消息传入传出的时候，可以先通过钩子处理消息，然后再传递给 ChatGPT。来实现媒体数据的渲染。 diff --git a/docs/note/app_auth.md b/docs/note/app_auth.md deleted file mode 100644 index f6350a4c4..000000000 --- a/docs/note/app_auth.md +++ /dev/null @@ -1,17 +0,0 @@ -# 授权 - -## 任务的 Location - -~~发送方的匿名使得权限管理变得困难，所以我们做如下设计：~~ - -~~1. 添加 sender 参数~~ -~~2. 实行双向验权机制，(发送端验证订阅和权限（如果订阅+余额量就是可发） <- 自定义配置 -> 接收端拉取自定义配置)~~ - -难点： - -~~1. 查询费率系统要完善。 ---> 中间件绑定计费系统~~ -~~2. 绑定机制---> (平台:ID)~~ - -## 服务授权 - -![exp](../SeriveProvider.svg) diff --git a/docs/note/design_rule.md b/docs/note/design_rule.md deleted file mode 100644 index b669b7500..000000000 --- a/docs/note/design_rule.md +++ /dev/null @@ -1,15 +0,0 @@ -# 设计规范 - -- [ ] 函数的父类可以实现一些子类接口，但是默认必须 raise NotImplementedError -- [ ] 函数一次只做一件事，尽量不要有多个 return -- [ ] 函数参数尽量不要超过 3 个，尽量避免使用超过 5 个参数的函数 -- [ ] 类设计的时候，尽量保持单一职责，不要存在过多的方法和属性 -- [ ] 类的命名使用驼峰命名法，首字母大写 -- [ ] 函数的命名使用下划线命名法，全部小写 -- [ ] 类属性如果是一组的，可以后置命名，如：`class User: name_list = []` -- [ ] 类职责尽量内置，类数据处理尽量内置，避免同样的数据处理逻辑出现在多个地方。 -- [ ] 类的属性尽量不要直接暴露，可以通过 `@property` 和 `setter` 方法来实现 -- [ ] 内存占用大的对象尽量使用 `__slots__` 来定义 -- [ ] 统一术语 -- [ ] 内存管理:尽量使用 生成器、迭代器、装饰器、上下文管理器 等方式来减少内存占用 -- [ ] 代码尽量遵循 PEP8 规范 diff --git a/docs/note/release.md b/docs/note/release.md deleted file mode 100644 index 424786420..000000000 --- a/docs/note/release.md +++ /dev/null @@ -1,5 +0,0 @@ -# 备忘录 - -- [ ] 检查OPENAPI版本，添加支持 -- [ ] 更新核心库版本 -- [ ] 调试内置插件必须卸载核心库 diff --git a/llmkira/openapi/hook/__init__.py b/llmkira/openapi/hook/__init__.py new file mode 100644 index 000000000..1ba71fa16 --- /dev/null +++ b/llmkira/openapi/hook/__init__.py @@ -0,0 +1 @@ +# 用于在输出输入的时候，替换输入输出的数据，达成多媒体的目的。附加媒体文件查询，等等。 From 98dcec6e72d95c949558545748bb25a1ff5e1716 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 17:27:41 +0800 Subject: [PATCH 08/15] :hammer: chore(README): update roadmap and add LLM reference support and standalone support for Openai's new Schema. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ec4646baa..beedf156d 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The model adheres to the Openai Schema, other models are not supported. Please a |-----------------------------------| | ![sticker](./docs/chain_chat.gif) | -## 🍔 Roadmap +## 🔨 Roadmap - [x] Removal of legacy code - [x] Deletion of metric system @@ -56,6 +56,9 @@ The model adheres to the Openai Schema, other models are not supported. Please a - [x] Implementation of a more robust plugin system - [x] Project structure simplification - [x] Elimination of the Provider system +- [x] Hook support, access to TTS +- [ ] Add LLM reference support to the plugin environment. (extract && search in text) +- [ ] Add standalone support for Openai's new Schema.(vision) ## 📦 Features From c6ed6926b376a758263ae24115c13e2a06c03b11 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 17:28:20 +0800 Subject: [PATCH 09/15] :hammer: chore(README): update roadmap and add LLM reference support and standalone support for Openai's new Schema. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index beedf156d..f6b1447f2 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,10 @@ The model adheres to the Openai Schema, other models are not supported. Please a - [x] Implementation of a more robust plugin system - [x] Project structure simplification - [x] Elimination of the Provider system -- [x] Hook support, access to TTS +- [x] Hook support. +- [x] Access to TTS. - [ ] Add LLM reference support to the plugin environment. (extract && search in text) -- [ ] Add standalone support for Openai's new Schema.(vision) +- [ ] Add standalone support for Openai's new Schema. (vision) ## 📦 Features From 03289b1c332b0b5cf0c33692c14a8012689875a3 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 19:17:44 +0800 Subject: [PATCH 10/15] note --- docs/dev_note/hook.md | 2 ++ docs/dev_note/time.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 docs/dev_note/time.md diff --git a/docs/dev_note/hook.md b/docs/dev_note/hook.md index 2cd89b284..75a390667 100644 --- a/docs/dev_note/hook.md +++ b/docs/dev_note/hook.md @@ -1,5 +1,7 @@ ## 钩子 +author@sudoskys + 由于现阶段的 v1 OpenAI API 没有媒体支持，所以我们需要自己实现一个钩子来渲染媒体数据。 当消息传入传出的时候，可以先通过钩子处理消息，然后再传递给 ChatGPT。来实现媒体数据的渲染。 diff --git a/docs/dev_note/time.md b/docs/dev_note/time.md new file mode 100644 index 000000000..1cf41cabd --- /dev/null +++ b/docs/dev_note/time.md @@ -0,0 +1,29 @@ +# 聊天分片 + +author@sudoskys + +## 时间池化技术 + +人的谈话有一定的主题。在时间上相互关联。 +因此机器人可以在一段时间后直接舍弃历史消息。通过检索检索一些历史消息，最仿生。 + +难点不在于如何切分时间片，而是如何判断当前聊天与历史聊天的关联性。 + +### 解决方案假设 + +#### 古典搜索+机械匹配 + +- 对每个片段进行主题提取，然后相似性排序。 +- 对每个片段进行关键词抽取，然后相似性排序。 + +#### 古典搜索+LLM自选函数 + +通过 Toolcall 选择匹配的关键词。 + +#### 向量匹配+LLM结构化导出 + +向量数据库，然后通过LLM结构化导出主题。检索主题即可。 + +===== + +不管是哪种方案，都很麻烦，在实时性强的系统里，对性能额外要求。向量匹配只能放在后台处理。 From 1297d3d878143240b515e0b5d2e1adb2eb754e19 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 21:13:05 +0800 Subject: [PATCH 11/15] :sparkles: refactor: update hook system in openapi package - Added new hook system for triggering and running hooks - Created TestHook and TestHook2 classes for testing - Implemented run_hook function to execute hooks based on trigger --- llmkira/openapi/hook/__init__.py | 54 ++++++++++++++++++++++++++++++++ playground/hooks.py | 39 +++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 playground/hooks.py diff --git a/llmkira/openapi/hook/__init__.py b/llmkira/openapi/hook/__init__.py index 1ba71fa16..c655f9efb 100644 --- a/llmkira/openapi/hook/__init__.py +++ b/llmkira/openapi/hook/__init__.py @@ -1 +1,55 @@ # 用于在输出输入的时候，替换输入输出的数据，达成多媒体的目的。附加媒体文件查询，等等。 +from abc import abstractmethod +from enum import Enum +from typing import Type, Set, TypeVar + +from loguru import logger +from pydantic import BaseModel + +T = TypeVar("T") + + +class Trigger(Enum): + SENDER = "sender" + RECEIVER = "receiver" + + +class Hook(BaseModel): + trigger: Trigger = Trigger.RECEIVER + priority: int = 0 + + @abstractmethod + async def trigger_hook(self, *args, **kwargs) -> bool: + return True + + @abstractmethod + async def hook_run(self, *args, **kwargs) -> T: + pass + + +__hook__: Set[Type[Hook]] = set() + + +def resign_hook(): + def decorator(cls: Type[Hook]): + if issubclass(cls, Hook): + logger.success(f"📦 [Resign Hook] {type(cls)}") + __hook__.add(cls) + else: + raise ValueError(f"Resign Hook Error for unknown cls {type(cls)} ") + + return cls + + return decorator + + +async def run_hook(trigger: Trigger, *args: T, **kwargs) -> T: + hook_instances = [hook_cls() for hook_cls in __hook__] + sorted_hook_instances = sorted( + hook_instances, key=lambda x: x.priority, reverse=True + ) + for hook_instance in sorted_hook_instances: + if hook_instance.trigger == trigger: + if await hook_instance.trigger_hook(*args, **kwargs): + args, kwargs = await hook_instance.hook_run(*args, **kwargs) + return args, kwargs diff --git a/playground/hooks.py b/playground/hooks.py new file mode 100644 index 000000000..5fd3d8201 --- /dev/null +++ b/playground/hooks.py @@ -0,0 +1,39 @@ +from llmkira.openapi.hook import resign_hook, Hook, Trigger, run_hook + + +@resign_hook() +class TestHook(Hook): + trigger: Trigger = Trigger.SENDER + + async def trigger_hook(self, *args, **kwargs) -> bool: + print(f"Trigger {args} {kwargs}") + return True + + async def hook_run(self, *args, **kwargs): + print(f"Running {args} {kwargs}") + return args, kwargs + + +@resign_hook() +class TestHook2(Hook): + trigger: Trigger = Trigger.SENDER + priority: int = 1 + + async def trigger_hook(self, *args, **kwargs) -> bool: + print(f"Trigger {args} {kwargs}") + return True + + async def hook_run(self, *args, **kwargs): + print(f"Running {args} {kwargs}") + return args, kwargs + + +async def run_test(): + print("Before running hook") + arg, kwarg = await run_hook(Trigger.SENDER, 2, 3, a=4, b=5) + print(f"After running hook {arg} {kwarg}") + + +import asyncio # noqa + +asyncio.run(run_test()) From a7bc582b83bb68c4fc1a379bd9991e4b369669da Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 21:13:19 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20feat:=20add=20Tim?= =?UTF-8?q?eFeelManager=20to=20manage=20time=20feelings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Implement TimeFeelManager to calculate time differences and manage user feelings. --- app/middleware/llm_task.py | 12 ++++++- llmkira/kv_manager/time.py | 45 ++++++++++++++++++++++++ llmkira/memory/redis_storage/__init__.py | 2 +- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 llmkira/kv_manager/time.py diff --git a/app/middleware/llm_task.py b/app/middleware/llm_task.py index 03a71ab0b..be3cbe5b0 100644 --- a/app/middleware/llm_task.py +++ b/app/middleware/llm_task.py @@ -12,6 +12,7 @@ from app.components.credential import Credential from app.components.user_manager import record_cost from llmkira.kv_manager.instruction import InstructionManager +from llmkira.kv_manager.time import TimeFeelManager from llmkira.kv_manager.tool_call import GLOBAL_TOOLCALL_CACHE_HANDLER from llmkira.memory import global_message_runtime from llmkira.openai.cell import ( @@ -182,7 +183,16 @@ async def request_openai( logger.warning(f"llm_task:Tool is not unique {self.tools}") if isinstance(self.task.task_sign.instruction, str): messages.append(SystemMessage(content=self.task.task_sign.instruction)) - messages.extend(await self.build_message(remember=remember)) + # Feel time leave + time_feel = await TimeFeelManager(self.session_uid).get_leave() + if time_feel: + await self.remember( + message=SystemMessage( + content=f"statu:[After {time_feel} leave, user is back]" + ) + ) + once_build_message = await self.build_message(remember=remember) + messages.extend(once_build_message) # TODO:实现消息时序切片 # 日志 logger.info( diff --git a/llmkira/kv_manager/time.py b/llmkira/kv_manager/time.py new file mode 100644 index 000000000..bcbe591f5 --- /dev/null +++ b/llmkira/kv_manager/time.py @@ -0,0 +1,45 @@ +import datetime + +from llmkira.kv_manager._base import KvManager + + +def hours_difference(timestamp1: int, timestamp2: int) -> float: + """ + Calculate the difference between two timestamps in hours + :param timestamp1: timestamp 1 + :param timestamp2: timestamp 2 + :return: difference in hours + """ + # convert timestamp to datetime object + dt_object1 = datetime.datetime.fromtimestamp(timestamp1) + dt_object2 = datetime.datetime.fromtimestamp(timestamp2) + + # calculate the difference + time_diff = dt_object1 - dt_object2 + + # return the difference in hours + return round(abs(time_diff.total_seconds() / 3600), 2) + + +class TimeFeelManager(KvManager): + def __init__(self, user_id: str): + self.user_id = str(user_id) + + def prefix(self, key: str) -> str: + return f"time_feel:{key}" + + async def get_leave(self) -> str: + now_timestamp = int(datetime.datetime.now().timestamp()) + try: + hours = await self.read_data(self.user_id) + assert isinstance(hours, str) + last_timestamp = int(hours) + except ValueError: + last_timestamp = now_timestamp + finally: + # 存储当前时间戳 + await self.save_data(self.user_id, str(now_timestamp)) + # 计算时间小时差，如果大于 1 小时，返回小时，否则返回None + diff = hours_difference(now_timestamp, last_timestamp) + if diff > 1: + return f"{diff} hours" diff --git a/llmkira/memory/redis_storage/__init__.py b/llmkira/memory/redis_storage/__init__.py index 44e596fe3..a4d885c84 100644 --- a/llmkira/memory/redis_storage/__init__.py +++ b/llmkira/memory/redis_storage/__init__.py @@ -77,5 +77,5 @@ async def write(self, messages: List[BaseModel]): self.clear() self.append(messages) - def clear(self) -> None: + async def clear(self) -> None: self.redis_client.delete(self.key) From 0883fa228ddc54c21685f68a9c18e3a445a45841 Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 21:13:32 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=F0=9F=94=A7=20fix:=20fix=20indentation?= =?UTF-8?q?=20issues=20in=20receiver/telegram/=5F=5Finit=5F=5F.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix indentation in receiver/telegram/__init__.py file. --- app/receiver/discord/__init__.py | 21 ++++++++++------- app/receiver/kook/__init__.py | 21 ++++++++++------- app/receiver/receiver_client.py | 38 ++++++++++++------------------ app/receiver/slack/__init__.py | 25 ++++++++++++-------- app/receiver/telegram/__init__.py | 22 ++++++++++------- app/sender/discord/__init__.py | 23 ++++++++---------- app/sender/kook/__init__.py | 29 +++++++++++------------ app/sender/schema.py | 39 +++++++++++++------------------ app/sender/slack/__init__.py | 22 ++++++++--------- app/sender/telegram/__init__.py | 25 ++++++++++++-------- 10 files changed, 134 insertions(+), 131 deletions(-) diff --git a/app/receiver/discord/__init__.py b/app/receiver/discord/__init__.py index ec3b446e0..7d6bf577f 100644 --- a/app/receiver/discord/__init__.py +++ b/app/receiver/discord/__init__.py @@ -109,13 +109,18 @@ async def reply( """ 模型直转发，Message是Openai的类型 """ - for item in messages: - # raw_message = await self.loop_turn_from_openai(platform_name=__receiver__, message=item, locate=receiver) - event_message = EventMessage.from_openai_message( - message=item, locate=receiver - ) - await self.file_forward(receiver=receiver, file_list=event_message.files) - if not event_message.text: + event_message = [ + EventMessage.from_openai_message(message=item, locate=receiver) + for item in messages + ] + # 转析器 + _, event_message, receiver = await self.hook( + platform_name=__receiver__, messages=event_message, locate=receiver + ) + event_message: list + for event in event_message: + await self.file_forward(receiver=receiver, file_list=event.files) + if not event.text: continue async with self.bot as client: client: hikari.impl.RESTClientImpl @@ -129,7 +134,7 @@ async def reply( ) await client.create_message( channel=int(receiver.thread_id), - content=event_message.text, + content=event.text, reply=_reply, ) return logger.trace("reply message") diff --git a/app/receiver/kook/__init__.py b/app/receiver/kook/__init__.py index e1ad23a7e..e091a36f3 100644 --- a/app/receiver/kook/__init__.py +++ b/app/receiver/kook/__init__.py @@ -118,20 +118,25 @@ async def reply( """ 模型直转发，Message是Openai的类型 """ - for item in messages: - # raw_message = await self.loop_turn_from_openai(platform_name=__receiver__, message=item, locate=receiver) - event_message = EventMessage.from_openai_message( - message=item, locate=receiver - ) - await self.file_forward(receiver=receiver, file_list=event_message.files) - if not event_message.text: + event_message = [ + EventMessage.from_openai_message(message=item, locate=receiver) + for item in messages + ] + # 转析器 + _, event_message, receiver = await self.hook( + platform_name=__receiver__, messages=event_message, locate=receiver + ) + event_message: list + for event in event_message: + await self.file_forward(receiver=receiver, file_list=event.files) + if not event.text: continue await self.send_message( channel_id=receiver.thread_id, user_id=receiver.user_id, dm=receiver.thread_id == receiver.chat_id, message_type=MessageTypes.KMD, - content=event_message.text, + content=event.text, ) return logger.trace("reply message") diff --git a/app/receiver/receiver_client.py b/app/receiver/receiver_client.py index 5f8ef9642..8541704dd 100644 --- a/app/receiver/receiver_client.py +++ b/app/receiver/receiver_client.py @@ -14,7 +14,6 @@ import shortuuid from aio_pika.abc import AbstractIncomingMessage -from deprecated import deprecated from loguru import logger from telebot import formatting @@ -26,7 +25,7 @@ from llmkira.openai.cell import ToolCall, Message, Tool from llmkira.openai.request import OpenAIResult from llmkira.openapi.fuse import get_error_plugin -from llmkira.openapi.transducer import LoopRunner +from llmkira.openapi.hook import run_hook, Trigger from llmkira.sdk.tools import ToolRegister from llmkira.task import Task, TaskHeader from llmkira.task.schema import Location, EventMessage, Router @@ -116,30 +115,23 @@ async def reorganize_tools(task: TaskHeader, error_times_limit: int = 10) -> Lis class BaseSender(object, metaclass=ABCMeta): @staticmethod - @deprecated(reason="use another function") - async def loop_turn_from_openai(platform_name, message, locate): + async def hook(platform_name, messages: List[EventMessage], locate: Location): """ - # FIXME 删除这个函数 - 将 Openai 消息传入 Receiver Loop 进行修饰 - 此过程将忽略掉其他属性。只留下 content + :param platform_name: 平台名称 + :param messages: 消息 + :param locate: 位置 + :return: 平台名称，消息，位置 """ - loop_runner = LoopRunner() - trans_loop = loop_runner.get_receiver_loop(platform_name=platform_name) - _raw_message = EventMessage.from_openai_message(message=message, locate=locate) - await loop_runner.exec_loop( - pipe=trans_loop, - pipe_arg={ - "message": _raw_message, - }, + arg, kwarg = await run_hook( + Trigger.RECEIVER, + platform=platform_name, + messages=messages, + locate=locate, ) - arg: dict = loop_runner.result_pipe_arg - if not arg.get("message"): - logger.error("Message Loop Lose Message") - raw_message: EventMessage = arg.get("message", _raw_message) - assert isinstance( - raw_message, EventMessage - ), f"message type error {type(raw_message)}" - return raw_message + platform_name = kwarg.get("platform", platform_name) + messages = kwarg.get("messages", messages) + locate = kwarg.get("locate", locate) + return platform_name, messages, locate @abstractmethod async def file_forward(self, receiver: Location, file_list: list): diff --git a/app/receiver/slack/__init__.py b/app/receiver/slack/__init__.py index cadd69a97..d48659fa0 100644 --- a/app/receiver/slack/__init__.py +++ b/app/receiver/slack/__init__.py @@ -91,25 +91,30 @@ async def reply( """ 模型直转发，Message是Openai的类型 """ - for item in messages: - # raw_message = await self.loop_turn_from_openai(platform_name=__receiver__, message=item, locate=receiver) - event_message = EventMessage.from_openai_message( - message=item, locate=receiver - ) - await self.file_forward(receiver=receiver, file_list=event_message.files) - if not event_message.text: + event_message = [ + EventMessage.from_openai_message(message=item, locate=receiver) + for item in messages + ] + # 转析器 + _, event_message, receiver = await self.hook( + platform_name=__receiver__, messages=event_message, locate=receiver + ) + event_message: list + for event in event_message: + await self.file_forward(receiver=receiver, file_list=event.files) + if not event.text: continue _message = ( ChatMessageCreator( channel=receiver.chat_id, thread_ts=receiver.message_id ) - .update_content(message_text=event_message.text) - .get_message_payload(message_text=event_message.text) + .update_content(message_text=event.text) + .get_message_payload(message_text=event.text) ) await self.bot.chat_postMessage( channel=receiver.chat_id, thread_ts=receiver.message_id, - text=event_message.text, + text=event.text, ) return logger.trace("reply message") diff --git a/app/receiver/telegram/__init__.py b/app/receiver/telegram/__init__.py index c8a6cc4e0..57e35b916 100644 --- a/app/receiver/telegram/__init__.py +++ b/app/receiver/telegram/__init__.py @@ -118,17 +118,23 @@ async def reply( :param messages: OPENAI Format Message :param reply_to_message: 是否回复消息 """ - for message in messages: - event_message = EventMessage.from_openai_message( - message=message, locate=receiver - ) - await self.file_forward(receiver=receiver, file_list=event_message.files) - if not event_message.text: + event_message = [ + EventMessage.from_openai_message(message=item, locate=receiver) + for item in messages + ] + # 转析器 + _, event_message, receiver = await self.hook( + platform_name=__receiver__, messages=event_message, locate=receiver + ) + event_message: list + for event in event_message: + await self.file_forward(receiver=receiver, file_list=event.files) + if not event.text: continue try: await self.bot.send_message( chat_id=receiver.chat_id, - text=convert(event_message.text), + text=convert(event.text), reply_to_message_id=receiver.message_id if reply_to_message else None, @@ -137,7 +143,7 @@ async def reply( except telebot.apihelper.ApiTelegramException as e: if "message to reply not found" in str(e): await self.bot.send_message( - chat_id=receiver.chat_id, text=convert(event_message.text) + chat_id=receiver.chat_id, text=convert(event.text) ) else: logger.error(f"User {receiver.user_id} send message error") diff --git a/app/sender/discord/__init__.py b/app/sender/discord/__init__.py index 149d38acd..b51ef400c 100644 --- a/app/sender/discord/__init__.py +++ b/app/sender/discord/__init__.py @@ -197,24 +197,21 @@ async def create_task( ) # 任务构建 try: - """ + event_message = await self.transcribe(last_message=message, files=_file) + sign = Sign.from_root( + disable_tool_action=disable_tool_action, + response_snapshot=True, + platform=__sender__, + ) # 转析器 - message, _file = await self.loop_turn_only_message( - platform_name=__sender__, - message=message, - file_list=_file + _, event_message, sign = await self.hook( + platform=__sender__, messages=event_message, sign=sign ) - """ # Reply - messages = await self.transcribe(last_message=message, files=_file) success, logs = await DiscordTask.send_task( task=TaskHeader.from_sender( - messages, - task_sign=Sign.from_root( - disable_tool_action=disable_tool_action, - response_snapshot=True, - platform=__sender__, - ), + event_message, + task_sign=sign, chat_id=str( message.guild_id if message.guild_id else message.channel_id ), diff --git a/app/sender/kook/__init__.py b/app/sender/kook/__init__.py index ce2e5314e..ededb53d6 100644 --- a/app/sender/kook/__init__.py +++ b/app/sender/kook/__init__.py @@ -206,26 +206,23 @@ async def create_task(_event: Message, disable_tool_action: bool = False): ) # 任务构建 try: - """ - # 转析器 - message, _file = await self.loop_turn_only_message( - platform_name=__sender__, - message=message, - file_list=_file - ) - """ - # Reply - messages = await self.transcribe( + event_message = await self.transcribe( last_message=message, files=_file, ) + sign = Sign.from_root( + disable_tool_action=disable_tool_action, + response_snapshot=True, + platform=__sender__, + ) + # 转析器 + _, event_message, sign = await self.hook( + platform=__sender__, messages=event_message, sign=sign + ) + # Reply kook_task = TaskHeader.from_sender( - messages, - task_sign=Sign.from_root( - disable_tool_action=disable_tool_action, - response_snapshot=True, - platform=__sender__, - ), + event_message, + task_sign=sign, message_id=None, chat_id=message.ctx.channel.id, user_id=message.author_id, diff --git a/app/sender/schema.py b/app/sender/schema.py index 733b92429..5d13a6f95 100644 --- a/app/sender/schema.py +++ b/app/sender/schema.py @@ -6,36 +6,29 @@ from abc import abstractmethod, ABC from typing import List -from deprecated import deprecated -from loguru import logger - -from llmkira.openapi.transducer import LoopRunner -from llmkira.task.schema import EventMessage +from llmkira.openapi.hook import run_hook, Trigger +from llmkira.task.schema import EventMessage, Sign class Runner(ABC): @staticmethod - @deprecated(reason="Temporarily Deprecated") - async def loop_turn_only_message(platform_name, message, file_list) -> tuple: + async def hook(platform: str, messages: List[EventMessage], sign: Sign) -> tuple: """ - 将 Openai 消息传入 Loop 进行修饰 - 此过程将忽略掉其他属性。只留下 content + :param platform: 平台 + :param messages: 消息 + :param sign: 签名 + :return: 平台,消息,文件列表 """ - loop_runner = LoopRunner() - trans_loop = loop_runner.get_sender_loop(platform_name=platform_name) - await loop_runner.exec_loop( - pipe=trans_loop, - # sender Parser 约定的参数组合 - pipe_arg={"message": message, "files": file_list}, - ) - _arg = loop_runner.result_pipe_arg - if not _arg.get("message"): - logger.error("Message Loop Lose Message") - new_message, new_file_list = ( - _arg.get("message", message), - _arg.get("files", file_list), + arg, kwarg = await run_hook( + Trigger.SENDER, + platform=platform, + messages=messages, + sign=sign, ) - return new_message, new_file_list + platform = kwarg.get("platform", platform) + messages = kwarg.get("messages", messages) + sign = kwarg.get("sign", sign) + return platform, messages, sign @abstractmethod async def upload(self, *args, **kwargs): diff --git a/app/sender/slack/__init__.py b/app/sender/slack/__init__.py index 4723d5281..3c2bd2ca3 100644 --- a/app/sender/slack/__init__.py +++ b/app/sender/slack/__init__.py @@ -195,22 +195,20 @@ async def create_task( # 任务构建 try: # 转析器 - """ - message, _file = await self.loop_turn_only_message( - platform_name=__sender__, message=message, file_list=_file + event_message = await self.transcribe(last_message=message, files=_file) + sign = Sign.from_root( + disable_tool_action=disable_tool_action, + response_snapshot=True, + platform=__sender__, + ) + _, event_message, sign = await self.hook( + platform=__sender__, messages=event_message, sign=sign ) - """ # Reply success, logs = await SlackTask.send_task( task=TaskHeader.from_sender( - event_messages=await self.transcribe( - last_message=message, files=_file - ), - task_sign=Sign.from_root( - disable_tool_action=disable_tool_action, - response_snapshot=True, - platform=__sender__, - ), + event_messages=event_message, + task_sign=sign, message_id=message.thread_ts, chat_id=message.channel, user_id=message.user, diff --git a/app/sender/telegram/__init__.py b/app/sender/telegram/__init__.py index f790758c9..4a6e7414a 100644 --- a/app/sender/telegram/__init__.py +++ b/app/sender/telegram/__init__.py @@ -184,19 +184,24 @@ async def create_task(message: types.Message, disable_tool_action: bool = True): history_message_list = [] if message.reply_to_message: history_message_list.append(message.reply_to_message) + event_message = await self.transcribe( + last_message=message, + messages=history_message_list, + files=uploaded_file, + ) + sign = Sign.from_root( + disable_tool_action=disable_tool_action, + response_snapshot=True, + platform=__sender__, + ) + _, event_message, sign = await self.hook( + platform=__sender__, messages=event_message, sign=sign + ) # Reply success, logs = await TelegramTask.send_task( task=TaskHeader.from_sender( - task_sign=Sign.from_root( - disable_tool_action=disable_tool_action, - response_snapshot=True, - platform=__sender__, - ), - event_messages=await self.transcribe( - last_message=message, - messages=history_message_list, - files=uploaded_file, - ), + task_sign=sign, + event_messages=event_message, chat_id=str(message.chat.id), user_id=str(message.from_user.id), message_id=str(message.message_id), From 01450bb8e161809ca65d3014e6fe7c569e79a89f Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 23:06:07 +0800 Subject: [PATCH 14/15] =?UTF-8?q?=F0=9F=94=A7=20fix:=20fix=20indentation?= =?UTF-8?q?=20issues=20in=20receiver/telegram/=5F=5Finit=5F=5F.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix indentation in receiver/telegram/__init__.py file. --- app/middleware/llm_task.py | 35 +++++++++-- app/receiver/app.py | 1 + app/receiver/receiver_client.py | 5 +- app/sender/discord/__init__.py | 4 +- app/sender/kook/__init__.py | 4 +- app/sender/slack/__init__.py | 4 +- app/sender/telegram/__init__.py | 4 +- app/sender/util_func.py | 4 +- llmkira/extra/voice/__init__.py | 105 +++++++++++++++++++++++++++++++ llmkira/extra/voice_hook.py | 87 +++++++++++++++++++++++++ llmkira/kv_manager/env.py | 28 ++++++--- llmkira/kv_manager/file.py | 2 +- llmkira/kv_manager/time.py | 5 +- llmkira/openapi/hook/__init__.py | 5 +- pdm.lock | 35 ++++++++++- pyproject.toml | 5 +- 16 files changed, 305 insertions(+), 28 deletions(-) create mode 100644 llmkira/extra/voice/__init__.py create mode 100644 llmkira/extra/voice_hook.py diff --git a/app/middleware/llm_task.py b/app/middleware/llm_task.py index be3cbe5b0..80c80294c 100644 --- a/app/middleware/llm_task.py +++ b/app/middleware/llm_task.py @@ -126,10 +126,9 @@ async def remember(self, *, message: Optional[Message] = None): if message: await self.message_history.append(messages=[message]) - async def build_message(self, remember=True): + async def build_history_messages(self): """ 从任务会话和历史消息中构建消息 - :param remember: 是否写入历史消息 :return: None """ system_prompt = await InstructionManager( @@ -148,6 +147,15 @@ async def build_message(self, remember=True): continue else: message_run.append(msg) + return message_run + + async def build_task_messages(self, remember=True): + """ + 从任务会话和历史消息中构建消息 + :param remember: 是否写入历史消息 + :return: None + """ + message_run = [] # 处理 人类 发送的消息 task_message = self.task.message task_message: List[EventMessage] @@ -172,7 +180,14 @@ async def request_openai( :param disable_tool: 禁用函数 :param credential: 凭证 :return: OpenaiResult 返回结果 - :raise RuntimeError: 无法处理消息 + :raise RuntimeError: # Feel time leave + time_feel = await TimeFeelManager(self.session_uid).get_leave() + if time_feel: + await self.remember( + message=SystemMessage( + content=f"statu:[After {time_feel} leave, user is back]" + ) + ) 无法处理消息 :raise AssertionError: 无法处理消息 :raise OpenaiError: Openai错误 """ @@ -183,16 +198,24 @@ async def request_openai( logger.warning(f"llm_task:Tool is not unique {self.tools}") if isinstance(self.task.task_sign.instruction, str): messages.append(SystemMessage(content=self.task.task_sign.instruction)) - # Feel time leave + # 先读取历史消息才能操作 + message_head = await self.build_history_messages() + messages.extend(message_head) # 历史消息 + # 操作先写入一个状态 time_feel = await TimeFeelManager(self.session_uid).get_leave() if time_feel: + messages.append( + SystemMessage(content=f"statu:[After {time_feel} leave, user is back]") + ) # 插入消息 await self.remember( message=SystemMessage( content=f"statu:[After {time_feel} leave, user is back]" ) ) - once_build_message = await self.build_message(remember=remember) - messages.extend(once_build_message) + # 同步状态到历史消息 + message_body = await self.build_task_messages(remember=remember) + messages.extend(message_body) + # TODO:实现消息时序切片 # 日志 logger.info( diff --git a/app/receiver/app.py b/app/receiver/app.py index 30ec9a4ee..f4e6e6d0e 100644 --- a/app/receiver/app.py +++ b/app/receiver/app.py @@ -51,6 +51,7 @@ async def _main(_func): # 导入插件 load_plugins("llmkira/extra/plugins") load_from_entrypoint("llmkira.extra.plugin") + import llmkira.extra.voice_hook # noqa loaded_message = "\n >>".join(get_entrypoint_plugins()) logger.success( diff --git a/app/receiver/receiver_client.py b/app/receiver/receiver_client.py index 8541704dd..905f2772a 100644 --- a/app/receiver/receiver_client.py +++ b/app/receiver/receiver_client.py @@ -257,7 +257,10 @@ async def _flash( ) return exc except AssertionError as exc: - await self.sender.error(receiver=task.receiver, text=str(exc)) + logger.exception(exc) + await self.sender.error( + receiver=task.receiver, text=f"Assert {str(exc)}" + ) return exc except Exception as exc: logger.exception(exc) diff --git a/app/sender/discord/__init__.py b/app/sender/discord/__init__.py index b51ef400c..23ce8c3cc 100644 --- a/app/sender/discord/__init__.py +++ b/app/sender/discord/__init__.py @@ -371,7 +371,9 @@ async def listen_tool_command(ctx: crescent.Context): async def listen_env_command(ctx: crescent.Context, env_string: str): _manager = EnvManager(user_id=uid_make(__sender__, ctx.user.id)) try: - env_map = _manager.set_env(env_value=env_string, update=True) + env_map = await _manager.set_env( + env_value=env_string, update=True, return_all=True + ) except Exception as e: logger.exception(f"[1202359]env update failed {e}") text = formatting.format_text( diff --git a/app/sender/kook/__init__.py b/app/sender/kook/__init__.py index ededb53d6..69685df6c 100644 --- a/app/sender/kook/__init__.py +++ b/app/sender/kook/__init__.py @@ -374,7 +374,9 @@ async def listen_auth_command(msg: Message, credential: str): async def listen_env_command(msg: Message, env_string: str): _manager = EnvManager(user_id=uid_make(__sender__, msg.author_id)) try: - env_map = _manager.set_env(env_value=env_string, update=True) + env_map = await _manager.set_env( + env_value=env_string, update=True, return_all=True + ) except Exception as e: logger.exception(f"[1202359]env update failed {e}") text = formatting.format_text( diff --git a/app/sender/slack/__init__.py b/app/sender/slack/__init__.py index 3c2bd2ca3..07b4d781e 100644 --- a/app/sender/slack/__init__.py +++ b/app/sender/slack/__init__.py @@ -282,7 +282,9 @@ async def listen_env_command(ack: AsyncAck, respond: AsyncRespond, command): _arg = command.text _manager = EnvManager(user_id=uid_make(__sender__, command.user_id)) try: - env_map = _manager.set_env(env_value=_arg, update=True) + env_map = await _manager.set_env( + env_value=_arg, update=True, return_all=True + ) except Exception as e: logger.exception(f"[213562]env update failed {e}") text = formatting.mbold("🧊 Failed") diff --git a/app/sender/telegram/__init__.py b/app/sender/telegram/__init__.py index 4a6e7414a..8253abb75 100644 --- a/app/sender/telegram/__init__.py +++ b/app/sender/telegram/__init__.py @@ -270,7 +270,9 @@ async def listen_env_command(message: types.Message): return None _manager = EnvManager(user_id=uid_make(__sender__, message.from_user.id)) try: - env_map = _manager.set_env(env_value=_arg, update=True) + env_map = await _manager.set_env( + env_value=_arg, update=True, return_all=True + ) except Exception as e: logger.exception(f"[213562]env update failed {e}") text = formatting.format_text( diff --git a/app/sender/util_func.py b/app/sender/util_func.py index 9840f4131..75757a58b 100644 --- a/app/sender/util_func.py +++ b/app/sender/util_func.py @@ -100,8 +100,10 @@ async def auth_reloader(snapshot_credential: str, platform: str, user_id: str) - snap_shot: SnapData = await global_snapshot_storage.read( user_id=uid_make(platform, user_id) ) - if not snap_shot.data: + if not snap_shot: raise LookupError("Auth Task Not Found") + if not snap_shot.data: + raise LookupError("Auth Task Data Not Found") logger.info(f"Snap Auth:{snapshot_credential},by user {user_id}") for snap in snap_shot.data: if snap.snapshot_credential == snapshot_credential: diff --git a/llmkira/extra/voice/__init__.py b/llmkira/extra/voice/__init__.py new file mode 100644 index 000000000..d2ca8d086 --- /dev/null +++ b/llmkira/extra/voice/__init__.py @@ -0,0 +1,105 @@ +import base64 +import json + +import aiohttp + + +async def request_cn_speech(text): + """ + Call the DuerOS endpoint to generate synthesized voice. + :param text: The text to synthesize + :return: The synthesized voice audio data + """ + base_url = "https://dds.dui.ai/runtime/v1/synthesize" + params = { + "voiceId": "hchunf_ctn", + "text": text, + "speed": "1", + "volume": "50", + "audioType": "wav", + } + + async with aiohttp.ClientSession() as session: + async with session.get(base_url, params=params) as response: + if ( + response.status != 200 + or response.headers.get("Content-Type") != "audio/wav" + ): + return None + return await response.read() + + +def get_audio_bytes_from_data_url(data_url): + """ + Extract audio bytes from a Data URL. + + Parameters: + data_url (str): A Data URL containing base64 encoded audio. + + Returns: + bytes: The extracted audio data if the extraction is successful, otherwise None. + """ + try: + audio_base64 = data_url.split(",")[1] + audio_bytes = base64.b64decode(audio_base64) + return audio_bytes + except Exception: + return None + + +async def request_reecho_speech( + text: str, reecho_api_key: str, voiceId="eb5d7f8c-eea1-483f-b718-1f28d6649339" +): + """ + Call the Reecho endpoint to generate synthesized voice. + :param text: The text to synthesize + :param voiceId: The voiceId to use + :param reecho_api_key: The Reecho API token + :return: The synthesized voice audio data, or None if the request failed + """ + if not reecho_api_key: + return await request_cn_speech(text) + url = "https://v1.reecho.ai/api/tts/simple-generate" + headers = { + "User-Agent": "Apifox/1.0.0 (https://apifox.com)", + "Content-Type": "application/json", + "Authorization": f"Bearer {reecho_api_key}", # Replace {token} with your Reecho API token + } + data = {"voiceId": voiceId, "text": text, "origin_audio": True} + async with aiohttp.ClientSession() as session: + async with session.post( + url, headers=headers, data=json.dumps(data) + ) as response: + if response.status == 200: + response_json = await response.json() + audio_url = response_json["data"].get("audio", None) + audio_bytes = get_audio_bytes_from_data_url(audio_url) + if not audio_bytes: + return await request_cn_speech(text) + return audio_bytes + + +async def request_en_speech(text): + """ + Call the NovelAI endpoint to generate synthesized voice. + :param text: The text to synthesize + :return: The synthesized voice audio data + """ + base_url = "https://api.novelai.net/ai/generate-voice" + headers = {"accept": "*/*"} + params = { + "text": text, + "seed": "Claea", + "voice": "-1", + "opus": "false", + "version": "v2", + } + async with aiohttp.ClientSession() as session: + async with session.get(base_url, params=params, headers=headers) as response: + if response.status != 200: + return None + audio_content_type = response.headers.get("Content-Type") + valid_content_types = ["audio/mpeg", "audio/ogg", "audio/opus"] + if audio_content_type not in valid_content_types: + return None + return await response.read() diff --git a/llmkira/extra/voice_hook.py b/llmkira/extra/voice_hook.py new file mode 100644 index 000000000..e59a1a878 --- /dev/null +++ b/llmkira/extra/voice_hook.py @@ -0,0 +1,87 @@ +from typing import List + +from fast_langdetect import parse_sentence +from loguru import logger + +from llmkira.extra.voice import request_en_speech, request_reecho_speech +from llmkira.kv_manager.env import EnvManager +from llmkira.kv_manager.file import File +from llmkira.openapi.hook import resign_hook, Hook, Trigger +from llmkira.sdk.utils import Ffmpeg +from llmkira.task.schema import EventMessage, Location + + +def check_string(text): + """ + 检查字符串是否符合要求 + :param text: 字符串 + :return: 是否符合要求 + """ + parsed_text = parse_sentence(text) + if not parsed_text: + return False + lang_kinds = [] + for lang in parsed_text: + if lang.get("lang", "RU") not in ["ZH", "EN"]: + return False + lang_kinds.append(lang.get("lang")) + limit = 200 + if len(set(lang_kinds)) == 1: + if lang_kinds[0] in ["EN"]: + limit = 500 + if "\n\n" in text or text.count("\n") > 3 or len(text) > limit or "```" in text: + return False + return True + + +@resign_hook() +class VoiceHook(Hook): + trigger: Trigger = Trigger.RECEIVER + + async def trigger_hook(self, *args, **kwargs) -> bool: + platform_name: str = kwargs.get("platform") # noqa + messages: List[EventMessage] = kwargs.get("messages") + locate: Location = kwargs.get("locate") + for message in messages: + if not check_string(message.text): + return False + if await EnvManager(locate.uid).get_env("VOICE_REPLY_ME", None): + return True + return False + + async def hook_run(self, *args, **kwargs): + logger.debug(f"Voice Hook {args} {kwargs}") + platform_name: str = kwargs.get("platform") # noqa + messages: List[EventMessage] = kwargs.get("messages") + locate: Location = kwargs.get("locate") + for message in messages: + if not check_string(message.text): + return args, kwargs + parsed_text = parse_sentence(message.text) + if not parsed_text: + return args, kwargs + lang_kinds = [] + for lang in parsed_text: + lang_kinds.append(lang.get("lang")) + reecho_api_key = await EnvManager(locate.uid).get_env( + "REECHO_VOICE_KEY", None + ) + if (len(set(lang_kinds)) == 1) and (lang_kinds[0] in ["EN"]): + voice_data = await request_en_speech(message.text) + else: + voice_data = await request_reecho_speech( + message.text, reecho_api_key=reecho_api_key + ) + if voice_data is not None: + ogg_data = Ffmpeg.convert( + input_c="mp3", output_c="ogg", stream_data=voice_data, quiet=True + ) + file = await File.upload_file( + creator=locate.uid, file_name="speech.ogg", file_data=ogg_data + ) + file.caption = message.text + message.text = "" + message.files.append(file) + else: + logger.error(f"Voice Generation Failed:{message.text}") + return args, kwargs diff --git a/llmkira/kv_manager/env.py b/llmkira/kv_manager/env.py index 3463cbd55..1fa6084b8 100644 --- a/llmkira/kv_manager/env.py +++ b/llmkira/kv_manager/env.py @@ -12,14 +12,16 @@ def parse_env_string(env_string) -> Dict[str, str]: env_string = env_string + ";" pattern = r"(\w+)\s*=\s*(.*?)\s*;" matches = re.findall(pattern, env_string, re.MULTILINE) - _env_table = {} + env_table = {} for match in matches: - _key = match[0] - _value = match[1] - _value = _value.strip().strip('"') - _key = _key.upper() - _env_table[_key] = _value - return _env_table + env_key = f"{match[0]}" + env_value = f"{match[1]}" + env_value = env_value.strip().strip('"') + env_key = env_key.upper() + if env_value.lower() == "none": + env_value = None + env_table[env_key] = env_value + return env_table class EnvManager(KvManager): @@ -29,11 +31,19 @@ def __init__(self, user_id: str): def prefix(self, key: str) -> str: return f"env:{key}" + async def get_env(self, env_name, default) -> Optional[str]: + result = await self.read_env() + if not result: + return default + return result.get(env_name, default) + async def read_env(self) -> Optional[dict]: result = await self.read_data(self.user_id) if not result: return None try: + if isinstance(result, dict): + return result return json.loads(result) except Exception as e: logger.error( @@ -42,7 +52,7 @@ async def read_env(self) -> Optional[dict]: return None async def set_env( - self, env_value: Union[dict, str], update=False + self, env_value: Union[dict, str], update=False, return_all=False ) -> Dict[str, str]: current_env = {} if update: @@ -57,4 +67,6 @@ async def set_env( raise ValueError("Env String Should be dict or str") current_env.update(env_map) await self.save_data(self.user_id, json.dumps(current_env)) + if return_all: + return current_env return env_map diff --git a/llmkira/kv_manager/file.py b/llmkira/kv_manager/file.py index 1a96c6ddd..525ba11dd 100644 --- a/llmkira/kv_manager/file.py +++ b/llmkira/kv_manager/file.py @@ -49,7 +49,7 @@ class File(BaseModel): creator: str = Field(description="创建用户") file_name: str = Field(description="文件名") file_key: str = Field(description="文件 Key") - caption: Optional[str] = Field(description="文件描述") + caption: Optional[str] = Field(default=None, description="文件描述") async def download_file(self) -> Optional[bytes]: """ diff --git a/llmkira/kv_manager/time.py b/llmkira/kv_manager/time.py index bcbe591f5..cffacae95 100644 --- a/llmkira/kv_manager/time.py +++ b/llmkira/kv_manager/time.py @@ -32,9 +32,10 @@ async def get_leave(self) -> str: now_timestamp = int(datetime.datetime.now().timestamp()) try: hours = await self.read_data(self.user_id) - assert isinstance(hours, str) + if not hours: + raise LookupError("No data") last_timestamp = int(hours) - except ValueError: + except LookupError: last_timestamp = now_timestamp finally: # 存储当前时间戳 diff --git a/llmkira/openapi/hook/__init__.py b/llmkira/openapi/hook/__init__.py index c655f9efb..c13f6b6f9 100644 --- a/llmkira/openapi/hook/__init__.py +++ b/llmkira/openapi/hook/__init__.py @@ -51,5 +51,8 @@ async def run_hook(trigger: Trigger, *args: T, **kwargs) -> T: for hook_instance in sorted_hook_instances: if hook_instance.trigger == trigger: if await hook_instance.trigger_hook(*args, **kwargs): - args, kwargs = await hook_instance.hook_run(*args, **kwargs) + try: + args, kwargs = await hook_instance.hook_run(*args, **kwargs) + except Exception as ex: + logger.exception(f"Hook Run Error {ex}") return args, kwargs diff --git a/pdm.lock b/pdm.lock index ff52ca334..01051b036 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "bot", "dev"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:c425b7765cfad8ef9d05993e7bb48f321af2602ba1033bb6ad08e6181e7fea12" +content_hash = "sha256:d41c0d0ef9f2c52297c898cc0bf9e1d135608dcdeac1eca80138dda6fb2bfda1" [[package]] name = "aio-pika" @@ -518,7 +518,7 @@ name = "colorlog" version = "6.8.2" requires_python = ">=3.6" summary = "Add colours to the output of Python's logging module." -groups = ["bot"] +groups = ["bot", "default"] dependencies = [ "colorama; sys_platform == \"win32\"", ] @@ -705,6 +705,22 @@ files = [ {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] +[[package]] +name = "fast-langdetect" +version = "0.1.0" +requires_python = "<3.12,>=3.8" +summary = "Quickly detect text language and segment language" +groups = ["default"] +dependencies = [ + "fasttext-wheel>=0.9.2", + "requests>=2.31.0", + "robust-downloader>=0.0.2", +] +files = [ + {file = "fast_langdetect-0.1.0-py3-none-any.whl", hash = "sha256:b886e1f1b15ca1b69027df652f118fa8a26f30424f3d7cffa823ff1a37155643"}, + {file = "fast_langdetect-0.1.0.tar.gz", hash = "sha256:4878152b51dcfbfaace86fda224c12be8cf00adeafd10ffe3022c97674b21d5f"}, +] + [[package]] name = "fasttext-wheel" version = "0.9.2" @@ -2240,6 +2256,21 @@ files = [ {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, ] +[[package]] +name = "robust-downloader" +version = "0.0.2" +summary = "A Simple Robust Downloader written in Python" +groups = ["default"] +dependencies = [ + "colorlog", + "requests", + "tqdm", +] +files = [ + {file = "robust-downloader-0.0.2.tar.gz", hash = "sha256:08c938b96e317abe6b037e34230a91bda9b5d613f009bca4a47664997c61de90"}, + {file = "robust_downloader-0.0.2-py3-none-any.whl", hash = "sha256:8fe08bfb64d714fd1a048a7df6eb7b413eb4e624309a49db2c16fbb80a62869d"}, +] + [[package]] name = "safer" version = "4.12.3" diff --git a/pyproject.toml b/pyproject.toml index 793a522f9..002373b85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llmkira" -version = "1.0.0" +version = "1.0.1" description = "A chain message bot based on OpenAI" authors = [ { name = "sudoskys", email = "me@dianas.cyou" }, @@ -62,10 +62,11 @@ dependencies = [ "apscheduler>=3.10.4", "montydb[lmdb]>=2.5.2", "pymongo>=4.6.3", + "fast-langdetect>=0.1.0", ] requires-python = ">=3.9,<3.12" readme = "README.md" -license = { text = "Apache-2.0", file = "LICENSE" } +license = { text = "Apache-2.0" } keywords = ["llmbot", "llmkira", "openai", "chatgpt", "llm"] classifiers = ["Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9"] From cd7a09df139feb3e8b93a7633f325d1a9623e98d Mon Sep 17 00:00:00 2001 From: sudoskys Date: Wed, 17 Apr 2024 23:10:36 +0800 Subject: [PATCH 15/15] :bulb: chore(README): update hooks section with voice_hook details Hooks control the EventMessage in sender and receiver. For example, we have `voice_hook` in built-in hooks. you can enable it by setting `VOICE_REPLY_ME=true` in `.env`. /env VOICE_REPLY_ME=true /env REECHO_VOICE_KEY= check the source code in `llmkira/extra/voice_hook.py`, learn to write your own hooks. --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index f6b1447f2..4dab60d90 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,19 @@ env - Environment variables of the function Refer to the example plugins in the `plugins` directory and the [🧀 Plugin Development Document](https://llmkira.github.io/Docs/dev/basic) for plugin development documentation. +### Hooks + +Hooks control the EventMessage in sender and receiver. For example, we have `voice_hook` in built-in hooks. + +you can enable it by setting `VOICE_REPLY_ME=true` in `.env`. + +```shell +/env VOICE_REPLY_ME=true +/env REECHO_VOICE_KEY= +``` + +check the source code in `llmkira/extra/voice_hook.py`, learn to write your own hooks. + ## 🧀 Sponsor [![sponsor](./.github/sponsor_ohmygpt.png)](https://www.ohmygpt.com)