diff --git a/src/bot.py b/src/bot.py index 953c972..bb0dadd 100644 --- a/src/bot.py +++ b/src/bot.py @@ -222,7 +222,7 @@ def __init__( self.lc_prog = re.compile(r"\s*!lc\s+(.+)$") self.lcadmin_prog = re.compile(r"\s*!lcadmin\s+(.+)$") self.agent_prog = re.compile(r"\s*!agent\s+(.+)$") - self.gpt_vision_prog = re.compile(r"\s*!v\s+(.+)$") + self.escape_user_id = re.compile("\s*:\s*(.+)$") self.help_prog = re.compile(r"\s*!help\s*.*$") self.new_prog = re.compile(r"\s*!new\s+(.+)$") @@ -271,6 +271,7 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No content_body = re.sub("\r\n|\r|\n", " ", raw_user_message) # @bot and reply in thread + # gpt vision for element web if "m.mentions" in event_source["content"]: if "user_ids" in event_source["content"]["m.mentions"]: # @bot @@ -278,20 +279,110 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No self.user_id in event_source["content"]["m.mentions"]["user_ids"] ): - try: - asyncio.create_task( - self.thread_chat( - room_id, - reply_to_event_id, - sender_id=sender_id, - thread_root_id=reply_to_event_id, - prompt=content_body, + if "m.relates_to" in event.source["content"]: + if ( + "m.in_reply_to" + in event_source["content"]["m.relates_to"] + ): + in_reply_to_event_id = event_source["content"][ + "m.relates_to" + ]["m.in_reply_to"]["event_id"] + event_info = await self.get_event( + room_id, in_reply_to_event_id ) - ) - except Exception as e: - logger.error(e, exe_info=True) - # element android does not have m.mentions, we use another way to make it work + msgtype = event_info["content"]["msgtype"] + + # gpt vision, don't work in E2EE room + if "m.image" == msgtype: + v = self.escape_user_id.search(content_body) + if v: + prompt = v.group(1) + image_mimetype = event_info["content"]["info"][ + "mimetype" + ] + url = event_info["content"]["url"] + resp = await self.download_mxc(url) + if isinstance(resp, DownloadError): + logger.error("Download of image failed") + else: + b64_image = base64.b64encode( + resp.body + ).decode("utf-8") + image_url = f"data:{image_mimetype};base64,{b64_image}" + asyncio.create_task( + self.gpt_vision_cmd( + room_id, + reply_to_event_id, + prompt, + image_url, + sender_id, + raw_user_message, + ) + ) + return + # thread level chatting + else: + try: + asyncio.create_task( + self.thread_chat( + room_id, + reply_to_event_id, + sender_id=sender_id, + thread_root_id=reply_to_event_id, + prompt=content_body, + ) + ) + except Exception as e: + logger.error(e, exe_info=True) + + return + + # gpt vision for element android elif "formatted_body" in event_source["content"]: + if ( + self.user_id in event_source["content"]["formatted_body"] + and "m.relates_to" in event_source["content"] + ): + if "m.in_reply_to" in event_source["content"]["m.relates_to"]: + in_reply_to_event_id = event_source["content"]["m.relates_to"][ + "m.in_reply_to" + ]["event_id"] + event_info = await self.get_event(room_id, in_reply_to_event_id) + msgtype = event_info["content"]["msgtype"] + + # gpt vision, don't work in E2EE room + if "m.image" == msgtype: + v = self.escape_user_id.search(content_body) + if v: + prompt = v.group(1) + image_mimetype = event_info["content"]["info"][ + "mimetype" + ] + url = event_info["content"]["url"] + resp = await self.download_mxc(url) + if isinstance(resp, DownloadError): + logger.error("Download of image failed") + else: + b64_image = base64.b64encode(resp.body).decode( + "utf-8" + ) + image_url = ( + f"data:{image_mimetype};base64,{b64_image}" + ) + asyncio.create_task( + self.gpt_vision_cmd( + room_id, + reply_to_event_id, + prompt, + image_url, + sender_id, + raw_user_message, + ) + ) + return + + # element android does not have m.mentions, we use another way to make thread level chatting work + if "formatted_body" in event_source["content"]: if ( self.user_id in event_source["content"]["formatted_body"] and "m.relates_to" not in event_source["content"] @@ -309,6 +400,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No except Exception as e: logger.error(e, exe_info=True) + return + # thread converstaion if "m.relates_to" in event_source["content"]: if "rel_type" in event_source["content"]["m.relates_to"]: @@ -328,6 +421,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No except Exception as e: logger.error(e, exe_info=True) + return + # common command # !gpt command @@ -351,6 +446,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No except Exception as e: logger.error(e, exc_info=True) + return + # !chat command if ( self.openai_api_key is not None @@ -372,54 +469,7 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No except Exception as e: logger.error(e, exc_info=True) - # !v command - # not work in E2EE room - if self.gpt_vision_api_endpoint and self.gpt_vision_model: - if ( - "m.relates_to" in event.source["content"] - and "m.mentions" in event.source["content"] - and "user_ids" in event.source["content"]["m.mentions"] - ): - if ( - self.user_id - in event.source["content"]["m.mentions"]["user_ids"] - ): - v = self.gpt_vision_prog.search(content_body) - if v: - prompt = v.group(1) - # Trigger gpt vision flow - in_reply_to_event_id = event.source["content"][ - "m.relates_to" - ]["m.in_reply_to"]["event_id"] - event_info = await self.get_event( - room_id, in_reply_to_event_id - ) - msgtype = event_info["content"]["msgtype"] - if "m.image" == msgtype: - image_mimetype = event_info["content"]["info"][ - "mimetype" - ] - url = event_info["content"]["url"] - resp = await self.download_mxc(url) - if isinstance(resp, DownloadError): - logger.error("Download of image failed") - else: - b64_image = base64.b64encode(resp.body).decode( - "utf-8" - ) - image_url = ( - f"data:{image_mimetype};base64,{b64_image}" - ) - asyncio.create_task( - self.gpt_vision_cmd( - room_id, - reply_to_event_id, - prompt, - image_url, - sender_id, - raw_user_message, - ) - ) + return # lc command if self.lc_admin is not None: @@ -1096,6 +1146,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No except Exception as e: logger.error(e, exc_info=True) + return + # !pic command p = self.pic_prog.search(content_body) if p: @@ -1113,6 +1165,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No except Exception as e: logger.error(e, exc_info=True) + return + # help command h = self.help_prog.search(content_body) if h: @@ -1125,6 +1179,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No except Exception as e: logger.error(e, exc_info=True) + return + # message_callback decryption_failure event async def decryption_failure(self, room: MatrixRoom, event: MegolmEvent) -> None: if not isinstance(event, MegolmEvent): @@ -1601,10 +1657,11 @@ async def help(self, room_id, reply_to_event_id, sender_id, user_message): help_info = ( "!gpt [prompt], generate a one time response without context conversation\n" + "!chat [prompt], chat with context conversation\n" - + "!pic [prompt], Image generation by DALL·E or LocalAI or stable-diffusion-webui\n" # noqa: E501 + + "!pic [prompt], Image generation by DALL-E-3 or LocalAI or stable-diffusion-webui\n" # noqa: E501 + "!new + chat, start a new conversation \n" + "!lc [prompt], chat using langchain api\n" - + "!v [prompt], gpt_vision\n" + + "quote a image and @bot with prompt, gpt vision function\n" + + "@bot with prompt, create a thread level chatting\n" + "!help, help message" ) # noqa: E501