Skip to content

Commit

Permalink
🔧 Refactor gpt vision trigger method
Browse files Browse the repository at this point in the history
  • Loading branch information
@hibobmaster
hibobmaster committed Apr 24, 2024
1 parent 90a00c9 commit 81543d5
Showing 1 changed file with 120 additions and 63 deletions.
183 changes: 120 additions & 63 deletions src/bot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ def __init__(
self.lc_prog = re.compile(r"\s*!lc\s+(.+)$")
self.lcadmin_prog = re.compile(r"\s*!lcadmin\s+(.+)$")
self.agent_prog = re.compile(r"\s*!agent\s+(.+)$")
self.gpt_vision_prog = re.compile(r"\s*!v\s+(.+)$")
self.escape_user_id = re.compile("\s*:\s*(.+)$")
self.help_prog = re.compile(r"\s*!help\s*.*$")
self.new_prog = re.compile(r"\s*!new\s+(.+)$")

Expand Down Expand Up @@ -271,27 +271,118 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
content_body = re.sub("\r\n|\r|\n", " ", raw_user_message)

# @bot and reply in thread
# gpt vision for element web
if "m.mentions" in event_source["content"]:
if "user_ids" in event_source["content"]["m.mentions"]:
# @bot
if (
self.user_id
in event_source["content"]["m.mentions"]["user_ids"]
):
try:
asyncio.create_task(
self.thread_chat(
room_id,
reply_to_event_id,
sender_id=sender_id,
thread_root_id=reply_to_event_id,
prompt=content_body,
if "m.relates_to" in event.source["content"]:
if (
"m.in_reply_to"
in event_source["content"]["m.relates_to"]
):
in_reply_to_event_id = event_source["content"][
"m.relates_to"
]["m.in_reply_to"]["event_id"]
event_info = await self.get_event(
room_id, in_reply_to_event_id
)
)
except Exception as e:
logger.error(e, exe_info=True)
# element android does not have m.mentions, we use another way to make it work
msgtype = event_info["content"]["msgtype"]

# gpt vision, don't work in E2EE room
if "m.image" == msgtype:
v = self.escape_user_id.search(content_body)
if v:
prompt = v.group(1)
image_mimetype = event_info["content"]["info"][
"mimetype"
]
url = event_info["content"]["url"]
resp = await self.download_mxc(url)
if isinstance(resp, DownloadError):
logger.error("Download of image failed")
else:
b64_image = base64.b64encode(
resp.body
).decode("utf-8")
image_url = f"data:{image_mimetype};base64,{b64_image}"
asyncio.create_task(
self.gpt_vision_cmd(
room_id,
reply_to_event_id,
prompt,
image_url,
sender_id,
raw_user_message,
)
)
return
# thread level chatting
else:
try:
asyncio.create_task(
self.thread_chat(
room_id,
reply_to_event_id,
sender_id=sender_id,
thread_root_id=reply_to_event_id,
prompt=content_body,
)
)
except Exception as e:
logger.error(e, exe_info=True)

return

# gpt vision for element android
elif "formatted_body" in event_source["content"]:
if (
self.user_id in event_source["content"]["formatted_body"]
and "m.relates_to" in event_source["content"]
):
if "m.in_reply_to" in event_source["content"]["m.relates_to"]:
in_reply_to_event_id = event_source["content"]["m.relates_to"][
"m.in_reply_to"
]["event_id"]
event_info = await self.get_event(room_id, in_reply_to_event_id)
msgtype = event_info["content"]["msgtype"]

# gpt vision, don't work in E2EE room
if "m.image" == msgtype:
v = self.escape_user_id.search(content_body)
if v:
prompt = v.group(1)
image_mimetype = event_info["content"]["info"][
"mimetype"
]
url = event_info["content"]["url"]
resp = await self.download_mxc(url)
if isinstance(resp, DownloadError):
logger.error("Download of image failed")
else:
b64_image = base64.b64encode(resp.body).decode(
"utf-8"
)
image_url = (
f"data:{image_mimetype};base64,{b64_image}"
)
asyncio.create_task(
self.gpt_vision_cmd(
room_id,
reply_to_event_id,
prompt,
image_url,
sender_id,
raw_user_message,
)
)
return

# element android does not have m.mentions, we use another way to make thread level chatting work
if "formatted_body" in event_source["content"]:
if (
self.user_id in event_source["content"]["formatted_body"]
and "m.relates_to" not in event_source["content"]
Expand All @@ -309,6 +400,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
except Exception as e:
logger.error(e, exe_info=True)

return

# thread converstaion
if "m.relates_to" in event_source["content"]:
if "rel_type" in event_source["content"]["m.relates_to"]:
Expand All @@ -328,6 +421,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
except Exception as e:
logger.error(e, exe_info=True)

return

# common command

# !gpt command
Expand All @@ -351,6 +446,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
except Exception as e:
logger.error(e, exc_info=True)

return

# !chat command
if (
self.openai_api_key is not None
Expand All @@ -372,54 +469,7 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
except Exception as e:
logger.error(e, exc_info=True)

# !v command
# not work in E2EE room
if self.gpt_vision_api_endpoint and self.gpt_vision_model:
if (
"m.relates_to" in event.source["content"]
and "m.mentions" in event.source["content"]
and "user_ids" in event.source["content"]["m.mentions"]
):
if (
self.user_id
in event.source["content"]["m.mentions"]["user_ids"]
):
v = self.gpt_vision_prog.search(content_body)
if v:
prompt = v.group(1)
# Trigger gpt vision flow
in_reply_to_event_id = event.source["content"][
"m.relates_to"
]["m.in_reply_to"]["event_id"]
event_info = await self.get_event(
room_id, in_reply_to_event_id
)
msgtype = event_info["content"]["msgtype"]
if "m.image" == msgtype:
image_mimetype = event_info["content"]["info"][
"mimetype"
]
url = event_info["content"]["url"]
resp = await self.download_mxc(url)
if isinstance(resp, DownloadError):
logger.error("Download of image failed")
else:
b64_image = base64.b64encode(resp.body).decode(
"utf-8"
)
image_url = (
f"data:{image_mimetype};base64,{b64_image}"
)
asyncio.create_task(
self.gpt_vision_cmd(
room_id,
reply_to_event_id,
prompt,
image_url,
sender_id,
raw_user_message,
)
)
return

# lc command
if self.lc_admin is not None:
Expand Down Expand Up @@ -1096,6 +1146,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
except Exception as e:
logger.error(e, exc_info=True)

return

# !pic command
p = self.pic_prog.search(content_body)
if p:
Expand All @@ -1113,6 +1165,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
except Exception as e:
logger.error(e, exc_info=True)

return

# help command
h = self.help_prog.search(content_body)
if h:
Expand All @@ -1125,6 +1179,8 @@ async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> No
except Exception as e:
logger.error(e, exc_info=True)

return

# message_callback decryption_failure event
async def decryption_failure(self, room: MatrixRoom, event: MegolmEvent) -> None:
if not isinstance(event, MegolmEvent):
Expand Down Expand Up @@ -1601,10 +1657,11 @@ async def help(self, room_id, reply_to_event_id, sender_id, user_message):
help_info = (
"!gpt [prompt], generate a one time response without context conversation\n"
+ "!chat [prompt], chat with context conversation\n"
+ "!pic [prompt], Image generation by DALL·E or LocalAI or stable-diffusion-webui\n" # noqa: E501
+ "!pic [prompt], Image generation by DALL-E-3 or LocalAI or stable-diffusion-webui\n" # noqa: E501
+ "!new + chat, start a new conversation \n"
+ "!lc [prompt], chat using langchain api\n"
+ "!v [prompt], gpt_vision\n"
+ "quote a image and @bot with prompt, gpt vision function\n"
+ "@bot with prompt, create a thread level chatting\n"
+ "!help, help message"
) # noqa: E501

Expand Down

0 comments on commit 81543d5

Please sign in to comment.