mirror of
https://github.com/hibobmaster/matrix_chatgpt_bot.git
synced 2024-09-27 15:45:38 +00:00
🔧 Refactor gpt vision trigger method
This commit is contained in:
parent
90a00c9b5a
commit
81543d561b
281
src/bot.py
281
src/bot.py
@ -222,7 +222,7 @@ class Bot:
|
|||||||
self.lc_prog = re.compile(r"\s*!lc\s+(.+)$")
|
self.lc_prog = re.compile(r"\s*!lc\s+(.+)$")
|
||||||
self.lcadmin_prog = re.compile(r"\s*!lcadmin\s+(.+)$")
|
self.lcadmin_prog = re.compile(r"\s*!lcadmin\s+(.+)$")
|
||||||
self.agent_prog = re.compile(r"\s*!agent\s+(.+)$")
|
self.agent_prog = re.compile(r"\s*!agent\s+(.+)$")
|
||||||
self.gpt_vision_prog = re.compile(r"\s*!v\s+(.+)$")
|
self.escape_user_id = re.compile("\s*:\s*(.+)$")
|
||||||
self.help_prog = re.compile(r"\s*!help\s*.*$")
|
self.help_prog = re.compile(r"\s*!help\s*.*$")
|
||||||
self.new_prog = re.compile(r"\s*!new\s+(.+)$")
|
self.new_prog = re.compile(r"\s*!new\s+(.+)$")
|
||||||
|
|
||||||
@ -271,6 +271,7 @@ class Bot:
|
|||||||
content_body = re.sub("\r\n|\r|\n", " ", raw_user_message)
|
content_body = re.sub("\r\n|\r|\n", " ", raw_user_message)
|
||||||
|
|
||||||
# @bot and reply in thread
|
# @bot and reply in thread
|
||||||
|
# gpt vision for element web
|
||||||
if "m.mentions" in event_source["content"]:
|
if "m.mentions" in event_source["content"]:
|
||||||
if "user_ids" in event_source["content"]["m.mentions"]:
|
if "user_ids" in event_source["content"]["m.mentions"]:
|
||||||
# @bot
|
# @bot
|
||||||
@ -278,124 +279,82 @@ class Bot:
|
|||||||
self.user_id
|
self.user_id
|
||||||
in event_source["content"]["m.mentions"]["user_ids"]
|
in event_source["content"]["m.mentions"]["user_ids"]
|
||||||
):
|
):
|
||||||
try:
|
if "m.relates_to" in event.source["content"]:
|
||||||
asyncio.create_task(
|
|
||||||
self.thread_chat(
|
|
||||||
room_id,
|
|
||||||
reply_to_event_id,
|
|
||||||
sender_id=sender_id,
|
|
||||||
thread_root_id=reply_to_event_id,
|
|
||||||
prompt=content_body,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e, exe_info=True)
|
|
||||||
# element android does not have m.mentions, we use another way to make it work
|
|
||||||
elif "formatted_body" in event_source["content"]:
|
|
||||||
if (
|
if (
|
||||||
self.user_id in event_source["content"]["formatted_body"]
|
"m.in_reply_to"
|
||||||
and "m.relates_to" not in event_source["content"]
|
in event_source["content"]["m.relates_to"]
|
||||||
):
|
):
|
||||||
try:
|
in_reply_to_event_id = event_source["content"][
|
||||||
asyncio.create_task(
|
|
||||||
self.thread_chat(
|
|
||||||
room_id,
|
|
||||||
reply_to_event_id,
|
|
||||||
sender_id=sender_id,
|
|
||||||
thread_root_id=reply_to_event_id,
|
|
||||||
prompt=content_body,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e, exe_info=True)
|
|
||||||
|
|
||||||
# thread converstaion
|
|
||||||
if "m.relates_to" in event_source["content"]:
|
|
||||||
if "rel_type" in event_source["content"]["m.relates_to"]:
|
|
||||||
thread_root_id = event_source["content"]["m.relates_to"]["event_id"]
|
|
||||||
# thread is created by @bot
|
|
||||||
if thread_root_id in self.chatbot.conversation:
|
|
||||||
try:
|
|
||||||
asyncio.create_task(
|
|
||||||
self.thread_chat(
|
|
||||||
room_id,
|
|
||||||
reply_to_event_id,
|
|
||||||
sender_id=sender_id,
|
|
||||||
thread_root_id=thread_root_id,
|
|
||||||
prompt=content_body,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e, exe_info=True)
|
|
||||||
|
|
||||||
# common command
|
|
||||||
|
|
||||||
# !gpt command
|
|
||||||
if (
|
|
||||||
self.openai_api_key is not None
|
|
||||||
or self.gpt_api_endpoint != "https://api.openai.com/v1/chat/completions"
|
|
||||||
):
|
|
||||||
m = self.gpt_prog.match(content_body)
|
|
||||||
if m:
|
|
||||||
prompt = m.group(1)
|
|
||||||
try:
|
|
||||||
asyncio.create_task(
|
|
||||||
self.gpt(
|
|
||||||
room_id,
|
|
||||||
reply_to_event_id,
|
|
||||||
prompt,
|
|
||||||
sender_id,
|
|
||||||
raw_user_message,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e, exc_info=True)
|
|
||||||
|
|
||||||
# !chat command
|
|
||||||
if (
|
|
||||||
self.openai_api_key is not None
|
|
||||||
or self.gpt_api_endpoint != "https://api.openai.com/v1/chat/completions"
|
|
||||||
):
|
|
||||||
n = self.chat_prog.match(content_body)
|
|
||||||
if n:
|
|
||||||
prompt = n.group(1)
|
|
||||||
try:
|
|
||||||
asyncio.create_task(
|
|
||||||
self.chat(
|
|
||||||
room_id,
|
|
||||||
reply_to_event_id,
|
|
||||||
prompt,
|
|
||||||
sender_id,
|
|
||||||
raw_user_message,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e, exc_info=True)
|
|
||||||
|
|
||||||
# !v command
|
|
||||||
# not work in E2EE room
|
|
||||||
if self.gpt_vision_api_endpoint and self.gpt_vision_model:
|
|
||||||
if (
|
|
||||||
"m.relates_to" in event.source["content"]
|
|
||||||
and "m.mentions" in event.source["content"]
|
|
||||||
and "user_ids" in event.source["content"]["m.mentions"]
|
|
||||||
):
|
|
||||||
if (
|
|
||||||
self.user_id
|
|
||||||
in event.source["content"]["m.mentions"]["user_ids"]
|
|
||||||
):
|
|
||||||
v = self.gpt_vision_prog.search(content_body)
|
|
||||||
if v:
|
|
||||||
prompt = v.group(1)
|
|
||||||
# Trigger gpt vision flow
|
|
||||||
in_reply_to_event_id = event.source["content"][
|
|
||||||
"m.relates_to"
|
"m.relates_to"
|
||||||
]["m.in_reply_to"]["event_id"]
|
]["m.in_reply_to"]["event_id"]
|
||||||
event_info = await self.get_event(
|
event_info = await self.get_event(
|
||||||
room_id, in_reply_to_event_id
|
room_id, in_reply_to_event_id
|
||||||
)
|
)
|
||||||
msgtype = event_info["content"]["msgtype"]
|
msgtype = event_info["content"]["msgtype"]
|
||||||
|
|
||||||
|
# gpt vision, don't work in E2EE room
|
||||||
if "m.image" == msgtype:
|
if "m.image" == msgtype:
|
||||||
|
v = self.escape_user_id.search(content_body)
|
||||||
|
if v:
|
||||||
|
prompt = v.group(1)
|
||||||
|
image_mimetype = event_info["content"]["info"][
|
||||||
|
"mimetype"
|
||||||
|
]
|
||||||
|
url = event_info["content"]["url"]
|
||||||
|
resp = await self.download_mxc(url)
|
||||||
|
if isinstance(resp, DownloadError):
|
||||||
|
logger.error("Download of image failed")
|
||||||
|
else:
|
||||||
|
b64_image = base64.b64encode(
|
||||||
|
resp.body
|
||||||
|
).decode("utf-8")
|
||||||
|
image_url = f"data:{image_mimetype};base64,{b64_image}"
|
||||||
|
asyncio.create_task(
|
||||||
|
self.gpt_vision_cmd(
|
||||||
|
room_id,
|
||||||
|
reply_to_event_id,
|
||||||
|
prompt,
|
||||||
|
image_url,
|
||||||
|
sender_id,
|
||||||
|
raw_user_message,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
# thread level chatting
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
asyncio.create_task(
|
||||||
|
self.thread_chat(
|
||||||
|
room_id,
|
||||||
|
reply_to_event_id,
|
||||||
|
sender_id=sender_id,
|
||||||
|
thread_root_id=reply_to_event_id,
|
||||||
|
prompt=content_body,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e, exe_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
# gpt vision for element android
|
||||||
|
elif "formatted_body" in event_source["content"]:
|
||||||
|
if (
|
||||||
|
self.user_id in event_source["content"]["formatted_body"]
|
||||||
|
and "m.relates_to" in event_source["content"]
|
||||||
|
):
|
||||||
|
if "m.in_reply_to" in event_source["content"]["m.relates_to"]:
|
||||||
|
in_reply_to_event_id = event_source["content"]["m.relates_to"][
|
||||||
|
"m.in_reply_to"
|
||||||
|
]["event_id"]
|
||||||
|
event_info = await self.get_event(room_id, in_reply_to_event_id)
|
||||||
|
msgtype = event_info["content"]["msgtype"]
|
||||||
|
|
||||||
|
# gpt vision, don't work in E2EE room
|
||||||
|
if "m.image" == msgtype:
|
||||||
|
v = self.escape_user_id.search(content_body)
|
||||||
|
if v:
|
||||||
|
prompt = v.group(1)
|
||||||
image_mimetype = event_info["content"]["info"][
|
image_mimetype = event_info["content"]["info"][
|
||||||
"mimetype"
|
"mimetype"
|
||||||
]
|
]
|
||||||
@ -420,6 +379,97 @@ class Bot:
|
|||||||
raw_user_message,
|
raw_user_message,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
# element android does not have m.mentions, we use another way to make thread level chatting work
|
||||||
|
if "formatted_body" in event_source["content"]:
|
||||||
|
if (
|
||||||
|
self.user_id in event_source["content"]["formatted_body"]
|
||||||
|
and "m.relates_to" not in event_source["content"]
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
asyncio.create_task(
|
||||||
|
self.thread_chat(
|
||||||
|
room_id,
|
||||||
|
reply_to_event_id,
|
||||||
|
sender_id=sender_id,
|
||||||
|
thread_root_id=reply_to_event_id,
|
||||||
|
prompt=content_body,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e, exe_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
# thread converstaion
|
||||||
|
if "m.relates_to" in event_source["content"]:
|
||||||
|
if "rel_type" in event_source["content"]["m.relates_to"]:
|
||||||
|
thread_root_id = event_source["content"]["m.relates_to"]["event_id"]
|
||||||
|
# thread is created by @bot
|
||||||
|
if thread_root_id in self.chatbot.conversation:
|
||||||
|
try:
|
||||||
|
asyncio.create_task(
|
||||||
|
self.thread_chat(
|
||||||
|
room_id,
|
||||||
|
reply_to_event_id,
|
||||||
|
sender_id=sender_id,
|
||||||
|
thread_root_id=thread_root_id,
|
||||||
|
prompt=content_body,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e, exe_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
# common command
|
||||||
|
|
||||||
|
# !gpt command
|
||||||
|
if (
|
||||||
|
self.openai_api_key is not None
|
||||||
|
or self.gpt_api_endpoint != "https://api.openai.com/v1/chat/completions"
|
||||||
|
):
|
||||||
|
m = self.gpt_prog.match(content_body)
|
||||||
|
if m:
|
||||||
|
prompt = m.group(1)
|
||||||
|
try:
|
||||||
|
asyncio.create_task(
|
||||||
|
self.gpt(
|
||||||
|
room_id,
|
||||||
|
reply_to_event_id,
|
||||||
|
prompt,
|
||||||
|
sender_id,
|
||||||
|
raw_user_message,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e, exc_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
# !chat command
|
||||||
|
if (
|
||||||
|
self.openai_api_key is not None
|
||||||
|
or self.gpt_api_endpoint != "https://api.openai.com/v1/chat/completions"
|
||||||
|
):
|
||||||
|
n = self.chat_prog.match(content_body)
|
||||||
|
if n:
|
||||||
|
prompt = n.group(1)
|
||||||
|
try:
|
||||||
|
asyncio.create_task(
|
||||||
|
self.chat(
|
||||||
|
room_id,
|
||||||
|
reply_to_event_id,
|
||||||
|
prompt,
|
||||||
|
sender_id,
|
||||||
|
raw_user_message,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e, exc_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
# lc command
|
# lc command
|
||||||
if self.lc_admin is not None:
|
if self.lc_admin is not None:
|
||||||
@ -1096,6 +1146,8 @@ class Bot:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(e, exc_info=True)
|
logger.error(e, exc_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
# !pic command
|
# !pic command
|
||||||
p = self.pic_prog.search(content_body)
|
p = self.pic_prog.search(content_body)
|
||||||
if p:
|
if p:
|
||||||
@ -1113,6 +1165,8 @@ class Bot:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(e, exc_info=True)
|
logger.error(e, exc_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
# help command
|
# help command
|
||||||
h = self.help_prog.search(content_body)
|
h = self.help_prog.search(content_body)
|
||||||
if h:
|
if h:
|
||||||
@ -1125,6 +1179,8 @@ class Bot:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(e, exc_info=True)
|
logger.error(e, exc_info=True)
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
# message_callback decryption_failure event
|
# message_callback decryption_failure event
|
||||||
async def decryption_failure(self, room: MatrixRoom, event: MegolmEvent) -> None:
|
async def decryption_failure(self, room: MatrixRoom, event: MegolmEvent) -> None:
|
||||||
if not isinstance(event, MegolmEvent):
|
if not isinstance(event, MegolmEvent):
|
||||||
@ -1601,10 +1657,11 @@ class Bot:
|
|||||||
help_info = (
|
help_info = (
|
||||||
"!gpt [prompt], generate a one time response without context conversation\n"
|
"!gpt [prompt], generate a one time response without context conversation\n"
|
||||||
+ "!chat [prompt], chat with context conversation\n"
|
+ "!chat [prompt], chat with context conversation\n"
|
||||||
+ "!pic [prompt], Image generation by DALL·E or LocalAI or stable-diffusion-webui\n" # noqa: E501
|
+ "!pic [prompt], Image generation by DALL-E-3 or LocalAI or stable-diffusion-webui\n" # noqa: E501
|
||||||
+ "!new + chat, start a new conversation \n"
|
+ "!new + chat, start a new conversation \n"
|
||||||
+ "!lc [prompt], chat using langchain api\n"
|
+ "!lc [prompt], chat using langchain api\n"
|
||||||
+ "!v [prompt], gpt_vision\n"
|
+ "quote a image and @bot with prompt, gpt vision function\n"
|
||||||
|
+ "@bot with prompt, create a thread level chatting\n"
|
||||||
+ "!help, help message"
|
+ "!help, help message"
|
||||||
) # noqa: E501
|
) # noqa: E501
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user