From 8ee7a481510969b3de999efa53703d3a1be55dff Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Sun, 9 Apr 2023 18:00:34 +0800
Subject: [PATCH 01/14] fix: wechatmp's deadloop when reply is None

---
 channel/chat_channel.py              |  5 +++++
 channel/wechatmp/ServiceAccount.py   |  9 +++++----
 channel/wechatmp/SubscribeAccount.py | 29 +++++++++++++++++-----------
 channel/wechatmp/wechatmp_channel.py | 12 ++++++------
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/channel/chat_channel.py b/channel/chat_channel.py
index 5000ae3de..9ef3d4182 100644
--- a/channel/chat_channel.py
+++ b/channel/chat_channel.py
@@ -233,6 +233,9 @@ def _send(self, reply: Reply, context: Context, retry_cnt = 0):
                 time.sleep(3+3*retry_cnt)
                 self._send(reply, context, retry_cnt+1)
 
+    def _success_callback(self, session_id, **kwargs): # 线程正常结束时的回调函数
+        pass
+
     def _fail_callback(self, session_id, exception, **kwargs): # 线程异常结束时的回调函数
         logger.exception("Worker return exception: {}".format(exception))
 
@@ -242,6 +245,8 @@ def func(worker:Future):
                 worker_exception = worker.exception()
                 if worker_exception:
                     self._fail_callback(session_id, exception = worker_exception, **kwargs)
+                else:
+                    self._success_callback(session_id, **kwargs)
             except CancelledError as e:
                 logger.info("Worker cancelled, session_id = {}".format(session_id))
             except Exception as e:
diff --git a/channel/wechatmp/ServiceAccount.py b/channel/wechatmp/ServiceAccount.py
index db9dff3e0..60d40db6c 100644
--- a/channel/wechatmp/ServiceAccount.py
+++ b/channel/wechatmp/ServiceAccount.py
@@ -16,7 +16,7 @@ def GET(self):
 
     def POST(self):
         # Make sure to return the instance that first created, @singleton will do that. 
-        channel_instance = WechatMPChannel()
+        channel = WechatMPChannel()
         try:
             webData = web.data()
             # logger.debug("[wechatmp] Receive request:\n" + webData.decode("utf-8"))
@@ -27,14 +27,15 @@ def POST(self):
                 message_id = wechatmp_msg.msg_id
 
                 logger.info("[wechatmp] {}:{} Receive post query {} {}: {}".format(web.ctx.env.get('REMOTE_ADDR'), web.ctx.env.get('REMOTE_PORT'), from_user, message_id, message))
-                context = channel_instance._compose_context(ContextType.TEXT, message, isgroup=False, msg=wechatmp_msg)
+                context = channel._compose_context(ContextType.TEXT, message, isgroup=False, msg=wechatmp_msg)
                 if context:
                     # set private openai_api_key
                     # if from_user is not changed in itchat, this can be placed at chat_channel
                     user_data = conf().get_user_data(from_user)
                     context['openai_api_key'] = user_data.get('openai_api_key') # None or user openai_api_key
-                    channel_instance.produce(context)
-                # The reply will be sent by channel_instance.send() in another thread
+                    channel.produce(context)
+                    channel.running.add(from_user)
+                # The reply will be sent by channel.send() in another thread
                 return "success"
 
             elif wechatmp_msg.msg_type == 'event':
diff --git a/channel/wechatmp/SubscribeAccount.py b/channel/wechatmp/SubscribeAccount.py
index 745ef0e46..b1047c394 100644
--- a/channel/wechatmp/SubscribeAccount.py
+++ b/channel/wechatmp/SubscribeAccount.py
@@ -41,7 +41,8 @@ def POST(self):
                     context = channel._compose_context(ContextType.TEXT, message, isgroup=False, msg=wechatmp_msg)
                     logger.debug("[wechatmp] context: {} {}".format(context, wechatmp_msg))
                     if message_id in channel.received_msgs: # received and finished
-                        return 
+                        # no return because of bandwords or other reasons
+                        return "success"
                     if supported and context:
                         # set private openai_api_key
                         # if from_user is not changed in itchat, this can be placed at chat_channel
@@ -71,11 +72,12 @@ def POST(self):
                     channel.query1[cache_key] = False
                     channel.query2[cache_key] = False
                     channel.query3[cache_key] = False
-                # Request again
+                # User request again, and the answer is not ready
                 elif cache_key in channel.running and channel.query1.get(cache_key) == True and channel.query2.get(cache_key) == True and channel.query3.get(cache_key) == True:
                     channel.query1[cache_key] = False  #To improve waiting experience, this can be set to True.
                     channel.query2[cache_key] = False  #To improve waiting experience, this can be set to True.
                     channel.query3[cache_key] = False
+                # User request again, and the answer is ready
                 elif cache_key in channel.cache_dict:
                     # Skip the waiting phase
                     channel.query1[cache_key] = True
@@ -89,7 +91,7 @@ def POST(self):
                     logger.debug("[wechatmp] query1 {}".format(cache_key))
                     channel.query1[cache_key] = True
                     cnt = 0
-                    while cache_key not in channel.cache_dict and cnt < 45:
+                    while cache_key in channel.running and cnt < 45:
                         cnt = cnt + 1
                         time.sleep(0.1)
                     if cnt == 45:
@@ -104,7 +106,7 @@ def POST(self):
                     logger.debug("[wechatmp] query2 {}".format(cache_key))
                     channel.query2[cache_key] = True
                     cnt = 0
-                    while cache_key not in channel.cache_dict and cnt < 45:
+                    while cache_key in channel.running and cnt < 45:
                         cnt = cnt + 1
                         time.sleep(0.1)
                     if cnt == 45:
@@ -119,7 +121,7 @@ def POST(self):
                     logger.debug("[wechatmp] query3 {}".format(cache_key))
                     channel.query3[cache_key] = True
                     cnt = 0
-                    while cache_key not in channel.cache_dict and cnt < 40:
+                    while cache_key in channel.running and cnt < 40:
                         cnt = cnt + 1
                         time.sleep(0.1)
                     if cnt == 40:
@@ -132,12 +134,17 @@ def POST(self):
                     else:
                         pass
 
-                if float(time.time()) - float(query_time) > 4.8:
-                    reply_text = "【正在思考中，回复任意文字尝试获取回复】"
-                    logger.info("[wechatmp] Timeout for {} {}, return".format(from_user, message_id))
-                    replyPost = reply.TextMsg(from_user, to_user, reply_text).send()
-                    return replyPost
-                
+
+                if cache_key not in channel.cache_dict and cache_key not in channel.running:
+                    # no return because of bandwords or other reasons
+                    return "success"
+
+                # if float(time.time()) - float(query_time) > 4.8:
+                #     reply_text = "【正在思考中，回复任意文字尝试获取回复】"
+                #     logger.info("[wechatmp] Timeout for {} {}, return".format(from_user, message_id))
+                #     replyPost = reply.TextMsg(from_user, to_user, reply_text).send()
+                #     return replyPost
+
                 if cache_key in channel.cache_dict:
                     content = channel.cache_dict[cache_key]
                     if len(content.encode('utf8'))<=MAX_UTF8_LEN:
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index 49f45e013..940f9e37b 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -97,8 +97,7 @@ def send(self, reply: Reply, context: Context):
         if self.passive_reply:
             receiver = context["receiver"]
             self.cache_dict[receiver] = reply.content
-            self.running.remove(receiver)
-            logger.debug("[send] reply to {} saved to cache: {}".format(receiver, reply))
+            logger.info("[send] reply to {} saved to cache: {}".format(receiver, reply))
         else:
             receiver = context["receiver"]
             reply_text = reply.content
@@ -115,11 +114,12 @@ def send(self, reply: Reply, context: Context):
             logger.info("[send] Do send to {}: {}".format(receiver, reply_text))
         return
 
+    def _success_callback(self, session_id, **kwargs): # 线程正常结束时的回调函数
+        self.running.remove(session_id)
 
-    def _fail_callback(self, session_id, exception, context, **kwargs):
+    def _fail_callback(self, session_id, exception, context, **kwargs): # 线程异常结束时的回调函数
         logger.exception("[wechatmp] Fail to generation message to user, msgId={}, exception={}".format(context['msg'].msg_id, exception))
-        assert session_id not in self.cache_dict
+        if self.passive_reply:
+            assert session_id not in self.cache_dict
         self.running.remove(session_id)
 
-
-

From f687b2b6f4580b00c9d58e2cf48ff43e59811cd8 Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Sun, 9 Apr 2023 18:32:09 +0800
Subject: [PATCH 02/14] remove _success_callback

---
 channel/chat_channel.py              | 5 -----
 channel/wechatmp/ServiceAccount.py   | 1 -
 channel/wechatmp/wechatmp_channel.py | 5 ++---
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/channel/chat_channel.py b/channel/chat_channel.py
index 9ef3d4182..5000ae3de 100644
--- a/channel/chat_channel.py
+++ b/channel/chat_channel.py
@@ -233,9 +233,6 @@ def _send(self, reply: Reply, context: Context, retry_cnt = 0):
                 time.sleep(3+3*retry_cnt)
                 self._send(reply, context, retry_cnt+1)
 
-    def _success_callback(self, session_id, **kwargs): # 线程正常结束时的回调函数
-        pass
-
     def _fail_callback(self, session_id, exception, **kwargs): # 线程异常结束时的回调函数
         logger.exception("Worker return exception: {}".format(exception))
 
@@ -245,8 +242,6 @@ def func(worker:Future):
                 worker_exception = worker.exception()
                 if worker_exception:
                     self._fail_callback(session_id, exception = worker_exception, **kwargs)
-                else:
-                    self._success_callback(session_id, **kwargs)
             except CancelledError as e:
                 logger.info("Worker cancelled, session_id = {}".format(session_id))
             except Exception as e:
diff --git a/channel/wechatmp/ServiceAccount.py b/channel/wechatmp/ServiceAccount.py
index 60d40db6c..ae535ea0b 100644
--- a/channel/wechatmp/ServiceAccount.py
+++ b/channel/wechatmp/ServiceAccount.py
@@ -34,7 +34,6 @@ def POST(self):
                     user_data = conf().get_user_data(from_user)
                     context['openai_api_key'] = user_data.get('openai_api_key') # None or user openai_api_key
                     channel.produce(context)
-                    channel.running.add(from_user)
                 # The reply will be sent by channel.send() in another thread
                 return "success"
 
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index 940f9e37b..04576060d 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -97,6 +97,7 @@ def send(self, reply: Reply, context: Context):
         if self.passive_reply:
             receiver = context["receiver"]
             self.cache_dict[receiver] = reply.content
+            self.running.remove(receiver)
             logger.info("[send] reply to {} saved to cache: {}".format(receiver, reply))
         else:
             receiver = context["receiver"]
@@ -114,12 +115,10 @@ def send(self, reply: Reply, context: Context):
             logger.info("[send] Do send to {}: {}".format(receiver, reply_text))
         return
 
-    def _success_callback(self, session_id, **kwargs): # 线程正常结束时的回调函数
-        self.running.remove(session_id)
 
     def _fail_callback(self, session_id, exception, context, **kwargs): # 线程异常结束时的回调函数
         logger.exception("[wechatmp] Fail to generation message to user, msgId={}, exception={}".format(context['msg'].msg_id, exception))
         if self.passive_reply:
             assert session_id not in self.cache_dict
-        self.running.remove(session_id)
+            self.running.remove(session_id)
 

From 68ef5aa3aeadaf885c64dde546a5e53a38ca8ccf Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Wed, 12 Apr 2023 05:35:31 +0800
Subject: [PATCH 03/14] ctrl+c exit

---
 app.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app.py b/app.py
index 81dae855b..145b9c7a9 100644
--- a/app.py
+++ b/app.py
@@ -15,6 +15,7 @@ def func(_signo, _stack_frame):
         conf().save_user_datas()
         if callable(old_handler): #  check old_handler
             return old_handler(_signo, _stack_frame)
+        sys.exit(0)
     signal.signal(_signo, func)
 
 def run():

From 8fa3da9ca548fad0165d27aae03e87fd7f1e1a68 Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Wed, 12 Apr 2023 05:41:48 +0800
Subject: [PATCH 04/14] wechatmp: voice input support

---
 channel/wechatmp/README.md           | 2 +-
 channel/wechatmp/ServiceAccount.py   | 2 +-
 channel/wechatmp/SubscribeAccount.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/channel/wechatmp/README.md b/channel/wechatmp/README.md
index 5fb2edad2..225facc62 100644
--- a/channel/wechatmp/README.md
+++ b/channel/wechatmp/README.md
@@ -46,7 +46,7 @@ sudo iptables-save > /etc/iptables/rules.v4
 公共api有访问频率限制（免费账号每分钟最多20次ChatGPT的API调用），这在服务多人的时候会遇到问题。因此这里多加了一个设置私有api_key的功能。目前通过godcmd插件的命令来设置私有api_key。
 
 ## 测试范围
-目前在`RoboStyle`这个公众号上进行了测试(基于[wechatmp-stable分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp-stable),而[master分支](https://github.com/zhayujie/chatgpt-on-wechat)含有最新功能，但是稳定性有待测试)，感兴趣的可以关注并体验。开启了godcmd, Banwords, role, dungeon, finish这五个插件，其他的插件还没有测试。百度的接口暂未测试。语音对话没有测试。图片直接以链接形式回复（没有临时素材上传接口的权限）。
+目前在`RoboStyle`这个公众号上进行了测试（基于[wechatmp分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp)），感兴趣的可以关注并体验。开启了godcmd, Banwords, role, dungeon, finish这五个插件，其他的插件还没有测试。百度的接口暂未测试。语音对话没有测试。图片直接以链接形式回复（没有临时素材上传接口的权限）。
 
 ## TODO
 * 服务号交互完善
diff --git a/channel/wechatmp/ServiceAccount.py b/channel/wechatmp/ServiceAccount.py
index ae535ea0b..eeef66d39 100644
--- a/channel/wechatmp/ServiceAccount.py
+++ b/channel/wechatmp/ServiceAccount.py
@@ -21,7 +21,7 @@ def POST(self):
             webData = web.data()
             # logger.debug("[wechatmp] Receive request:\n" + webData.decode("utf-8"))
             wechatmp_msg = receive.parse_xml(webData)
-            if wechatmp_msg.msg_type == 'text':
+            if wechatmp_msg.msg_type == 'text' or wechatmp_msg.msg_type == 'voice':
                 from_user = wechatmp_msg.from_user_id
                 message = wechatmp_msg.content.decode("utf-8")
                 message_id = wechatmp_msg.msg_id
diff --git a/channel/wechatmp/SubscribeAccount.py b/channel/wechatmp/SubscribeAccount.py
index b1047c394..7de2abd4a 100644
--- a/channel/wechatmp/SubscribeAccount.py
+++ b/channel/wechatmp/SubscribeAccount.py
@@ -22,7 +22,7 @@ def POST(self):
             webData = web.data()
             logger.debug("[wechatmp] Receive request:\n" + webData.decode("utf-8"))
             wechatmp_msg = receive.parse_xml(webData)
-            if wechatmp_msg.msg_type == 'text':
+            if wechatmp_msg.msg_type == 'text' or wechatmp_msg.msg_type == 'voice':
                 from_user = wechatmp_msg.from_user_id
                 to_user = wechatmp_msg.to_user_id
                 message = wechatmp_msg.content.decode("utf-8")

From 7fb4f72b84d6eea1cf0c182b0f30194542b9c28b Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Wed, 12 Apr 2023 05:52:26 +0800
Subject: [PATCH 05/14] update wechatmp README

---
 channel/wechatmp/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/channel/wechatmp/README.md b/channel/wechatmp/README.md
index 225facc62..c69ca92de 100644
--- a/channel/wechatmp/README.md
+++ b/channel/wechatmp/README.md
@@ -45,6 +45,9 @@ sudo iptables-save > /etc/iptables/rules.v4
 ## 私有api_key
 公共api有访问频率限制（免费账号每分钟最多20次ChatGPT的API调用），这在服务多人的时候会遇到问题。因此这里多加了一个设置私有api_key的功能。目前通过godcmd插件的命令来设置私有api_key。
 
+## 语音输入
+利用微信自带的语音识别功能，提供语音输入能力。需要在公众号管理页面的“设置与开发”->“接口权限”页面开启“接收语音识别结果”。
+
 ## 测试范围
 目前在`RoboStyle`这个公众号上进行了测试（基于[wechatmp分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp)），感兴趣的可以关注并体验。开启了godcmd, Banwords, role, dungeon, finish这五个插件，其他的插件还没有测试。百度的接口暂未测试。语音对话没有测试。图片直接以链接形式回复（没有临时素材上传接口的权限）。
 

From 68361cddd256b9d309696fd5a5abb2f11dbc91ad Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Tue, 18 Apr 2023 03:08:18 +0800
Subject: [PATCH 06/14] wechatmp_service: image and voice reply supported

---
 .../{ServiceAccount.py => service_account.py} |   6 +-
 ...bscribeAccount.py => subscribe_account.py} |   0
 channel/wechatmp/wechatmp_channel.py          | 134 ++++++++----------
 channel/wechatmp/wechatmp_client.py           | 125 ++++++++++++++++
 voice/pytts/pytts_voice.py                    |  18 ++-
 5 files changed, 200 insertions(+), 83 deletions(-)
 rename channel/wechatmp/{ServiceAccount.py => service_account.py} (93%)
 rename channel/wechatmp/{SubscribeAccount.py => subscribe_account.py} (100%)
 create mode 100644 channel/wechatmp/wechatmp_client.py

diff --git a/channel/wechatmp/ServiceAccount.py b/channel/wechatmp/service_account.py
similarity index 93%
rename from channel/wechatmp/ServiceAccount.py
rename to channel/wechatmp/service_account.py
index 699581d99..adaa754f6 100644
--- a/channel/wechatmp/ServiceAccount.py
+++ b/channel/wechatmp/service_account.py
@@ -23,7 +23,11 @@ def POST(self):
             webData = web.data()
             # logger.debug("[wechatmp] Receive request:\n" + webData.decode("utf-8"))
             wechatmp_msg = receive.parse_xml(webData)
-            if wechatmp_msg.msg_type == "text" or wechatmp_msg.msg_type == "voice":
+            if (
+                wechatmp_msg.msg_type == "text" 
+                or wechatmp_msg.msg_type == "voice" 
+                # or wechatmp_msg.msg_type == "image"
+            ):
                 from_user = wechatmp_msg.from_user_id
                 message = wechatmp_msg.content.decode("utf-8")
                 message_id = wechatmp_msg.msg_id
diff --git a/channel/wechatmp/SubscribeAccount.py b/channel/wechatmp/subscribe_account.py
similarity index 100%
rename from channel/wechatmp/SubscribeAccount.py
rename to channel/wechatmp/subscribe_account.py
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index ac3c3ac1a..453a1a4a2 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -1,26 +1,25 @@
 # -*- coding: utf-8 -*-
-import json
-import threading
-import time
-
-import requests
 import web
-
+import io
+import imghdr
+import requests
 from bridge.context import *
 from bridge.reply import *
 from channel.chat_channel import ChatChannel
+from channel.wechatmp.wechatmp_client import WechatMPClient
 from channel.wechatmp.common import *
 from common.expired_dict import ExpiredDict
 from common.log import logger
+from common.tmp_dir import TmpDir
 from common.singleton import singleton
 from config import conf
 
 # If using SSL, uncomment the following lines, and modify the certificate path.
-# from cheroot.server import HTTPServer
-# from cheroot.ssl.builtin import BuiltinSSLAdapter
-# HTTPServer.ssl_adapter = BuiltinSSLAdapter(
-#         certificate='/ssl/cert.pem',
-#         private_key='/ssl/cert.key')
+from cheroot.server import HTTPServer
+from cheroot.ssl.builtin import BuiltinSSLAdapter
+HTTPServer.ssl_adapter = BuiltinSSLAdapter(
+        certificate='/ssl/cert.pem',
+        private_key='/ssl/cert.key')
 
 
 @singleton
@@ -30,6 +29,7 @@ def __init__(self, passive_reply=True):
         self.passive_reply = passive_reply
         self.running = set()
         self.received_msgs = ExpiredDict(60 * 60 * 24)
+        self.client = WechatMPClient()
         if self.passive_reply:
             self.NOT_SUPPORT_REPLYTYPE = [ReplyType.IMAGE, ReplyType.VOICE]
             self.cache_dict = dict()
@@ -37,85 +37,65 @@ def __init__(self, passive_reply=True):
             self.query2 = dict()
             self.query3 = dict()
         else:
-            # TODO support image
-            self.NOT_SUPPORT_REPLYTYPE = [ReplyType.IMAGE, ReplyType.VOICE]
-            self.app_id = conf().get("wechatmp_app_id")
-            self.app_secret = conf().get("wechatmp_app_secret")
-            self.access_token = None
-            self.access_token_expires_time = 0
-            self.access_token_lock = threading.Lock()
-            self.get_access_token()
+            self.NOT_SUPPORT_REPLYTYPE = []
+
 
     def startup(self):
         if self.passive_reply:
-            urls = ("/wx", "channel.wechatmp.SubscribeAccount.Query")
+            urls = ("/wx", "channel.wechatmp.subscribe_account.Query")
         else:
-            urls = ("/wx", "channel.wechatmp.ServiceAccount.Query")
+            urls = ("/wx", "channel.wechatmp.service_account.Query")
         app = web.application(urls, globals(), autoreload=False)
         port = conf().get("wechatmp_port", 8080)
         web.httpserver.runsimple(app.wsgifunc(), ("0.0.0.0", port))
 
-    def wechatmp_request(self, method, url, **kwargs):
-        r = requests.request(method=method, url=url, **kwargs)
-        r.raise_for_status()
-        r.encoding = "utf-8"
-        ret = r.json()
-        if "errcode" in ret and ret["errcode"] != 0:
-            raise WeChatAPIException("{}".format(ret))
-        return ret
-
-    def get_access_token(self):
-        # return the access_token
-        if self.access_token:
-            if self.access_token_expires_time - time.time() > 60:
-                return self.access_token
-
-        # Get new access_token
-        # Do not request access_token in parallel! Only the last obtained is valid.
-        if self.access_token_lock.acquire(blocking=False):
-            # Wait for other threads that have previously obtained access_token to complete the request
-            # This happens every 2 hours, so it doesn't affect the experience very much
-            time.sleep(1)
-            self.access_token = None
-            url = "https://api.weixin.qq.com/cgi-bin/token"
-            params = {
-                "grant_type": "client_credential",
-                "appid": self.app_id,
-                "secret": self.app_secret,
-            }
-            data = self.wechatmp_request(method="get", url=url, params=params)
-            self.access_token = data["access_token"]
-            self.access_token_expires_time = int(time.time()) + data["expires_in"]
-            logger.info("[wechatmp] access_token: {}".format(self.access_token))
-            self.access_token_lock.release()
-        else:
-            # Wait for token update
-            while self.access_token_lock.locked():
-                time.sleep(0.1)
-        return self.access_token
 
     def send(self, reply: Reply, context: Context):
+        receiver = context["receiver"]
         if self.passive_reply:
-            receiver = context["receiver"]
             self.cache_dict[receiver] = reply.content
-            logger.info("[send] reply to {} saved to cache: {}".format(receiver, reply))
+            logger.info("[wechatmp] reply cached reply to {}: {}".format(receiver, reply))
         else:
-            receiver = context["receiver"]
-            reply_text = reply.content
-            url = "https://api.weixin.qq.com/cgi-bin/message/custom/send"
-            params = {"access_token": self.get_access_token()}
-            json_data = {
-                "touser": receiver,
-                "msgtype": "text",
-                "text": {"content": reply_text},
-            }
-            self.wechatmp_request(
-                method="post",
-                url=url,
-                params=params,
-                data=json.dumps(json_data, ensure_ascii=False).encode("utf8"),
-            )
-            logger.info("[send] Do send to {}: {}".format(receiver, reply_text))
+            if reply.type == ReplyType.TEXT or reply.type == ReplyType.INFO or reply.type == ReplyType.ERROR:
+                reply_text = reply.content
+                self.client.send_text(receiver, reply_text)
+                logger.info("[wechatmp] Do send to {}: {}".format(receiver, reply_text))
+
+            elif reply.type == ReplyType.VOICE:
+                voice_file_path = reply.content
+                logger.info("[wechatmp] voice file path {}".format(voice_file_path))
+
+                with open(voice_file_path, 'rb') as f:
+                    filename = receiver + "-" + context["msg"].msg_id + ".mp3"
+                    media_id = self.client.upload_media("voice", (filename, f, "audio/mpeg"))
+                    self.client.send_voice(receiver, media_id)
+                    logger.info("[wechatmp] Do send voice to {}".format(receiver))
+
+            elif reply.type == ReplyType.IMAGE_URL:  # 从网络下载图片
+                img_url = reply.content
+                pic_res = requests.get(img_url, stream=True)
+                print(pic_res.headers)
+                image_storage = io.BytesIO()
+                for block in pic_res.iter_content(1024):
+                    image_storage.write(block)
+                image_storage.seek(0)
+                image_type = imghdr.what(image_storage)
+                filename = receiver + "-" + context["msg"].msg_id + "." + image_type
+                content_type = "image/" + image_type
+                # content_type = pic_res.headers.get('content-type')
+                media_id = self.client.upload_media("image", (filename, image_storage, content_type))
+                self.client.send_image(receiver, media_id)
+                logger.info("[wechatmp] sendImage url={}, receiver={}".format(img_url, receiver))
+            elif reply.type == ReplyType.IMAGE:  # 从文件读取图片
+                image_storage = reply.content
+                image_storage.seek(0)
+                image_type = imghdr.what(image_storage)
+                filename = receiver + "-" + context["msg"].msg_id + "." + image_type
+                content_type = "image/" + image_type
+                media_id = self.client.upload_media("image", (filename, image_storage, content_type))
+                self.client.send_image(receiver, media_id)
+                logger.info("[wechatmp] sendImage, receiver={}".format(receiver))
+
         return
 
     def _success_callback(self, session_id, context, **kwargs):  # 线程异常结束时的回调函数
diff --git a/channel/wechatmp/wechatmp_client.py b/channel/wechatmp/wechatmp_client.py
new file mode 100644
index 000000000..7a2cd022c
--- /dev/null
+++ b/channel/wechatmp/wechatmp_client.py
@@ -0,0 +1,125 @@
+import time
+import json
+import requests
+import threading
+from channel.wechatmp.common import *
+from common.log import logger
+from config import conf
+
+
+class WechatMPClient:
+    def __init__(self):
+        self.app_id = conf().get("wechatmp_app_id")
+        self.app_secret = conf().get("wechatmp_app_secret")
+        self.access_token = None
+        self.access_token_expires_time = 0
+        self.access_token_lock = threading.Lock()
+        self.get_access_token()
+
+
+    def wechatmp_request(self, method, url, **kwargs):
+        r = requests.request(method=method, url=url, **kwargs)
+        r.raise_for_status()
+        r.encoding = "utf-8"
+        ret = r.json()
+        if "errcode" in ret and ret["errcode"] != 0:
+            raise WeChatAPIException("{}".format(ret))
+        return ret
+
+    def get_access_token(self):
+        # return the access_token
+        if self.access_token:
+            if self.access_token_expires_time - time.time() > 60:
+                return self.access_token
+
+        # Get new access_token
+        # Do not request access_token in parallel! Only the last obtained is valid.
+        if self.access_token_lock.acquire(blocking=False):
+            # Wait for other threads that have previously obtained access_token to complete the request
+            # This happens every 2 hours, so it doesn't affect the experience very much
+            time.sleep(1)
+            self.access_token = None
+            url = "https://api.weixin.qq.com/cgi-bin/token"
+            params = {
+                "grant_type": "client_credential",
+                "appid": self.app_id,
+                "secret": self.app_secret,
+            }
+            ret = self.wechatmp_request(method="get", url=url, params=params)
+            self.access_token = ret["access_token"]
+            self.access_token_expires_time = int(time.time()) + ret["expires_in"]
+            logger.info("[wechatmp] access_token: {}".format(self.access_token))
+            self.access_token_lock.release()
+        else:
+            # Wait for token update
+            while self.access_token_lock.locked():
+                time.sleep(0.1)
+        return self.access_token
+
+
+    def send_text(self, receiver, reply_text):
+        url = "https://api.weixin.qq.com/cgi-bin/message/custom/send"
+        params = {"access_token": self.get_access_token()}
+        json_data = {
+            "touser": receiver,
+            "msgtype": "text",
+            "text": {"content": reply_text},
+        }
+        self.wechatmp_request(
+            method="post",
+            url=url,
+            params=params,
+            data=json.dumps(json_data, ensure_ascii=False).encode("utf8"),
+        )
+
+
+    def send_voice(self, receiver, media_id):
+        url="https://api.weixin.qq.com/cgi-bin/message/custom/send"
+        params = {"access_token": self.get_access_token()}
+        json_data = {
+            "touser": receiver,
+            "msgtype": "voice",
+            "voice": {
+                "media_id": media_id
+            }
+        }
+        self.wechatmp_request(
+            method="post",
+            url=url,
+            params=params,
+            data=json.dumps(json_data, ensure_ascii=False).encode("utf8"),
+        )
+
+    def send_image(self, receiver, media_id):
+        url="https://api.weixin.qq.com/cgi-bin/message/custom/send"
+        params = {"access_token": self.get_access_token()}
+        json_data = {
+            "touser": receiver,
+            "msgtype": "image",
+            "image": {
+                "media_id": media_id
+            }
+        }
+        self.wechatmp_request(
+            method="post",
+            url=url,
+            params=params,
+            data=json.dumps(json_data, ensure_ascii=False).encode("utf8"),
+        )
+
+
+    def upload_media(self, media_type, media_file):
+        url="https://api.weixin.qq.com/cgi-bin/media/upload"
+        params={
+            "access_token": self.get_access_token(),
+            "type": media_type
+        }
+        files={"media": media_file}
+        logger.info("[wechatmp] media {} uploaded".format(media_file))
+        ret = self.wechatmp_request(
+            method="post",
+            url=url,
+            params=params,
+            files=files
+        )
+        return ret["media_id"]
diff --git a/voice/pytts/pytts_voice.py b/voice/pytts/pytts_voice.py
index 072e28b41..57d0cfb31 100644
--- a/voice/pytts/pytts_voice.py
+++ b/voice/pytts/pytts_voice.py
@@ -10,7 +10,7 @@
 from common.log import logger
 from common.tmp_dir import TmpDir
 from voice.voice import Voice
-
+import os
 
 class PyttsVoice(Voice):
     engine = pyttsx3.init()
@@ -24,15 +24,23 @@ def __init__(self):
             if "Chinese" in voice.name:
                 self.engine.setProperty("voice", voice.id)
 
+        self.engine.setProperty("voice", "zh")
+
     def textToVoice(self, text):
         try:
-            wavFile = TmpDir().path() + "reply-" + str(int(time.time())) + ".wav"
-            self.engine.save_to_file(text, wavFile)
+            mp3FileName = "reply-" + str(int(time.time())) + ".mp3"
+            mp3File = TmpDir().path() + mp3FileName
+            self.engine.save_to_file(text, mp3File)
             self.engine.runAndWait()
+
+            # engine.runAndWait() will return before the file created
+            while mp3FileName not in os.listdir(TmpDir().path()):
+                time.sleep(0.1)
+
             logger.info(
-                "[Pytts] textToVoice text={} voice file name={}".format(text, wavFile)
+                "[Pytts] textToVoice text={} voice file name={}".format(text, mp3File)
             )
-            reply = Reply(ReplyType.VOICE, wavFile)
+            reply = Reply(ReplyType.VOICE, mp3File)
         except Exception as e:
             reply = Reply(ReplyType.ERROR, str(e))
         finally:

From 9a86a679848f259d142fa4a6be9ab73c86ba0100 Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Wed, 19 Apr 2023 01:54:20 +0800
Subject: [PATCH 07/14] update readme

---
 channel/wechatmp/README.md | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/channel/wechatmp/README.md b/channel/wechatmp/README.md
index 69b8037b9..58732faea 100644
--- a/channel/wechatmp/README.md
+++ b/channel/wechatmp/README.md
@@ -22,10 +22,13 @@ pip3 install web.py
 相关的服务器验证代码已经写好，你不需要再添加任何代码。你只需要在本项目根目录的`config.json`中添加
 ```
 "channel_type": "wechatmp",
-"wechatmp_token": "Token",  # 微信公众平台的Token
-"wechatmp_port": 8080,      # 微信公众平台的端口,需要端口转发到80或443
-"wechatmp_app_id": "",      # 微信公众平台的appID，仅服务号需要
-"wechatmp_app_secret": "",  # 微信公众平台的appsecret，仅服务号需要
+"wechatmp_token": "Token",      # 微信公众平台的Token
+"wechatmp_port": 8080,          # 微信公众平台的端口,需要端口转发到80或443
+"wechatmp_app_id": "",          # 微信公众平台的appID，仅服务号需要
+"wechatmp_app_secret": "",      # 微信公众平台的appsecret，仅服务号需要
+"single_chat_prefix": [""],     # 推荐设置，任意对话都可以触发回复，不添加前缀
+"single_chat_reply_prefix": "", # 推荐设置，回复不设置前缀
+"plugin_trigger_prefix": "&",   # 推荐设置，在手机微信客户端中，$%^等符号与中文连在一起时会自动显示一段较大的间隔，用户体验不好。请不要使用管理员指令前缀"#"，这会造成未知问题。
 ```
 然后运行`python3 app.py`启动web服务器。这里会默认监听8080端口，但是微信公众号的服务器配置只支持80/443端口，有两种方法来解决这个问题。第一个是推荐的方法，使用端口转发命令将80端口转发到8080端口（443同理，注意需要支持SSL，也就是https的访问，在`wechatmp_channel.py`需要修改相应的证书路径）：
 ```

From df4c1f04010f5a263721d8b509bc92bc9be1d2fe Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Wed, 19 Apr 2023 01:56:25 +0800
Subject: [PATCH 08/14] wechatmp: logic simplification

---
 channel/wechatmp/subscribe_account.py | 187 +++++++++-----------------
 channel/wechatmp/wechatmp_channel.py  |  30 ++---
 2 files changed, 82 insertions(+), 135 deletions(-)

diff --git a/channel/wechatmp/subscribe_account.py b/channel/wechatmp/subscribe_account.py
index 8eeedb4bf..fcdb2d8cb 100644
--- a/channel/wechatmp/subscribe_account.py
+++ b/channel/wechatmp/subscribe_account.py
@@ -17,12 +17,11 @@ def GET(self):
         return verify_server(web.input())
 
     def POST(self):
-        # Make sure to return the instance that first created, @singleton will do that.
-        channel = WechatMPChannel()
         try:
-            query_time = time.time()
+            request_time = time.time()
+            channel = WechatMPChannel()
             webData = web.data()
-            logger.debug("[wechatmp] Receive request:\n" + webData.decode("utf-8"))
+            logger.debug("[wechatmp] Receive post data:\n" + webData.decode("utf-8"))
             wechatmp_msg = receive.parse_xml(webData)
             if wechatmp_msg.msg_type == "text" or wechatmp_msg.msg_type == "voice":
                 from_user = wechatmp_msg.from_user_id
@@ -30,48 +29,34 @@ def POST(self):
                 message = wechatmp_msg.content.decode("utf-8")
                 message_id = wechatmp_msg.msg_id
 
-                logger.info(
-                    "[wechatmp] {}:{} Receive post query {} {}: {}".format(
-                        web.ctx.env.get("REMOTE_ADDR"),
-                        web.ctx.env.get("REMOTE_PORT"),
-                        from_user,
-                        message_id,
-                        message,
-                    )
-                )
                 supported = True
                 if "【收到不支持的消息类型，暂无法显示】" in message:
                     supported = False  # not supported, used to refresh
-                cache_key = from_user
 
-                reply_text = ""
                 # New request
                 if (
-                    cache_key not in channel.cache_dict
-                    and cache_key not in channel.running
+                    from_user not in channel.cache_dict
+                    and from_user not in channel.running
+                    or message.startswith("#") 
+                    and message_id not in channel.request_cnt # insert the godcmd
                 ):
-                    # The first query begin, reset the cache
+                    # The first query begin
                     context = channel._compose_context(
                         ContextType.TEXT, message, isgroup=False, msg=wechatmp_msg
                     )
                     logger.debug(
                         "[wechatmp] context: {} {}".format(context, wechatmp_msg)
                     )
-                    if message_id in channel.received_msgs:  # received and finished
-                        # no return because of bandwords or other reasons
-                        return "success"
+
                     if supported and context:
                         # set private openai_api_key
                         # if from_user is not changed in itchat, this can be placed at chat_channel
                         user_data = conf().get_user_data(from_user)
-                        context["openai_api_key"] = user_data.get(
-                            "openai_api_key"
-                        )  # None or user openai_api_key
-                        channel.received_msgs[message_id] = wechatmp_msg
-                        channel.running.add(cache_key)
+                        context["openai_api_key"] = user_data.get("openai_api_key")
+                        channel.running.add(from_user)
                         channel.produce(context)
                     else:
-                        trigger_prefix = conf().get("single_chat_prefix", [""])[0]
+                        trigger_prefix = conf().get("single_chat_prefix", [""])
                         if trigger_prefix or not supported:
                             if trigger_prefix:
                                 content = textwrap.dedent(
@@ -92,108 +77,67 @@ def POST(self):
                                 """\
                                 未知错误，请稍后再试"""
                             )
-                        replyMsg = reply.TextMsg(
-                            wechatmp_msg.from_user_id, wechatmp_msg.to_user_id, content
-                        )
-                        return replyMsg.send()
-                    channel.query1[cache_key] = False
-                    channel.query2[cache_key] = False
-                    channel.query3[cache_key] = False
-                # User request again, and the answer is not ready
-                elif (
-                    cache_key in channel.running
-                    and channel.query1.get(cache_key) == True
-                    and channel.query2.get(cache_key) == True
-                    and channel.query3.get(cache_key) == True
-                ):
-                    channel.query1[
-                        cache_key
-                    ] = False  # To improve waiting experience, this can be set to True.
-                    channel.query2[
-                        cache_key
-                    ] = False  # To improve waiting experience, this can be set to True.
-                    channel.query3[cache_key] = False
-                # User request again, and the answer is ready
-                elif cache_key in channel.cache_dict:
-                    # Skip the waiting phase
-                    channel.query1[cache_key] = True
-                    channel.query2[cache_key] = True
-                    channel.query3[cache_key] = True
-
-                assert not (
-                    cache_key in channel.cache_dict and cache_key in channel.running
+                        replyPost = reply.TextMsg(wechatmp_msg.from_user_id, wechatmp_msg.to_user_id, content).send()
+                        return replyPost
+
+
+                # Wechat official server will request 3 times (5 seconds each), with the same message_id.
+                # Because the interval is 5 seconds, here assumed that do not have multithreading problems.
+                request_cnt = channel.request_cnt.get(message_id, 0) + 1
+                channel.request_cnt[message_id] = request_cnt
+                logger.info(
+                    "[wechatmp] Request {} from {} {}\n{}\n{}:{}".format(
+                        request_cnt,
+                        from_user,
+                        message_id,
+                        message,
+                        web.ctx.env.get("REMOTE_ADDR"),
+                        web.ctx.env.get("REMOTE_PORT"),
+                    )
                 )
 
-                if channel.query1.get(cache_key) == False:
-                    # The first query from wechat official server
-                    logger.debug("[wechatmp] query1 {}".format(cache_key))
-                    channel.query1[cache_key] = True
-                    cnt = 0
-                    while cache_key in channel.running and cnt < 45:
-                        cnt = cnt + 1
+                task_running = True
+                waiting_until = request_time + 4
+                while time.time() < waiting_until:
+                    if from_user in channel.running:
                         time.sleep(0.1)
-                    if cnt == 45:
-                        # waiting for timeout (the POST query will be closed by wechat official server)
-                        time.sleep(1)
-                        # and do nothing
-                        return
                     else:
-                        pass
-                elif channel.query2.get(cache_key) == False:
-                    # The second query from wechat official server
-                    logger.debug("[wechatmp] query2 {}".format(cache_key))
-                    channel.query2[cache_key] = True
-                    cnt = 0
-                    while cache_key in channel.running and cnt < 45:
-                        cnt = cnt + 1
-                        time.sleep(0.1)
-                    if cnt == 45:
-                        # waiting for timeout (the POST query will be closed by wechat official server)
-                        time.sleep(1)
-                        # and do nothing
-                        return
-                    else:
-                        pass
-                elif channel.query3.get(cache_key) == False:
-                    # The third query from wechat official server
-                    logger.debug("[wechatmp] query3 {}".format(cache_key))
-                    channel.query3[cache_key] = True
-                    cnt = 0
-                    while cache_key in channel.running and cnt < 40:
-                        cnt = cnt + 1
-                        time.sleep(0.1)
-                    if cnt == 40:
-                        # Have waiting for 3x5 seconds
+                        task_running = False
+                        break
+
+                reply_text = ""
+                if task_running:
+                    if request_cnt < 3:
+                        # waiting for timeout (the POST request will be closed by Wechat official server)
+                        time.sleep(2)
+                        # and do nothing, waiting for the next request
+                        return "success"
+                    else: # request_cnt == 3:
                         # return timeout message
                         reply_text = "【正在思考中，回复任意文字尝试获取回复】"
-                        logger.info(
-                            "[wechatmp] Three queries has finished For {}: {}".format(
-                                from_user, message_id
-                            )
-                        )
-                        replyPost = reply.TextMsg(from_user, to_user, reply_text).send()
-                        return replyPost
-                    else:
-                        pass
+                        # replyPost = reply.TextMsg(from_user, to_user, reply_text).send()
+                        # return replyPost
+
+                # reply or reply_text is ready
+                channel.request_cnt.pop(message_id)
 
+                # no return because of bandwords or other reasons
                 if (
-                    cache_key not in channel.cache_dict
-                    and cache_key not in channel.running
+                    from_user not in channel.cache_dict
+                    and from_user not in channel.running
                 ):
-                    # no return because of bandwords or other reasons
                     return "success"
 
-                # if float(time.time()) - float(query_time) > 4.8:
-                #     reply_text = "【正在思考中，回复任意文字尝试获取回复】"
-                #     logger.info("[wechatmp] Timeout for {} {}, return".format(from_user, message_id))
-                #     replyPost = reply.TextMsg(from_user, to_user, reply_text).send()
-                #     return replyPost
+                # reply is ready
+                if from_user in channel.cache_dict:
+                    # Only one message thread can access to the cached data
+                    try:
+                        content = channel.cache_dict.pop(from_user)
+                    except KeyError:
+                        return "success"
 
-                if cache_key in channel.cache_dict:
-                    content = channel.cache_dict[cache_key]
                     if len(content.encode("utf8")) <= MAX_UTF8_LEN:
-                        reply_text = channel.cache_dict[cache_key]
-                        channel.cache_dict.pop(cache_key)
+                        reply_text = content
                     else:
                         continue_text = "\n【未完待续，回复任意文字以继续】"
                         splits = split_string_by_utf8_length(
@@ -202,11 +146,14 @@ def POST(self):
                             max_split=1,
                         )
                         reply_text = splits[0] + continue_text
-                        channel.cache_dict[cache_key] = splits[1]
+                        channel.cache_dict[from_user] = splits[1]
+
                 logger.info(
-                    "[wechatmp] {}:{} Do send {}".format(
-                        web.ctx.env.get("REMOTE_ADDR"),
-                        web.ctx.env.get("REMOTE_PORT"),
+                    "[wechatmp] Request {} do send to {} {}: {}\n{}".format(
+                        request_cnt,
+                        from_user,
+                        message_id,
+                        message,
                         reply_text,
                     )
                 )
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index 453a1a4a2..b3a91088b 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-import web
 import io
 import imghdr
 import requests
@@ -8,18 +7,17 @@
 from channel.chat_channel import ChatChannel
 from channel.wechatmp.wechatmp_client import WechatMPClient
 from channel.wechatmp.common import *
-from common.expired_dict import ExpiredDict
 from common.log import logger
-from common.tmp_dir import TmpDir
 from common.singleton import singleton
 from config import conf
 
+import web
 # If using SSL, uncomment the following lines, and modify the certificate path.
-from cheroot.server import HTTPServer
-from cheroot.ssl.builtin import BuiltinSSLAdapter
-HTTPServer.ssl_adapter = BuiltinSSLAdapter(
-        certificate='/ssl/cert.pem',
-        private_key='/ssl/cert.key')
+# from cheroot.server import HTTPServer
+# from cheroot.ssl.builtin import BuiltinSSLAdapter
+# HTTPServer.ssl_adapter = BuiltinSSLAdapter(
+#         certificate='/ssl/cert.pem',
+#         private_key='/ssl/cert.key')
 
 
 @singleton
@@ -27,15 +25,17 @@ class WechatMPChannel(ChatChannel):
     def __init__(self, passive_reply=True):
         super().__init__()
         self.passive_reply = passive_reply
-        self.running = set()
-        self.received_msgs = ExpiredDict(60 * 60 * 24)
+        self.flag = 0
+
         self.client = WechatMPClient()
         if self.passive_reply:
             self.NOT_SUPPORT_REPLYTYPE = [ReplyType.IMAGE, ReplyType.VOICE]
+            # Cache the reply to the user's first message
             self.cache_dict = dict()
-            self.query1 = dict()
-            self.query2 = dict()
-            self.query3 = dict()
+            # Record whether the current message is being processed
+            self.running = set()
+            # Count the request from wechat official server by message_id
+            self.request_cnt = dict()
         else:
             self.NOT_SUPPORT_REPLYTYPE = []
 
@@ -53,8 +53,8 @@ def startup(self):
     def send(self, reply: Reply, context: Context):
         receiver = context["receiver"]
         if self.passive_reply:
+            logger.info("[wechatmp] reply to {} cached:\n{}".format(receiver, reply))
             self.cache_dict[receiver] = reply.content
-            logger.info("[wechatmp] reply cached reply to {}: {}".format(receiver, reply))
         else:
             if reply.type == ReplyType.TEXT or reply.type == ReplyType.INFO or reply.type == ReplyType.ERROR:
                 reply_text = reply.content
@@ -64,7 +64,6 @@ def send(self, reply: Reply, context: Context):
             elif reply.type == ReplyType.VOICE:
                 voice_file_path = reply.content
                 logger.info("[wechatmp] voice file path {}".format(voice_file_path))
-
                 with open(voice_file_path, 'rb') as f:
                     filename = receiver + "-" + context["msg"].msg_id + ".mp3"
                     media_id = self.client.upload_media("voice", (filename, f, "audio/mpeg"))
@@ -86,6 +85,7 @@ def send(self, reply: Reply, context: Context):
                 media_id = self.client.upload_media("image", (filename, image_storage, content_type))
                 self.client.send_image(receiver, media_id)
                 logger.info("[wechatmp] sendImage url={}, receiver={}".format(img_url, receiver))
+
             elif reply.type == ReplyType.IMAGE:  # 从文件读取图片
                 image_storage = reply.content
                 image_storage.seek(0)

From f32f8aa08e88f6012b9cbdd56d6d1f996f898374 Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Thu, 20 Apr 2023 03:18:21 +0800
Subject: [PATCH 09/14] Update readme, and make the structure more clear

---
 channel/wechatmp/README.md                    | 32 +++++++++----------
 .../{service_account.py => active_reply.py}   |  8 ++---
 ...{subscribe_account.py => passive_reply.py} | 12 +++----
 .../{reply.py => passive_reply_message.py}    | 23 +++++++++++++
 channel/wechatmp/wechatmp_channel.py          |  4 +--
 .../{receive.py => wechatmp_message.py}       |  0
 6 files changed, 51 insertions(+), 28 deletions(-)
 rename channel/wechatmp/{service_account.py => active_reply.py} (93%)
 rename channel/wechatmp/{subscribe_account.py => passive_reply.py} (95%)
 rename channel/wechatmp/{reply.py => passive_reply_message.py} (68%)
 rename channel/wechatmp/{receive.py => wechatmp_message.py} (100%)

diff --git a/channel/wechatmp/README.md b/channel/wechatmp/README.md
index 58732faea..12b429de0 100644
--- a/channel/wechatmp/README.md
+++ b/channel/wechatmp/README.md
@@ -1,9 +1,7 @@
 # 微信公众号channel
 
 鉴于个人微信号在服务器上通过itchat登录有封号风险，这里新增了微信公众号channel，提供无风险的服务。
-目前支持订阅号（个人）和服务号（企业）两种类型的公众号，它们的主要区别就是被动回复和主动回复。
-个人微信订阅号有许多接口限制，目前仅支持最基本的文本对话和语音输入，支持加载插件，支持私有api_key。
-暂未实现图片输入输出、语音输出等交互形式。
+目前支持订阅号和服务号两种类型的公众号。个人主体的微信订阅号由于无法通过微信认证，接口存在限制，目前仅支持最基本的文本交互和语音输入。通过微信认证的订阅号或者服务号可以回复图片和语音。
 
 ## 使用方法（订阅号，服务号类似）
 
@@ -21,40 +19,42 @@ pip3 install web.py
 
 相关的服务器验证代码已经写好，你不需要再添加任何代码。你只需要在本项目根目录的`config.json`中添加
 ```
-"channel_type": "wechatmp",
+"channel_type": "wechatmp",     # 如果通过了微信认证，将"wechatmp"替换为"wechatmp_service"，可极大的优化使用体验
 "wechatmp_token": "Token",      # 微信公众平台的Token
 "wechatmp_port": 8080,          # 微信公众平台的端口,需要端口转发到80或443
-"wechatmp_app_id": "",          # 微信公众平台的appID，仅服务号需要
-"wechatmp_app_secret": "",      # 微信公众平台的appsecret，仅服务号需要
+"wechatmp_app_id": "",          # 微信公众平台的appID，wechatmp_service需要填写
+"wechatmp_app_secret": "",      # 微信公众平台的appsecret，wechatmp_service需要填写
 "single_chat_prefix": [""],     # 推荐设置，任意对话都可以触发回复，不添加前缀
 "single_chat_reply_prefix": "", # 推荐设置，回复不设置前缀
 "plugin_trigger_prefix": "&",   # 推荐设置，在手机微信客户端中，$%^等符号与中文连在一起时会自动显示一段较大的间隔，用户体验不好。请不要使用管理员指令前缀"#"，这会造成未知问题。
 ```
-然后运行`python3 app.py`启动web服务器。这里会默认监听8080端口，但是微信公众号的服务器配置只支持80/443端口，有两种方法来解决这个问题。第一个是推荐的方法，使用端口转发命令将80端口转发到8080端口（443同理，注意需要支持SSL，也就是https的访问，在`wechatmp_channel.py`需要修改相应的证书路径）：
+然后运行`python3 app.py`启动web服务器。这里会默认监听8080端口，但是微信公众号的服务器配置只支持80/443端口，有两种方法来解决这个问题。第一个是推荐的方法，使用端口转发命令将80端口转发到8080端口：
 ```
 sudo iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 8080
 sudo iptables-save > /etc/iptables/rules.v4
 ```
-第二个方法是让python程序直接监听80端口。这样可能会导致权限问题，在linux上需要使用`sudo`。然而这会导致后续缓存文件的权限问题，因此不是推荐的方法。
-最后在刚才的“服务器配置”中点击`提交`即可验证你的服务器。
+第二个方法是让python程序直接监听80端口，在配置文件中设置`"wechatmp_port": 80`,在linux上需要使用`sudo python3 app`启动程序。然而这会导致一系列环境和权限问题，因此不是推荐的方法。
 
+443端口同理，注意需要支持SSL，也就是https的访问，在`wechatmp_channel.py`中需要修改相应的证书路径。
+
+程序启动并监听端口后，在刚才的“服务器配置”中点击`提交`即可验证你的服务器。
 随后在[微信公众平台](https://mp.weixin.qq.com)启用服务器，关闭手动填写规则的自动回复，即可实现ChatGPT的自动回复。
 
 ## 个人微信公众号的限制
 由于人微信公众号不能通过微信认证，所以没有客服接口，因此公众号无法主动发出消息，只能被动回复。而微信官方对被动回复有5秒的时间限制，最多重试2次，因此最多只有15秒的自动回复时间窗口。因此如果问题比较复杂或者我们的服务器比较忙，ChatGPT的回答就没办法及时回复给用户。为了解决这个问题，这里做了回答缓存，它需要你在回复超时后，再次主动发送任意文字（例如1）来尝试拿到回答缓存。为了优化使用体验，目前设置了两分钟（120秒）的timeout，用户在至多两分钟后即可得到查询到回复或者错误原因。
 
-另外，由于微信官方的限制，自动回复有长度限制。因此这里将ChatGPT的回答拆分，分成每段600字回复（限制大约在700字）。
+另外，由于微信官方的限制，自动回复有长度限制。因此这里将ChatGPT的回答进行了拆分，以满足限制。
 
 ## 私有api_key
-公共api有访问频率限制（免费账号每分钟最多20次ChatGPT的API调用），这在服务多人的时候会遇到问题。因此这里多加了一个设置私有api_key的功能。目前通过godcmd插件的命令来设置私有api_key。
+公共api有访问频率限制（免费账号每分钟最多3次ChatGPT的API调用），这在服务多人的时候会遇到问题。因此这里多加了一个设置私有api_key的功能。目前通过godcmd插件的命令来设置私有api_key。
 
 ## 语音输入
 利用微信自带的语音识别功能，提供语音输入能力。需要在公众号管理页面的“设置与开发”->“接口权限”页面开启“接收语音识别结果”。
 
-## 测试范围
-目前在`RoboStyle`这个公众号上进行了测试（基于[wechatmp分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp)），感兴趣的可以关注并体验。开启了godcmd, Banwords, role, dungeon, finish这五个插件，其他的插件还没有测试。百度的接口暂未测试。语音对话没有测试。图片直接以链接形式回复（没有临时素材上传接口的权限）。
+## 测试
+目前在`RoboStyle`这个公众号上进行了测试（基于[wechatmp-stable分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp-stable)），感兴趣的可以关注并体验。开启了godcmd, Banwords, role, dungeon, finish这五个插件，其他的插件还没有详尽测试。百度的接口暂未测试。pytts可用。
 
 ## TODO
-* 服务号交互完善
-* 服务号使用临时素材接口，提供图片回复能力
-* 插件测试
+* 图片输入
+* 使用永久素材接口提供未认证公众号的图片和语音回复
+* 高并发支持
diff --git a/channel/wechatmp/service_account.py b/channel/wechatmp/active_reply.py
similarity index 93%
rename from channel/wechatmp/service_account.py
rename to channel/wechatmp/active_reply.py
index adaa754f6..2df3f234a 100644
--- a/channel/wechatmp/service_account.py
+++ b/channel/wechatmp/active_reply.py
@@ -2,8 +2,8 @@
 
 import web
 
-import channel.wechatmp.receive as receive
-import channel.wechatmp.reply as reply
+from channel.wechatmp.wechatmp_message import parse_xml
+from channel.wechatmp.passive_reply_message import TextMsg
 from bridge.context import *
 from channel.wechatmp.common import *
 from channel.wechatmp.wechatmp_channel import WechatMPChannel
@@ -22,7 +22,7 @@ def POST(self):
         try:
             webData = web.data()
             # logger.debug("[wechatmp] Receive request:\n" + webData.decode("utf-8"))
-            wechatmp_msg = receive.parse_xml(webData)
+            wechatmp_msg = parse_xml(webData)
             if (
                 wechatmp_msg.msg_type == "text" 
                 or wechatmp_msg.msg_type == "voice" 
@@ -62,7 +62,7 @@ def POST(self):
                     )
                 )
                 content = subscribe_msg()
-                replyMsg = reply.TextMsg(
+                replyMsg = TextMsg(
                     wechatmp_msg.from_user_id, wechatmp_msg.to_user_id, content
                 )
                 return replyMsg.send()
diff --git a/channel/wechatmp/subscribe_account.py b/channel/wechatmp/passive_reply.py
similarity index 95%
rename from channel/wechatmp/subscribe_account.py
rename to channel/wechatmp/passive_reply.py
index fcdb2d8cb..c5decf489 100644
--- a/channel/wechatmp/subscribe_account.py
+++ b/channel/wechatmp/passive_reply.py
@@ -2,8 +2,8 @@
 
 import web
 
-import channel.wechatmp.receive as receive
-import channel.wechatmp.reply as reply
+from channel.wechatmp.wechatmp_message import parse_xml
+from channel.wechatmp.passive_reply_message import TextMsg
 from bridge.context import *
 from channel.wechatmp.common import *
 from channel.wechatmp.wechatmp_channel import WechatMPChannel
@@ -22,7 +22,7 @@ def POST(self):
             channel = WechatMPChannel()
             webData = web.data()
             logger.debug("[wechatmp] Receive post data:\n" + webData.decode("utf-8"))
-            wechatmp_msg = receive.parse_xml(webData)
+            wechatmp_msg = parse_xml(webData)
             if wechatmp_msg.msg_type == "text" or wechatmp_msg.msg_type == "voice":
                 from_user = wechatmp_msg.from_user_id
                 to_user = wechatmp_msg.to_user_id
@@ -77,7 +77,7 @@ def POST(self):
                                 """\
                                 未知错误，请稍后再试"""
                             )
-                        replyPost = reply.TextMsg(wechatmp_msg.from_user_id, wechatmp_msg.to_user_id, content).send()
+                        replyPost = TextMsg(wechatmp_msg.from_user_id, wechatmp_msg.to_user_id, content).send()
                         return replyPost
 
 
@@ -157,7 +157,7 @@ def POST(self):
                         reply_text,
                     )
                 )
-                replyPost = reply.TextMsg(from_user, to_user, reply_text).send()
+                replyPost = TextMsg(from_user, to_user, reply_text).send()
                 return replyPost
 
             elif wechatmp_msg.msg_type == "event":
@@ -167,7 +167,7 @@ def POST(self):
                     )
                 )
                 content = subscribe_msg()
-                replyMsg = reply.TextMsg(
+                replyMsg = TextMsg(
                     wechatmp_msg.from_user_id, wechatmp_msg.to_user_id, content
                 )
                 return replyMsg.send()
diff --git a/channel/wechatmp/reply.py b/channel/wechatmp/passive_reply_message.py
similarity index 68%
rename from channel/wechatmp/reply.py
rename to channel/wechatmp/passive_reply_message.py
index 2f852f99b..ef58d7093 100644
--- a/channel/wechatmp/reply.py
+++ b/channel/wechatmp/passive_reply_message.py
@@ -32,6 +32,29 @@ def send(self):
         return XmlForm.format(**self.__dict)
 
 
+class VoiceMsg(Msg):
+    def __init__(self, toUserName, fromUserName, mediaId):
+        self.__dict = dict()
+        self.__dict["ToUserName"] = toUserName
+        self.__dict["FromUserName"] = fromUserName
+        self.__dict["CreateTime"] = int(time.time())
+        self.__dict["MediaId"] = mediaId
+
+    def send(self):
+        XmlForm = """
+            <xml>
+                <ToUserName><![CDATA[{ToUserName}]]></ToUserName>
+                <FromUserName><![CDATA[{FromUserName}]]></FromUserName>
+                <CreateTime>{CreateTime}</CreateTime>
+                <MsgType><![CDATA[voice]]></MsgType>
+                <Voice>
+                <MediaId><![CDATA[{MediaId}]]></MediaId>
+                </Voice>
+            </xml>
+            """
+        return XmlForm.format(**self.__dict)
+
+
 class ImageMsg(Msg):
     def __init__(self, toUserName, fromUserName, mediaId):
         self.__dict = dict()
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index b3a91088b..48ef14ecd 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -42,9 +42,9 @@ def __init__(self, passive_reply=True):
 
     def startup(self):
         if self.passive_reply:
-            urls = ("/wx", "channel.wechatmp.subscribe_account.Query")
+            urls = ("/wx", "channel.wechatmp.passive_reply.Query")
         else:
-            urls = ("/wx", "channel.wechatmp.service_account.Query")
+            urls = ("/wx", "channel.wechatmp.active_reply.Query")
         app = web.application(urls, globals(), autoreload=False)
         port = conf().get("wechatmp_port", 8080)
         web.httpserver.runsimple(app.wsgifunc(), ("0.0.0.0", port))
diff --git a/channel/wechatmp/receive.py b/channel/wechatmp/wechatmp_message.py
similarity index 100%
rename from channel/wechatmp/receive.py
rename to channel/wechatmp/wechatmp_message.py

From 1e58c1ad2ba61bc20223afa568e92a0f0aa0205e Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Thu, 20 Apr 2023 04:35:06 +0800
Subject: [PATCH 10/14] fix: wechatmp channel now do not need client

---
 channel/wechatmp/README.md           | 6 +++---
 channel/wechatmp/wechatmp_channel.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/channel/wechatmp/README.md b/channel/wechatmp/README.md
index 12b429de0..8ace25bfb 100644
--- a/channel/wechatmp/README.md
+++ b/channel/wechatmp/README.md
@@ -20,10 +20,10 @@ pip3 install web.py
 相关的服务器验证代码已经写好，你不需要再添加任何代码。你只需要在本项目根目录的`config.json`中添加
 ```
 "channel_type": "wechatmp",     # 如果通过了微信认证，将"wechatmp"替换为"wechatmp_service"，可极大的优化使用体验
-"wechatmp_token": "Token",      # 微信公众平台的Token
+"wechatmp_token": "xxxx",       # 微信公众平台的Token
 "wechatmp_port": 8080,          # 微信公众平台的端口,需要端口转发到80或443
-"wechatmp_app_id": "",          # 微信公众平台的appID，wechatmp_service需要填写
-"wechatmp_app_secret": "",      # 微信公众平台的appsecret，wechatmp_service需要填写
+"wechatmp_app_id": "xxxx",      # 微信公众平台的appID
+"wechatmp_app_secret": "xxxx",  # 微信公众平台的appsecret
 "single_chat_prefix": [""],     # 推荐设置，任意对话都可以触发回复，不添加前缀
 "single_chat_reply_prefix": "", # 推荐设置，回复不设置前缀
 "plugin_trigger_prefix": "&",   # 推荐设置，在手机微信客户端中，$%^等符号与中文连在一起时会自动显示一段较大的间隔，用户体验不好。请不要使用管理员指令前缀"#"，这会造成未知问题。
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index 48ef14ecd..74e6e25a0 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -27,7 +27,6 @@ def __init__(self, passive_reply=True):
         self.passive_reply = passive_reply
         self.flag = 0
 
-        self.client = WechatMPClient()
         if self.passive_reply:
             self.NOT_SUPPORT_REPLYTYPE = [ReplyType.IMAGE, ReplyType.VOICE]
             # Cache the reply to the user's first message
@@ -38,6 +37,7 @@ def __init__(self, passive_reply=True):
             self.request_cnt = dict()
         else:
             self.NOT_SUPPORT_REPLYTYPE = []
+            self.client = WechatMPClient()
 
 
     def startup(self):

From 34209021c8667daacbbf649b2a4a0ece9d533f0c Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Thu, 20 Apr 2023 09:04:42 +0800
Subject: [PATCH 11/14] fix: pytts second round not work

---
 voice/pytts/pytts_voice.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/voice/pytts/pytts_voice.py b/voice/pytts/pytts_voice.py
index 57d0cfb31..968286cb8 100644
--- a/voice/pytts/pytts_voice.py
+++ b/voice/pytts/pytts_voice.py
@@ -13,9 +13,9 @@
 import os
 
 class PyttsVoice(Voice):
-    engine = pyttsx3.init()
 
     def __init__(self):
+        self.engine = pyttsx3.init()
         # 语速
         self.engine.setProperty("rate", 125)
         # 音量
@@ -23,25 +23,24 @@ def __init__(self):
         for voice in self.engine.getProperty("voices"):
             if "Chinese" in voice.name:
                 self.engine.setProperty("voice", voice.id)
-
         self.engine.setProperty("voice", "zh")
+        self.engine.startLoop(useDriverLoop=False)
 
     def textToVoice(self, text):
         try:
-            mp3FileName = "reply-" + str(int(time.time())) + ".mp3"
+            mp3FileName = "reply-" + str(int(time.time()*100)) + ".mp3"
             mp3File = TmpDir().path() + mp3FileName
-            self.engine.save_to_file(text, mp3File)
-            self.engine.runAndWait()
-
-            # engine.runAndWait() will return before the file created
-            while mp3FileName not in os.listdir(TmpDir().path()):
-                time.sleep(0.1)
-
             logger.info(
                 "[Pytts] textToVoice text={} voice file name={}".format(text, mp3File)
             )
+            self.engine.save_to_file(text, mp3File)
+            self.engine.iterate()
+            while self.engine.isBusy() or mp3FileName not in os.listdir(TmpDir().path()):
+                time.sleep(0.1)
+            logger.debug("[Pytts] Task finished")
             reply = Reply(ReplyType.VOICE, mp3File)
         except Exception as e:
+            print(e)
             reply = Reply(ReplyType.ERROR, str(e))
         finally:
             return reply

From a7772316f96d19cd3646cc451f893b3adb34b1c1 Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Thu, 20 Apr 2023 10:26:58 +0800
Subject: [PATCH 12/14] feat: wechatmp channel support voice/image reply

---
 channel/chat_channel.py              |  4 +-
 channel/wechatmp/active_reply.py     |  6 ++-
 channel/wechatmp/passive_reply.py    | 64 +++++++++++++++----------
 channel/wechatmp/wechatmp_channel.py | 72 +++++++++++++++++++++++-----
 channel/wechatmp/wechatmp_client.py  | 53 ++++++++++++++++++++
 channel/wechatmp/wechatmp_message.py |  9 ++--
 6 files changed, 164 insertions(+), 44 deletions(-)

diff --git a/channel/chat_channel.py b/channel/chat_channel.py
index 89dfe145a..1a7fa30ee 100644
--- a/channel/chat_channel.py
+++ b/channel/chat_channel.py
@@ -144,14 +144,14 @@ def _compose_context(self, ctype: ContextType, content, **kwargs):
                 context.type = ContextType.TEXT
             context.content = content.strip()
             if (
-                "desire_rtype" not in context
+                context["desire_rtype"] == None
                 and conf().get("always_reply_voice")
                 and ReplyType.VOICE not in self.NOT_SUPPORT_REPLYTYPE
             ):
                 context["desire_rtype"] = ReplyType.VOICE
         elif context.type == ContextType.VOICE:
             if (
-                "desire_rtype" not in context
+                context["desire_rtype"] == None
                 and conf().get("voice_reply_voice")
                 and ReplyType.VOICE not in self.NOT_SUPPORT_REPLYTYPE
             ):
diff --git a/channel/wechatmp/active_reply.py b/channel/wechatmp/active_reply.py
index 2df3f234a..c700ca3a4 100644
--- a/channel/wechatmp/active_reply.py
+++ b/channel/wechatmp/active_reply.py
@@ -5,6 +5,7 @@
 from channel.wechatmp.wechatmp_message import parse_xml
 from channel.wechatmp.passive_reply_message import TextMsg
 from bridge.context import *
+from bridge.reply import ReplyType
 from channel.wechatmp.common import *
 from channel.wechatmp.wechatmp_channel import WechatMPChannel
 from common.log import logger
@@ -29,7 +30,7 @@ def POST(self):
                 # or wechatmp_msg.msg_type == "image"
             ):
                 from_user = wechatmp_msg.from_user_id
-                message = wechatmp_msg.content.decode("utf-8")
+                message = wechatmp_msg.content
                 message_id = wechatmp_msg.msg_id
 
                 logger.info(
@@ -41,8 +42,9 @@ def POST(self):
                         message,
                     )
                 )
+                rtype = ReplyType.VOICE if wechatmp_msg.msg_type == "voice" else None
                 context = channel._compose_context(
-                    ContextType.TEXT, message, isgroup=False, msg=wechatmp_msg
+                    ContextType.TEXT, message, isgroup=False, desire_rtype=rtype, msg=wechatmp_msg
                 )
                 if context:
                     # set private openai_api_key
diff --git a/channel/wechatmp/passive_reply.py b/channel/wechatmp/passive_reply.py
index c5decf489..f489270c4 100644
--- a/channel/wechatmp/passive_reply.py
+++ b/channel/wechatmp/passive_reply.py
@@ -1,10 +1,12 @@
 import time
+import asyncio
 
 import web
 
 from channel.wechatmp.wechatmp_message import parse_xml
-from channel.wechatmp.passive_reply_message import TextMsg
+from channel.wechatmp.passive_reply_message import TextMsg, VoiceMsg, ImageMsg
 from bridge.context import *
+from bridge.reply import ReplyType
 from channel.wechatmp.common import *
 from channel.wechatmp.wechatmp_channel import WechatMPChannel
 from common.log import logger
@@ -26,7 +28,7 @@ def POST(self):
             if wechatmp_msg.msg_type == "text" or wechatmp_msg.msg_type == "voice":
                 from_user = wechatmp_msg.from_user_id
                 to_user = wechatmp_msg.to_user_id
-                message = wechatmp_msg.content.decode("utf-8")
+                message = wechatmp_msg.content
                 message_id = wechatmp_msg.msg_id
 
                 supported = True
@@ -41,8 +43,9 @@ def POST(self):
                     and message_id not in channel.request_cnt # insert the godcmd
                 ):
                     # The first query begin
+                    rtype = ReplyType.VOICE if wechatmp_msg.msg_type == "voice" else None
                     context = channel._compose_context(
-                        ContextType.TEXT, message, isgroup=False, msg=wechatmp_msg
+                        ContextType.TEXT, message, isgroup=False, desire_rtype=rtype, msg=wechatmp_msg
                     )
                     logger.debug(
                         "[wechatmp] context: {} {}".format(context, wechatmp_msg)
@@ -115,10 +118,10 @@ def POST(self):
                     else: # request_cnt == 3:
                         # return timeout message
                         reply_text = "【正在思考中，回复任意文字尝试获取回复】"
-                        # replyPost = reply.TextMsg(from_user, to_user, reply_text).send()
-                        # return replyPost
+                        replyPost = TextMsg(from_user, to_user, reply_text).send()
+                        return replyPost
 
-                # reply or reply_text is ready
+                # reply is ready
                 channel.request_cnt.pop(message_id)
 
                 # no return because of bandwords or other reasons
@@ -128,14 +131,13 @@ def POST(self):
                 ):
                     return "success"
 
-                # reply is ready
-                if from_user in channel.cache_dict:
-                    # Only one message thread can access to the cached data
-                    try:
-                        content = channel.cache_dict.pop(from_user)
-                    except KeyError:
-                        return "success"
+                # Only one request can access to the cached data
+                try:
+                    (reply_type, content) = channel.cache_dict.pop(from_user)
+                except KeyError:
+                    return "success"
 
+                if (reply_type == "text"):
                     if len(content.encode("utf8")) <= MAX_UTF8_LEN:
                         reply_text = content
                     else:
@@ -146,19 +148,31 @@ def POST(self):
                             max_split=1,
                         )
                         reply_text = splits[0] + continue_text
-                        channel.cache_dict[from_user] = splits[1]
-
-                logger.info(
-                    "[wechatmp] Request {} do send to {} {}: {}\n{}".format(
-                        request_cnt,
-                        from_user,
-                        message_id,
-                        message,
-                        reply_text,
+                        channel.cache_dict[from_user] = ("text", splits[1])
+                    
+                    logger.info(
+                        "[wechatmp] Request {} do send to {} {}: {}\n{}".format(
+                            request_cnt,
+                            from_user,
+                            message_id,
+                            message,
+                            reply_text,
+                        )
                     )
-                )
-                replyPost = TextMsg(from_user, to_user, reply_text).send()
-                return replyPost
+                    replyPost = TextMsg(from_user, to_user, reply_text).send()
+                    return replyPost
+
+                elif (reply_type == "voice"):
+                    media_id = content
+                    asyncio.run_coroutine_threadsafe(channel.delete_media(media_id), channel.delete_media_loop)
+                    replyPost = VoiceMsg(from_user, to_user, media_id).send()
+                    return replyPost
+
+                elif (reply_type == "image"):
+                    media_id = content
+                    asyncio.run_coroutine_threadsafe(channel.delete_media(media_id), channel.delete_media_loop)
+                    replyPost = ImageMsg(from_user, to_user, media_id).send()
+                    return replyPost
 
             elif wechatmp_msg.msg_type == "event":
                 logger.info(
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index 74e6e25a0..1c83149a6 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 import io
+import os
+import time
 import imghdr
 import requests
 from bridge.context import *
@@ -11,6 +13,9 @@
 from common.singleton import singleton
 from config import conf
 
+import asyncio
+from threading import Thread
+
 import web
 # If using SSL, uncomment the following lines, and modify the certificate path.
 # from cheroot.server import HTTPServer
@@ -25,19 +30,20 @@ class WechatMPChannel(ChatChannel):
     def __init__(self, passive_reply=True):
         super().__init__()
         self.passive_reply = passive_reply
-        self.flag = 0
-
+        self.NOT_SUPPORT_REPLYTYPE = []
+        self.client = WechatMPClient()
         if self.passive_reply:
-            self.NOT_SUPPORT_REPLYTYPE = [ReplyType.IMAGE, ReplyType.VOICE]
             # Cache the reply to the user's first message
             self.cache_dict = dict()
             # Record whether the current message is being processed
             self.running = set()
             # Count the request from wechat official server by message_id
             self.request_cnt = dict()
-        else:
-            self.NOT_SUPPORT_REPLYTYPE = []
-            self.client = WechatMPClient()
+            # The permanent media need to be deleted to avoid media number limit
+            self.delete_media_loop = asyncio.new_event_loop()
+            t = Thread(target=self.start_loop, args=(self.delete_media_loop,))
+            t.setDaemon(True)
+            t.start()
 
 
     def startup(self):
@@ -49,18 +55,63 @@ def startup(self):
         port = conf().get("wechatmp_port", 8080)
         web.httpserver.runsimple(app.wsgifunc(), ("0.0.0.0", port))
 
+    def start_loop(self, loop):
+        asyncio.set_event_loop(loop)
+        loop.run_forever()
+
+    async def delete_media(self, media_id):
+        logger.info("[wechatmp] media {} will be deleted in 10s".format(media_id))
+        await asyncio.sleep(10)
+        self.client.delete_permanent_media(media_id)
+        logger.info("[wechatmp] media {} has been deleted".format(media_id))
 
     def send(self, reply: Reply, context: Context):
         receiver = context["receiver"]
         if self.passive_reply:
-            logger.info("[wechatmp] reply to {} cached:\n{}".format(receiver, reply))
-            self.cache_dict[receiver] = reply.content
+            if reply.type == ReplyType.TEXT or reply.type == ReplyType.INFO or reply.type == ReplyType.ERROR:
+                reply_text = reply.content
+                logger.info("[wechatmp] reply to {} cached:\n{}".format(receiver, reply_text))
+                self.cache_dict[receiver] = ("text", reply_text)
+            elif reply.type == ReplyType.VOICE:
+                voice_file_path = reply.content
+                logger.info("[wechatmp] voice file path {}".format(voice_file_path))
+                with open(voice_file_path, 'rb') as f:
+                    filename = receiver + "-" + context["msg"].msg_id + ".mp3"
+                    media_id = self.client.upload_permanent_media("voice", (filename, f, "audio/mpeg"))
+                    # 根据文件大小估计一个微信自动审核的时间，审核结束前返回将会导致语音无法播放，这个估计有待验证
+                    f_size = os.fstat(f.fileno()).st_size
+                    print(f_size)
+                    time.sleep(1.0 + 2 * f_size / 1024 / 1024)
+                    logger.info("[wechatmp] voice reply to {} uploaded: {}".format(receiver, media_id))
+                    self.cache_dict[receiver] = ("voice", media_id)
+            elif reply.type == ReplyType.IMAGE_URL:  # 从网络下载图片
+                img_url = reply.content
+                pic_res = requests.get(img_url, stream=True)
+                print(pic_res.headers)
+                image_storage = io.BytesIO()
+                for block in pic_res.iter_content(1024):
+                    image_storage.write(block)
+                image_storage.seek(0)
+                image_type = imghdr.what(image_storage)
+                filename = receiver + "-" + context["msg"].msg_id + "." + image_type
+                content_type = "image/" + image_type
+                media_id = self.client.upload_permanent_media("image", (filename, image_storage, content_type))
+                logger.info("[wechatmp] image reply to {} uploaded: {}".format(receiver, media_id))
+                self.cache_dict[receiver] = ("image", media_id)
+            elif reply.type == ReplyType.IMAGE:  # 从文件读取图片
+                image_storage = reply.content
+                image_storage.seek(0)
+                image_type = imghdr.what(image_storage)
+                filename = receiver + "-" + context["msg"].msg_id + "." + image_type
+                content_type = "image/" + image_type
+                media_id = self.client.upload_permanent_media("image", (filename, image_storage, content_type))
+                logger.info("[wechatmp] image reply to {} uploaded: {}".format(receiver, media_id))
+                self.cache_dict[receiver] = ("image", media_id)
         else:
             if reply.type == ReplyType.TEXT or reply.type == ReplyType.INFO or reply.type == ReplyType.ERROR:
                 reply_text = reply.content
                 self.client.send_text(receiver, reply_text)
                 logger.info("[wechatmp] Do send to {}: {}".format(receiver, reply_text))
-
             elif reply.type == ReplyType.VOICE:
                 voice_file_path = reply.content
                 logger.info("[wechatmp] voice file path {}".format(voice_file_path))
@@ -69,7 +120,6 @@ def send(self, reply: Reply, context: Context):
                     media_id = self.client.upload_media("voice", (filename, f, "audio/mpeg"))
                     self.client.send_voice(receiver, media_id)
                     logger.info("[wechatmp] Do send voice to {}".format(receiver))
-
             elif reply.type == ReplyType.IMAGE_URL:  # 从网络下载图片
                 img_url = reply.content
                 pic_res = requests.get(img_url, stream=True)
@@ -85,7 +135,6 @@ def send(self, reply: Reply, context: Context):
                 media_id = self.client.upload_media("image", (filename, image_storage, content_type))
                 self.client.send_image(receiver, media_id)
                 logger.info("[wechatmp] sendImage url={}, receiver={}".format(img_url, receiver))
-
             elif reply.type == ReplyType.IMAGE:  # 从文件读取图片
                 image_storage = reply.content
                 image_storage.seek(0)
@@ -95,7 +144,6 @@ def send(self, reply: Reply, context: Context):
                 media_id = self.client.upload_media("image", (filename, image_storage, content_type))
                 self.client.send_image(receiver, media_id)
                 logger.info("[wechatmp] sendImage, receiver={}".format(receiver))
-
         return
 
     def _success_callback(self, session_id, context, **kwargs):  # 线程异常结束时的回调函数
diff --git a/channel/wechatmp/wechatmp_client.py b/channel/wechatmp/wechatmp_client.py
index 7a2cd022c..321726d22 100644
--- a/channel/wechatmp/wechatmp_client.py
+++ b/channel/wechatmp/wechatmp_client.py
@@ -23,6 +23,8 @@ def wechatmp_request(self, method, url, **kwargs):
         r.encoding = "utf-8"
         ret = r.json()
         if "errcode" in ret and ret["errcode"] != 0:
+            if ret["errcode"] == 45009:
+                self.clear_quota_v2()
             raise WeChatAPIException("{}".format(ret))
         return ret
 
@@ -123,3 +125,54 @@ def upload_media(self, media_type, media_file):
             files=files
         )
         return ret["media_id"]
+
+
+    def upload_permanent_media(self, media_type, media_file):
+        url="https://api.weixin.qq.com/cgi-bin/material/add_material"
+        params={
+            "access_token": self.get_access_token(),
+            "type": media_type
+        }
+        files={"media": media_file}
+        logger.info("[wechatmp] media {} uploaded".format(media_file))
+        ret = self.wechatmp_request(
+            method="post",
+            url=url,
+            params=params,
+            files=files
+        )
+        return ret["media_id"]
+
+
+    def delete_permanent_media(self, media_id):
+        url="https://api.weixin.qq.com/cgi-bin/material/del_material"
+        params={
+            "access_token": self.get_access_token()
+        }
+        logger.info("[wechatmp] media {} deleted".format(media_id))
+        self.wechatmp_request(
+            method="post",
+            url=url,
+            params=params,
+            data={"media_id": media_id}
+        )
+
+    def clear_quota(self):
+        url="https://api.weixin.qq.com/cgi-bin/clear_quota"
+        params = {
+            "access_token": self.get_access_token()
+        }
+        self.wechatmp_request(
+            method="post",
+            url=url,
+            params=params,
+            data={"appid": self.app_id}
+        )
+
+    def clear_quota_v2(self):
+        url="https://api.weixin.qq.com/cgi-bin/clear_quota/v2"
+        self.wechatmp_request(
+            method="post",
+            url=url,
+            data={"appid": self.app_id, "appsecret": self.app_secret}
+        )
diff --git a/channel/wechatmp/wechatmp_message.py b/channel/wechatmp/wechatmp_message.py
index 1285fd155..d385897c3 100644
--- a/channel/wechatmp/wechatmp_message.py
+++ b/channel/wechatmp/wechatmp_message.py
@@ -32,12 +32,15 @@ def __init__(self, xmlData):
 
         if self.msg_type == "text":
             self.ctype = ContextType.TEXT
-            self.content = xmlData.find("Content").text.encode("utf-8")
+            self.content = xmlData.find("Content").text
         elif self.msg_type == "voice":
             self.ctype = ContextType.TEXT
-            self.content = xmlData.find("Recognition").text.encode("utf-8")  # 接收语音识别结果
+            self.content = xmlData.find("Recognition").text  # 接收语音识别结果
+            # other voice_to_text method not implemented yet
+            if self.content == None:
+                self.content = "你好"
         elif self.msg_type == "image":
-            # not implemented
+            # not implemented yet
             self.pic_url = xmlData.find("PicUrl").text
             self.media_id = xmlData.find("MediaId").text
         elif self.msg_type == "event":

From 40264bc9cb611c9497202b20b29c6ec7e1527087 Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Thu, 20 Apr 2023 12:03:48 +0800
Subject: [PATCH 13/14] fix: delete permanent media

---
 channel/wechatmp/active_reply.py     |  5 ++++-
 channel/wechatmp/passive_reply.py    |  5 ++++-
 channel/wechatmp/wechatmp_channel.py |  4 ++--
 channel/wechatmp/wechatmp_client.py  | 10 ++++++----
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/channel/wechatmp/active_reply.py b/channel/wechatmp/active_reply.py
index c700ca3a4..d8a8ddee1 100644
--- a/channel/wechatmp/active_reply.py
+++ b/channel/wechatmp/active_reply.py
@@ -42,7 +42,10 @@ def POST(self):
                         message,
                     )
                 )
-                rtype = ReplyType.VOICE if wechatmp_msg.msg_type == "voice" else None
+                if (wechatmp_msg.msg_type == "voice" and conf().get("voice_reply_voice") == True):
+                    rtype = ReplyType.VOICE
+                else:
+                    rtype = None
                 context = channel._compose_context(
                     ContextType.TEXT, message, isgroup=False, desire_rtype=rtype, msg=wechatmp_msg
                 )
diff --git a/channel/wechatmp/passive_reply.py b/channel/wechatmp/passive_reply.py
index f489270c4..eca94ba36 100644
--- a/channel/wechatmp/passive_reply.py
+++ b/channel/wechatmp/passive_reply.py
@@ -43,7 +43,10 @@ def POST(self):
                     and message_id not in channel.request_cnt # insert the godcmd
                 ):
                     # The first query begin
-                    rtype = ReplyType.VOICE if wechatmp_msg.msg_type == "voice" else None
+                    if (wechatmp_msg.msg_type == "voice" and conf().get("voice_reply_voice") == True):
+                        rtype = ReplyType.VOICE
+                    else:
+                        rtype = None
                     context = channel._compose_context(
                         ContextType.TEXT, message, isgroup=False, desire_rtype=rtype, msg=wechatmp_msg
                     )
diff --git a/channel/wechatmp/wechatmp_channel.py b/channel/wechatmp/wechatmp_channel.py
index 1c83149a6..9780048b9 100644
--- a/channel/wechatmp/wechatmp_channel.py
+++ b/channel/wechatmp/wechatmp_channel.py
@@ -60,10 +60,10 @@ def start_loop(self, loop):
         loop.run_forever()
 
     async def delete_media(self, media_id):
-        logger.info("[wechatmp] media {} will be deleted in 10s".format(media_id))
+        logger.debug("[wechatmp] permanent media {} will be deleted in 10s".format(media_id))
         await asyncio.sleep(10)
         self.client.delete_permanent_media(media_id)
-        logger.info("[wechatmp] media {} has been deleted".format(media_id))
+        logger.info("[wechatmp] permanent media {} has been deleted".format(media_id))
 
     def send(self, reply: Reply, context: Context):
         receiver = context["receiver"]
diff --git a/channel/wechatmp/wechatmp_client.py b/channel/wechatmp/wechatmp_client.py
index 321726d22..96ebddb74 100644
--- a/channel/wechatmp/wechatmp_client.py
+++ b/channel/wechatmp/wechatmp_client.py
@@ -117,13 +117,13 @@ def upload_media(self, media_type, media_file):
             "type": media_type
         }
         files={"media": media_file}
-        logger.info("[wechatmp] media {} uploaded".format(media_file))
         ret = self.wechatmp_request(
             method="post",
             url=url,
             params=params,
             files=files
         )
+        logger.debug("[wechatmp] media {} uploaded".format(media_file))
         return ret["media_id"]
 
 
@@ -134,13 +134,13 @@ def upload_permanent_media(self, media_type, media_file):
             "type": media_type
         }
         files={"media": media_file}
-        logger.info("[wechatmp] media {} uploaded".format(media_file))
         ret = self.wechatmp_request(
             method="post",
             url=url,
             params=params,
             files=files
         )
+        logger.debug("[wechatmp] permanent media {} uploaded".format(media_file))
         return ret["media_id"]
 
 
@@ -149,13 +149,13 @@ def delete_permanent_media(self, media_id):
         params={
             "access_token": self.get_access_token()
         }
-        logger.info("[wechatmp] media {} deleted".format(media_id))
         self.wechatmp_request(
             method="post",
             url=url,
             params=params,
-            data={"media_id": media_id}
+            data=json.dumps({"media_id": media_id}, ensure_ascii=False).encode("utf8")
         )
+        logger.debug("[wechatmp] permanent media {} deleted".format(media_id))
 
     def clear_quota(self):
         url="https://api.weixin.qq.com/cgi-bin/clear_quota"
@@ -168,6 +168,7 @@ def clear_quota(self):
             params=params,
             data={"appid": self.app_id}
         )
+        logger.debug("[wechatmp] API quata has been cleard")
 
     def clear_quota_v2(self):
         url="https://api.weixin.qq.com/cgi-bin/clear_quota/v2"
@@ -176,3 +177,4 @@ def clear_quota_v2(self):
             url=url,
             data={"appid": self.app_id, "appsecret": self.app_secret}
         )
+        logger.debug("[wechatmp] API quata has been cleard")

From 71c804369941254d3d93b8fb50fd838b8f8be665 Mon Sep 17 00:00:00 2001
From: JS00000 <js00000000000000000@gmail.com>
Date: Thu, 20 Apr 2023 12:35:54 +0800
Subject: [PATCH 14/14] update README

---
 channel/wechatmp/README.md | 32 +++++++++++++++++++++++++++-----
 channel/wechatmp/common.py |  8 ++++----
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/channel/wechatmp/README.md b/channel/wechatmp/README.md
index 8ace25bfb..9655843ac 100644
--- a/channel/wechatmp/README.md
+++ b/channel/wechatmp/README.md
@@ -33,7 +33,7 @@ pip3 install web.py
 sudo iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 8080
 sudo iptables-save > /etc/iptables/rules.v4
 ```
-第二个方法是让python程序直接监听80端口，在配置文件中设置`"wechatmp_port": 80`,在linux上需要使用`sudo python3 app`启动程序。然而这会导致一系列环境和权限问题，因此不是推荐的方法。
+第二个方法是让python程序直接监听80端口，在配置文件中设置`"wechatmp_port": 80` ，在linux上需要使用`sudo python3 app.py`启动程序。然而这会导致一系列环境和权限问题，因此不是推荐的方法。
 
 443端口同理，注意需要支持SSL，也就是https的访问，在`wechatmp_channel.py`中需要修改相应的证书路径。
 
@@ -51,10 +51,32 @@ sudo iptables-save > /etc/iptables/rules.v4
 ## 语音输入
 利用微信自带的语音识别功能，提供语音输入能力。需要在公众号管理页面的“设置与开发”->“接口权限”页面开启“接收语音识别结果”。
 
+## 语音回复
+请在配置文件中添加以下词条：
+```
+  "voice_reply_voice": true,
+  "text_to_voice": "pytts",
+```
+这样公众号将会用语音回复语音消息，实现语音对话。
+pytts是语音合成引擎之一。还支持baidu,google,azure，这些你需要自行配置相关的依赖和key。
+如果使用pytts，在ubuntu上需要安装如下依赖：
+```
+sudo apt update
+sudo apt install espeak
+sudo apt install ffmpeg
+python3 -m pip install pyttsx3
+```
+不是很建议开启pytts语音回复，因为它是离线本地计算，算的慢会拖垮服务器，且声音不好听。
+
+## 图片回复
+现在认证公众号和非认证公众号都可以实现的图片和语音回复。但是非认证公众号使用了永久素材接口，每天有1000次的调用上限（每个月有10次重置机会，程序中已设定遇到上限会自动重置），且永久素材库存也有上限。因此对于非认证公众号，我们会在回复图片或者语音消息后的10秒内从永久素材库存内删除该素材。
+
 ## 测试
-目前在`RoboStyle`这个公众号上进行了测试（基于[wechatmp-stable分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp-stable)），感兴趣的可以关注并体验。开启了godcmd, Banwords, role, dungeon, finish这五个插件，其他的插件还没有详尽测试。百度的接口暂未测试。pytts可用。
+目前在`RoboStyle`这个公众号上进行了测试（基于[wechatmp分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp)），感兴趣的可以关注并体验。开启了godcmd, Banwords, role, dungeon, finish这五个插件，其他的插件还没有详尽测试。百度的接口暂未测试。[wechatmp-stable分支](https://github.com/JS00000/chatgpt-on-wechat/tree/wechatmp-stable)是较稳定的上个版本，但也缺少最新的功能支持。
 
 ## TODO
-* 图片输入
-* 使用永久素材接口提供未认证公众号的图片和语音回复
-* 高并发支持
+ - [x] 语音输入
+ - [ ] 图片输入
+ - [x] 使用临时素材接口提供认证公众号的图片和语音回复
+ - [x] 使用永久素材接口提供未认证公众号的图片和语音回复
+ - [ ] 高并发支持
diff --git a/channel/wechatmp/common.py b/channel/wechatmp/common.py
index 192b86bda..5efccfce1 100644
--- a/channel/wechatmp/common.py
+++ b/channel/wechatmp/common.py
@@ -36,16 +36,16 @@ def verify_server(data):
 
 
 def subscribe_msg():
-    trigger_prefix = conf().get("single_chat_prefix", [""])[0]
+    trigger_prefix = conf().get("single_chat_prefix", [""])
     msg = textwrap.dedent(
         f"""\
                     感谢您的关注！
                     这里是ChatGPT，可以自由对话。
                     资源有限，回复较慢，请勿着急。
-                    支持通用表情输入。
+                    支持语音对话。
                     暂时不支持图片输入。
-                    支持图片输出，画字开头的问题将回复图片链接。
-                    支持角色扮演和文字冒险两种定制模式对话。
+                    支持图片输出，画字开头的消息将按要求创作图片。
+                    支持tool、角色扮演和文字冒险等丰富的插件。
                     输入'{trigger_prefix}#帮助' 查看详细指令。"""
     )
     return msg