feat: 账号池功能 & feat: 优化黑名单检查性能 & feat: scheduler支持传参

MeoProject · Jan 5, 2024 · c08b455 · c08b455
1 parent ef4b4bf
commit c08b455
Show file tree

Hide file tree

Showing 11 changed files with 242 additions and 60 deletions.
diff --git a/common/config.py b/common/config.py
@@ -65,6 +65,8 @@ class ConfigReadException(Exception):
         "_proxy-desc": "代理配置，HTTP与HTTPS协议需分开配置",
         "log_file": True,
         "_log_file-desc": "是否开启日志文件",
+        'cookiepool': False,
+        '_cookiepool-desc': '是否开启cookie池，这将允许用户配置多个cookie并在请求时随机使用一个，启用后请在module.cookiepool中配置cookie，在user处配置的cookie会被忽略，cookiepool中格式统一为列表嵌套user处的cookie的字典',
         "allow_download_script": True,
         '_allow_download_script-desc': '是否允许直接从服务端下载脚本，开启后可以直接访问 /script?key=你的请求key 下载脚本',
         "download_config": {
@@ -174,14 +176,14 @@ class ConfigReadException(Exception):
                 "desc": "用户数据，可以通过浏览器获取，需要vip账号来获取会员歌曲，如果没有请留为空值，qqmusic_key可以从Cookie中/客户端的请求体中（comm.authst）获取",
                 "qqmusic_key": "",
                 "uin": "",
-                "_uin-desc": "key对应的QQ号"
+                "_uin-desc": "key对应的QQ号",
+                'refresh_login': {
+                    'desc': '刷新登录相关配置，enable是否启动，interval刷新间隔',
+                    'enable': False,
+                    'interval': 86000
+                }
             },
             "cdnaddr": "http://ws.stream.qqmusic.qq.com/",
-            'refresh_login': {
-                'desc': '刷新登录相关配置，enable是否启动，interval刷新间隔',
-                'enable': False,
-                'interval': 86000
-            }
         },
         "wy": {
             "desc": "网易云音乐相关配置",
@@ -197,9 +199,42 @@ class ConfigReadException(Exception):
                 "aversionid": "",
                 "token": "",
                 "osversion": "10",
-                "useragent": "Mozilla / 5.0 (Windows NT 10.0; Win64; x64) AppleWebKit / 537.36 (KHTML, like Gecko) Chrome / 89.0.4389.82 Safari / 537.36",
+                "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
             },
         },
+        'cookiepool': {
+            'kg': [
+                {
+                    'userid': '0',
+                    'token': '',
+                    'mid': '114514',
+                },
+            ],
+            'tx': [
+                {
+                    'qqmusic_key': '',
+                    'uin': '',
+                    'refresh_login': {
+                        'desc': 'cookie池中对于此账号刷新登录的配置，账号间互不干扰',
+                        'enable': False,
+                        'interval': 86000,
+                    }
+                }
+            ],
+            'wy': [
+                {
+                    'cookie': '',
+                }
+            ],
+            'mg': [
+                {
+                    'aversionid': '',
+                    'token': '',
+                    'osversion': '10',
+                    'useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36',
+                }
+            ],
+        },
     },
 }
 
@@ -413,7 +448,7 @@ def write_config(key, value):
     current[keys[-1]] = value
     variable.config = config
     with open('config.json', 'w', encoding='utf-8') as f:
-        json.dump(config, f, indent=2, ensure_ascii=False)
+        json.dump(config, f, indent=2, ensure_ascii=False, escape_forward_slashes=False)
         f.close()
 
 def read_default_config(key):
@@ -511,7 +546,7 @@ def initConfig():
                     logger.warning('配置文件并不是一个有效的字典，使用默认值')
                     variable.config = default
                     with open("./config.json", "w", encoding="utf-8") as f:
-                        f.write(json.dumps(variable.config, indent=2, ensure_ascii=False))
+                        f.write(json.dumps(variable.config, indent=2, ensure_ascii=False, escape_forward_slashes=False))
                         f.close()
             except:
                 if os.path.getsize("./config.json") != 0:
@@ -552,11 +587,17 @@ def initConfig():
 
     logger.debug('数据库初始化成功')
 
-    # print
-    if (load_data() == {}):
+    # handle data
+    all_data_keys = {'banList': [], 'requestTime': {}, 'banListRaw': []}
+    data = load_data()
+    if (data == {}):
         write_data('banList', [])
         write_data('requestTime', {})
         logger.info('数据库内容为空，已写入默认值')
+    for k, v in all_data_keys.items():
+        if (k not in data):
+            write_data(k, v)
+            logger.info(f'数据库中不存在{k}，已创建')
 
     # 处理代理配置
     if (read_config('common.proxy.enable')):
@@ -568,6 +609,36 @@ def initConfig():
             logger.info('HTTPS协议代理地址: ' + read_config('common.proxy.https_value'))
         logger.info('代理功能已开启，请确保代理地址正确，否则无法连接网络')
 
+    # cookie池
+    if (read_config('common.cookiepool')):
+        logger.info('已启用cookie池功能，请确定配置的cookie都能正确获取链接')
+        logger.info('传统的源 - 单用户cookie配置将被忽略')
+        logger.info('所以即使某个源你只有一个cookie，也请填写到cookiepool对应的源中，否则将无法使用该cookie')
+        variable.use_cookie_pool = True
+
+    # 移除已经过期的封禁数据
+    banlist = read_data('banList')
+    banlistRaw = read_data('banListRaw')
+    count = 0
+    for b in banlist:
+        if (b['expire'] and (time.time() > b['expire_time'])):
+            count += 1
+            banlist.remove(b)
+            if (b['ip'] in banlistRaw):
+                banlistRaw.remove(b['ip'])
+    write_data('banList', banlist)
+    write_data('banListRaw', banlistRaw)
+    if (count != 0):
+        logger.info(f'已移除{count}条过期封禁数据')
+
+    # 处理旧版数据库的banListRaw
+    banlist = read_data('banList')
+    banlistRaw = read_data('banListRaw')
+    if (banlist != [] and banlistRaw == []):
+        for b in banlist:
+            banlistRaw.append(b['ip'])
+    return 
+
 def ban_ip(ip_addr, ban_time=-1):
     if read_config('security.banlist.enable'):
         banList = read_data('banList')
@@ -586,20 +657,26 @@ def ban_ip(ip_addr, ban_time=-1):
 def check_ip_banned(ip_addr):
     if read_config('security.banlist.enable'):
         banList = read_data('banList')
-        for ban in banList:
-            if (ban['ip'] == ip_addr):
-                if (ban['expire']):
-                    if (ban['expire_time'] > int(time.time())):
-                        return True
+        banlistRaw = read_data('banListRaw')
+        if (ip_addr in banlistRaw):
+            for b in banList:
+                if (b['ip'] == ip_addr):
+                    if (b['expire']):
+                        if (b['expire_time'] > int(time.time())):
+                            return True
+                        else:
+                            banList.remove(b)
+                            banlistRaw.remove(b['ip'])
+                            write_data('banListRaw', banlistRaw)
+                            write_data('banList', banList)
+                            return False
                     else:
-                        banList.remove(ban)
-                        write_data('banList', banList)
-                        return False
+                        return True
                 else:
-                    return True
-            else:
-                return False
-        return False
+                    return False
+            return False
+        else:
+            return False
     else:
         if (variable.banList_suggest <= 10):
             variable.banList_suggest += 1

diff --git a/common/lx_script.py b/common/lx_script.py
@@ -20,7 +20,7 @@
 
 async def get_response(retry = 0):
     if (retry > 10):
-        raise Exception('请求源脚本内容失败')
+        logger.warning('请求源脚本内容失败')
     baseurl = 'https://raw.githubusercontent.com/lxmusics/lx-music-api-server/main/lx-music-source-example.js'
     try:
         if (iscn and (retry % 2) == 0):
@@ -39,7 +39,7 @@ async def get_script():
             f.close()
         logger.info('更新源脚本成功')
     else:
-        raise Exception('请求源脚本内容失败')
+        logger.warning('请求源脚本内容失败')
 
 async def generate_script_response(request):
     if (request.query.get('key') != config.read_config('security.key.value') and config.read_config('security.key.enable')):

diff --git a/common/scheduler.py b/common/scheduler.py
@@ -21,28 +21,29 @@
 tasks = []
 
 class taskWrapper:
-    def __init__(self, name, function, interval = 86400, latest_execute = 0):
+    def __init__(self, name, function, interval = 86400, args = {}, latest_execute = 0):
         self.function = function
         self.interval = interval
         self.name = name
         self.latest_execute = latest_execute
+        self.args = args
 
     def check_available(self):
         return (time.time() - self.latest_execute) >= self.interval
 
     async def run(self):
         try:
             logger.info(f"task {self.name} run start")
-            await self.function()
+            await self.function(**self.args)
             logger.info(f'task {self.name} run success, next execute: {timestamp_format(self.interval + self.latest_execute)}')
         except Exception as e:
             logger.error(f"task {self.name} run failed, waiting for next execute...")
             logger.error(traceback.format_exc())
 
-def append(name, task, interval = 86400):
+def append(name, task, interval = 86400, args = {}):
     global tasks
+    wrapper = taskWrapper(name, task, interval, args)
     logger.debug(f"new task ({name}) registered")
-    wrapper = taskWrapper(name, task, interval)
     return tasks.append(wrapper)
 
 # 在 thread_runner 函数中修改循环逻辑
@@ -51,7 +52,7 @@ async def thread_runner():
     while not running_event.is_set():
         tasks_runner = []
         for t in tasks:
-            if t.check_available() and not running_event.is_set():
+            if (t.check_available() and not running_event.is_set()):
                 t.latest_execute = int(time.time())
                 tasks_runner.append(t.run())
         if (tasks_runner):

diff --git a/common/variable.py b/common/variable.py
@@ -49,4 +49,5 @@ def _read_config(key):
 iscn = True
 fake_ip = None
 aioSession = None
-qdes_lib_loaded = False
+qdes_lib_loaded = False
+use_cookie_pool = False
diff --git a/modules/kg/player.py b/modules/kg/player.py
@@ -6,8 +6,9 @@
 # - license: MIT - 
 # ----------------------------------------
 # This file is part of the "lx-music-api-server" project.
+import random
 from common.exceptions import FailedException
-from common import utils
+from common import config, utils, variable
 from .utils import getKey, signRequest, tools
 from .musicInfo import getMusicInfo
 import time
@@ -25,20 +26,21 @@ async def url(songId, quality):
     if (not albumaudioid):
         albumaudioid = ""
     thash = thash.lower()
+    user_info = config.read_config('module.kg.user') if (not variable.use_cookie_pool) else random.choice(config.read_config('module.cookiepool.kg'))
     params = {
         'album_id': albumid,
-        'userid': tools.userid,
+        'userid': user_info['userid'],
         'area_code': 1,
         'hash': thash,
         'module': '',
-        'mid': tools.mid,
+        'mid': user_info['mid'],
         'appid': tools.appid,
         'ssa_flag': 'is_fromtrack',
         'clientver': tools.clientver,
         'open_time': time.strftime("%Y%m%d"),
         'vipType': 6,
         'ptype': 0,
-        'token': tools.token,
+        'token': user_info['token'],
         'auth': '',
         'mtype': 0,
         'album_audio_id': albumaudioid,

diff --git a/modules/kg/utils.py b/modules/kg/utils.py
@@ -21,9 +21,6 @@
     "x-router": config.read_config("module.kg.tracker.x-router"),
     "url": config.read_config("module.kg.tracker.host") + config.read_config("module.kg.tracker.path"),
     "version": config.read_config("module.kg.tracker.version"),
-    "userid": config.read_config("module.kg.user.userid"),
-    "token": config.read_config("module.kg.user.token"),
-    "mid": config.read_config("module.kg.user.mid"),
     "extra_params": config.read_config("module.kg.tracker.extra_params"),
     "appid": config.read_config("module.kg.client.appid"),
     'qualityHashMap': {

diff --git a/modules/mg/__init__.py b/modules/mg/__init__.py
@@ -7,8 +7,10 @@
 # ----------------------------------------
 # This file is part of the "lx-music-api-server" project.
 
+import random
 from common import Httpx
 from common import config
+from common import variable
 from common.exceptions import FailedException
 
 tools = {
@@ -25,24 +27,21 @@
         'SQ': 'flac',
         'ZQ': 'flac24bit',
     },
-    'token': config.read_config('module.mg.user.token'),
-    'aversionid': config.read_config('module.mg.user.aversionid'),
-    'useragent': config.read_config('module.mg.user.useragent'),
-    'osversion': config.read_config('module.mg.user.osversion'),
 }
 
 async def url(songId, quality):
+    user_info = config.read_config('module.mg.user') if (not variable.use_cookie_pool) else random.choice(config.read_config('module.cookiepool.mg'))
     req = await Httpx.AsyncRequest(tools['url'].replace('__quality__', tools['qualityMap'][quality]).replace('__songId__', songId), {
         'method': 'GET',
         'headers': {
-            'User-Agent': tools['useragent'],
-            'aversionid': tools['aversionid'],
-            'token': tools['token'],
+            'User-Agent': user_info['useragent'],
+            'aversionid': user_info['aversionid'],
+            'token': user_info['token'],
             'channel': '0146832',
             'language': 'Chinese',
             'ua': 'Android_migu',
             'mode': 'android',
-            'os': 'Android ' + tools['osversion'],
+            'os': 'Android ' + user_info['osversion'],
         },
     })
     try:

diff --git a/modules/tx/player.py b/modules/tx/player.py
@@ -8,33 +8,35 @@
 # This file is part of the "lx-music-api-server" project.
 
 from common.exceptions import FailedException
-from common import config, utils
+from common import config, utils, variable
 from .musicInfo import getMusicInfo
 from .utils import tools
 from .utils import signRequest
+import random
 
 createObject = utils.CreateObject
 
 async def url(songId, quality):
     infoBody = await getMusicInfo(songId)
     strMediaMid = infoBody['track_info']['file']['media_mid']
+    user_info = config.read_config('module.tx.user') if (not variable.use_cookie_pool) else random.choice(config.read_config('module.cookiepool.tx'))
     requestBody = {
         'req_0': {
             'module': 'vkey.GetVkeyServer',
             'method': 'CgiGetVkey',
             'param': {
                 'filename': [f"{tools.fileInfo[quality]['h']}{strMediaMid}{tools.fileInfo[quality]['e']}"],
-                'guid': tools.guid,
+                'guid': user_info['guid'],
                 'songmid': [songId],
                 'songtype': [0],
-                'uin': tools.uin,
+                'uin': user_info['uin'],
                 'loginflag': 1,
                 'platform': '20',
             },
         },
         'comm': {
-            "qq": config.read_config('module.tx.user.uin'),
-            "authst": config.read_config('module.tx.user.qqmusic_key'),
+            "qq": user_info['uin'],
+            "authst": user_info['qqmusic_key'],
             "ct": "26",
             "cv": "2010101",
             "v": "2010101"