From 52405c0c992ea52eba7f63d3cda0aceb2a1f5f8d Mon Sep 17 00:00:00 2001 From: hero1s Date: Thu, 20 Jun 2024 19:41:36 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=87=AA=E5=8A=A8=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/conf/admin.conf | 5 +- bin/conf/cachesvr.conf | 5 +- bin/conf/log.conf | 11 +++- bin/conf/monitor.conf | 7 ++- bin/conf/proxy.conf | 22 -------- script/agent/mongo_agent.lua | 78 +++++++++++++++++++++------- script/agent/rmsg_agent.lua | 4 +- script/store/reliable_msg.lua | 34 ++++++------ server/monitor/dbindex_mgr.lua | 95 ++++++++++++++-------------------- 9 files changed, 142 insertions(+), 119 deletions(-) delete mode 100644 bin/conf/proxy.conf diff --git a/bin/conf/admin.conf b/bin/conf/admin.conf index 71441774..a5e1a610 100644 --- a/bin/conf/admin.conf +++ b/bin/conf/admin.conf @@ -26,4 +26,7 @@ set_env("HIVE_ADMIN_LIMIT_IP","") --用户密码 set_env("HIVE_ADMIN_USER","admin") -set_env("HIVE_ADMIN_PWD","dsybs") \ No newline at end of file +set_env("HIVE_ADMIN_PWD","dsybs") + +-- http还是https请求 +set_env("HTTP_MODE", "http") diff --git a/bin/conf/cachesvr.conf b/bin/conf/cachesvr.conf index 889e28e8..4d41cd15 100644 --- a/bin/conf/cachesvr.conf +++ b/bin/conf/cachesvr.conf @@ -18,4 +18,7 @@ set_env("HIVE_ENTRY", "main_cachesvr") --定义服务名称 set_env("HIVE_SERVICE", "cachesvr") - +--开启对象计数 +set_env("HIVE_OPEN_TRACK", 0) +--每帧保存限频 +set_env("HIVE_SAVE_LIMIT", 100) diff --git a/bin/conf/log.conf b/bin/conf/log.conf index df4eb0e7..cc2e62cd 100644 --- a/bin/conf/log.conf +++ b/bin/conf/log.conf @@ -25,5 +25,14 @@ set_env("HIVE_LOG_ROLL", "1") --日志打印函数名和文件行号 set_env("HIVE_LOG_SHOW", "0") +--开启http调试日志 +set_env("HIVE_HTTP_OPEN_DEBUG", "0") + --飞书上报URL ---set_env("HIVE_LARK_URL", "https://open.feishu.cn/open-apis/bot/v2/hook/cf68991a-0580-4d90-9ab5-2da9ed5f9308") \ No newline at end of file +--set_env("HIVE_LARK_URL", "https://open.feishu.cn/open-apis/bot/v2/hook/cf68991a-0580-4d90-9ab5-2da9ed5f9308") + +----------------------------------------------------- +--数据统计开关[耗cpu,带宽,线上关闭] +set_env("HIVE_STATIS", "0") +--性能统计开关[耗cpu,线上关闭] +set_env("HIVE_PERFEVAL", "0") \ No newline at end of file diff --git a/bin/conf/monitor.conf b/bin/conf/monitor.conf index 0417bb0b..90a3c7fd 100644 --- a/bin/conf/monitor.conf +++ b/bin/conf/monitor.conf @@ -21,6 +21,11 @@ set_env("HIVE_SERVICE", "monitor") --IP地址相关 -------------------------------------------------------- --monitor监听地址 -set_env("HIVE_MONITOR_HTTP", "0.0.0.0:9101") set_env("HIVE_MONITOR_HOST", "127.0.0.1:9201") +--mongo是否分片(分片模式下过滤分片索引) +set_env("HIVE_MONGO_SHARDING", 0) +--是否自动构建mongo索引(线上建议手动) +set_env("HIVE_AUTO_BUILD_MONGO_INDEX", 1) +--mongo索引创建失败是否删除重建 +set_env("HIVE_MONGO_INDEX_REBUILD", 1) \ No newline at end of file diff --git a/bin/conf/proxy.conf b/bin/conf/proxy.conf deleted file mode 100644 index 1aaef874..00000000 --- a/bin/conf/proxy.conf +++ /dev/null @@ -1,22 +0,0 @@ ---[[ -hive启动环境配置 -启动： - 启动第一个参数是本配置文件的路径，后续跟环境变量 -备注： - 环境变量可在此文件配置，也可以配置在启动参数，从启动参数配置时，系统会自动补全HIVE_前缀 -案例： - ./hive.exe hive.conf --index=3 --service=test -]] - - ---包含通用环境变量配置 -dofile("conf/share.conf") - ---启动参数 ---------------------------------------------------------- ---定义启动文件路径 -set_env("HIVE_ENTRY", "main_proxy") ---定义服务名称 -set_env("HIVE_SERVICE", "proxy") - - diff --git a/script/agent/mongo_agent.lua b/script/agent/mongo_agent.lua index 8f09fdcd..6cff5414 100644 --- a/script/agent/mongo_agent.lua +++ b/script/agent/mongo_agent.lua @@ -1,19 +1,20 @@ --mongo_agent.lua -local tunpack = table.unpack -local check_failed = hive.failed -local log_err = logger.err -local log_info = logger.info -local log_warn = logger.warn -local mrandom = math_ext.random -local tequals = table_ext.equals -local tequal_keys = table_ext.equal_keys -local KernCode = enum("KernCode") -local router_mgr = hive.load("router_mgr") -local scheduler = hive.load("scheduler") - -local AUTOINCCC = environ.get("HIVE_DB_AUTOINCTB", "autoinctb") -local MongoAgent = singleton() -local prop = property(MongoAgent) +local tunpack = table.unpack +local check_failed = hive.failed +local log_err = logger.err +local log_info = logger.info +local log_warn = logger.warn +local mrandom = math_ext.random +local tequals = table_ext.equals +local tequal_keys = table_ext.equal_keys +local KernCode = enum("KernCode") +local router_mgr = hive.load("router_mgr") +local scheduler = hive.load("scheduler") +local check_success = hive.success + +local AUTOINCCC = environ.get("HIVE_DB_AUTOINCTB", "autoinctb") +local MongoAgent = singleton() +local prop = property(MongoAgent) prop:reader("service", "mongo") prop:reader("local_run", false) --本地线程服务 @@ -180,7 +181,7 @@ function MongoAgent:update_sheet(sheet_name, primary_id, primary_key, udata, db_ end --检测是否建立了索引(only_key仅检测字段名) -function MongoAgent:check_indexes(key, co_name, db_name, only_key) +function MongoAgent:check_indexes(index, co_name, db_name, only_key) local ok, code, indexs = self:get_indexes(co_name, db_name) if check_failed(code, ok) then log_err("[MongoAgent][check_indexes] failed:{},{},ok:{},code:{},indexs:{}", co_name, db_name, ok, code, indexs) @@ -188,11 +189,19 @@ function MongoAgent:check_indexes(key, co_name, db_name, only_key) end local cmp_func = only_key and tequal_keys or tequals for i, v in ipairs(indexs) do - if cmp_func(v.key, key) then + if cmp_func(v.key, index.key) then + if index.unique and not v.unique then + log_warn("[MongoAgent][check_indexes] db:{},table:{},index:{} --> {},is not unique", db_name, co_name, index, v) + return false + end + if index.expireAfterSeconds and v.expireAfterSeconds ~= index.expireAfterSeconds then + log_warn("[MongoAgent][check_indexes] db:{},table:{},index:{} --> {},expireAfterSeconds is not equal", db_name, co_name, index, v) + return false + end return true end end - log_warn("[MongoAgent][check_indexes] db:{},table:{},key:{},is not exist indexes:{}", db_name, co_name, key, indexs) + log_warn("[MongoAgent][check_indexes] db:{},table:{},index:{},is not exist indexes:{}", db_name, co_name, index, indexs) return false end @@ -221,6 +230,39 @@ function MongoAgent:push_array_data(selector, co_name, array, data, limit, hash_ return true end +--重建索引 +function MongoAgent:rebuild_create_index(index, table_name, db_name, rebuild) + --检测是否创建过索引 + if self:check_indexes(index, table_name, db_name, false) then + log_info("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] is exist index", db_name, table_name, index.name) + return true + end + local query = { table_name, { index } } + local ok, code = self:create_indexes(query, 1, db_name) + if check_success(code, ok) then + log_info("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] build index success", db_name, table_name, index.name) + return true + else + log_err("[MongoAgent][rebuild_create_index] db[{}],table[{}] build index[{}] fail:{}", db_name, table_name, index, code) + if rebuild then + ok, code = self:drop_indexes({ table_name, index.name }, 1, db_name) + if check_success(code, ok) then + log_warn("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] drop index success", db_name, table_name, index.name) + ok, code = self:create_indexes(query, 1, db_name) + if check_success(code, ok) then + log_info("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] drop and build index success", db_name, table_name, index.name) + return true + else + log_err("[MongoAgent][rebuild_create_index] db[{}],table[{}] drop and build index[{}] fail:{}", db_name, table_name, index, code) + end + else + log_err("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] drop index fail:{}", db_name, table_name, index.name, code) + end + end + end + return false +end + hive.mongo_agent = MongoAgent() return MongoAgent diff --git a/script/agent/rmsg_agent.lua b/script/agent/rmsg_agent.lua index 38f813e7..6d08898c 100644 --- a/script/agent/rmsg_agent.lua +++ b/script/agent/rmsg_agent.lua @@ -20,9 +20,9 @@ function RmsgAgent:setup() end end -function RmsgAgent:build_index(sharding) +function RmsgAgent:build_index(sharding, rebuild) for _, rmsg in pairs(self.rmsgs or {}) do - rmsg:build_index(sharding) + rmsg:build_index(sharding, rebuild) end end diff --git a/script/store/reliable_msg.lua b/script/store/reliable_msg.lua index 76167826..98090fd7 100644 --- a/script/store/reliable_msg.lua +++ b/script/store/reliable_msg.lua @@ -23,7 +23,7 @@ function ReliableMsg:__init(db_name, table_name, due_day) self.table_name = table_name end -function ReliableMsg:build_index(sharding) +function ReliableMsg:generate_index(sharding) local indexs = { { key = { uuid = 1 }, name = "uuid", unique = (not sharding), background = true }, } @@ -33,29 +33,27 @@ function ReliableMsg:build_index(sharding) if self.ttl then tinsert(indexs, { key = { ttl = 1 }, expireAfterSeconds = 0, name = "ttl", unique = false, background = true }) end - local query = { self.table_name, indexs } - local ok, code = mongo_agent:create_indexes(query, nil, self.db_name) - if check_success(code, ok) then - log_info("[ReliableMsg][build_index] rmsg table:{},sharding:{} build due index success", self.table_name, sharding) - else - log_err("[ReliableMsg][build_index] rmsg table:{},sharding:{} build due index failed", self.table_name, sharding) + return indexs +end + +function ReliableMsg:build_index(sharding, rebuild) + local indexs = self:generate_index(sharding) + for _, index in pairs(indexs) do + if not sharding or index.name ~= "to" then + mongo_agent:rebuild_create_index(index, self.table_name, self.db_name, rebuild) + end end end function ReliableMsg:check_indexes(sharding) + local indexs = self:generate_index(sharding) local success = true local miss = {} - if not mongo_agent:check_indexes({ uuid = 1 }, self.table_name, self.db_name, false) then - success = false - tinsert(miss, { db = self.db_name, co = self.table_name, key = { uuid = 1 } }) - end - if not mongo_agent:check_indexes({ to = 1 }, self.table_name, self.db_name, sharding) then - success = false - tinsert(miss, { db = self.db_name, co = self.table_name, key = { to = 1 } }) - end - if not mongo_agent:check_indexes({ ttl = 1 }, self.table_name, self.db_name, false) then - success = false - tinsert(miss, { db = self.db_name, co = self.table_name, key = { ttl = 1 }, expireAfterSeconds = 0 }) + for _, index in ipairs(indexs) do + if not mongo_agent:check_indexes(index, self.table_name, self.db_name, sharding) then + success = false + tinsert(miss, { db = self.db_name, co = self.table_name, index = index }) + end end return success, miss end diff --git a/server/monitor/dbindex_mgr.lua b/server/monitor/dbindex_mgr.lua index 8889fdf6..db0b12e2 100644 --- a/server/monitor/dbindex_mgr.lua +++ b/server/monitor/dbindex_mgr.lua @@ -5,7 +5,7 @@ local tjoin = table_ext.join local log_err = logger.err local log_info = logger.info local log_warn = logger.warn -local check_success = hive.success + local monitor = hive.get("monitor") local mongo_agent = hive.get("mongo_agent") local thread_mgr = hive.get("thread_mgr") @@ -15,6 +15,8 @@ local config_mgr = hive.get("config_mgr") local GMType = enum("GMType") +local dbindex_db = config_mgr:init_table("dbindex", "db_name", "table_name", "name") + local DBIndexMgr = singleton() local prop = property(DBIndexMgr) prop:accessor("status", 0) @@ -57,14 +59,12 @@ end function DBIndexMgr:build_index(rebuild) log_info("[DBIndexMgr][build_index] rebuild:{}", rebuild) self:build_dbindex(rebuild) - rmsg_agent:build_index(self.sharding) + rmsg_agent:build_index(self.sharding, rebuild) end function DBIndexMgr:check_dbindexes() - log_info("[DBIndexMgr][check_dbindexes]") - local success = true - local miss = {} - local dbindex_db = config_mgr:init_table("dbindex", "db_name", "table_name", "name") + local success = true + local miss = {} for _, conf in dbindex_db:iterator() do local db_name = conf.db_name local table_name = conf.table_name @@ -76,13 +76,10 @@ function DBIndexMgr:check_dbindexes() if self.sharding and conf.sharding then only_key = true end - local check_key = {} - for _, v in ipairs(conf.keys) do - check_key[v] = 1 - end - if not mongo_agent:check_indexes(check_key, table_name, db_name, only_key) then + local index = self:generate_index(conf) + if not mongo_agent:check_indexes(index, table_name, db_name, only_key) then success = false - tinsert(miss, { db = db_name, co = table_name, key = check_key }) + tinsert(miss, { db = db_name, co = table_name, index = index }) end end local ok, res = rmsg_agent:check_indexes(self.sharding) @@ -90,56 +87,44 @@ function DBIndexMgr:check_dbindexes() miss = tjoin(res, miss) success = false end + log_warn("[DBIndexMgr][check_dbindexes] ret:{},miss:{}", success, miss) return success, miss end +-- 检测表是否分片 +function DBIndexMgr:get_key_unique(db_name, table_name, unique) + if self.sharding then + for _, conf in dbindex_db:iterator() do + if db_name == conf.db_name and table_name == conf.table_name and conf.sharding then + return false + end + end + end + return unique +end + +function DBIndexMgr:generate_index(conf) + local index = {} + index.key = {} + index.name = conf.name + index.unique = self:get_key_unique(conf.db_name, conf.table_name, conf.unique) + index.background = true + if conf.expireAfterSeconds > 0 then + index.expireAfterSeconds = conf.expireAfterSeconds + end + for _, v in ipairs(conf.keys) do + index.key[v] = 1 + end + return index +end + function DBIndexMgr:build_dbindex(rebuild) log_info("[DBIndexMgr][build_dbindex] rebuild:{},sharding:{}", rebuild, self.sharding) - local dbindex_db = config_mgr:init_table("dbindex", "db_name", "table_name", "name") for _, conf in dbindex_db:iterator() do - if self.sharding and conf.sharding then - goto continue - end - local db_name = conf.db_name - local table_name = conf.table_name - local index = {} - index.key = {} - index.name = conf.name - index.unique = (not self.sharding) and conf.unique or false - index.background = true - if conf.expireAfterSeconds > 0 then - index.expireAfterSeconds = conf.expireAfterSeconds - end - for _, v in ipairs(conf.keys) do - index.key[v] = 1 - end - --检测是否创建过索引 - if mongo_agent:check_indexes(index.key, table_name, db_name, false) then - log_info("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] is exist index", db_name, table_name, index.name) - goto continue - end - local query = { table_name, { index } } - local ok, code = mongo_agent:create_indexes(query, 1, db_name) - if check_success(code, ok) then - log_info("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] build index success", db_name, table_name, index.name) - else - log_err("[DBIndexMgr][build_index] db[{}],table[{}] build index[{}] fail:{}", db_name, table_name, index, code) - if rebuild then - ok, code = mongo_agent:drop_indexes({ table_name, index.name }, 1, db_name) - if check_success(code, ok) then - log_warn("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] drop index success", db_name, table_name, index.name) - ok, code = mongo_agent:create_indexes(query, 1, db_name) - if check_success(code, ok) then - log_info("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] drop and build index success", db_name, table_name, index.name) - else - log_err("[DBIndexMgr][build_index] db[{}],table[{}] drop and build index[{}] fail:{}", db_name, table_name, index, code) - end - else - log_err("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] drop index fail:{}", db_name, table_name, index.name, code) - end - end + if not self.sharding or not conf.sharding then + local index = self:generate_index(conf) + mongo_agent:rebuild_create_index(index, conf.table_name, conf.db_name, rebuild) end - :: continue :: end end