优化自动构建索引

bin/conf/admin.conf

@@ -26,4 +26,7 @@ set_env("HIVE_ADMIN_LIMIT_IP","")
 
 --用户密码
 set_env("HIVE_ADMIN_USER","admin")
-set_env("HIVE_ADMIN_PWD","dsybs")
+set_env("HIVE_ADMIN_PWD","dsybs")
+
+-- http还是https请求
+set_env("HTTP_MODE", "http")

bin/conf/cachesvr.conf

@@ -18,4 +18,7 @@ set_env("HIVE_ENTRY", "main_cachesvr")
 --定义服务名称
 set_env("HIVE_SERVICE", "cachesvr")
 
-
+--开启对象计数
+set_env("HIVE_OPEN_TRACK", 0)
+--每帧保存限频
+set_env("HIVE_SAVE_LIMIT", 100)

bin/conf/log.conf

@@ -25,5 +25,14 @@ set_env("HIVE_LOG_ROLL", "1")
 --日志打印函数名和文件行号
 set_env("HIVE_LOG_SHOW", "0")
 
+--开启http调试日志
+set_env("HIVE_HTTP_OPEN_DEBUG", "0")
+
 --飞书上报URL
---set_env("HIVE_LARK_URL", "https://open.feishu.cn/open-apis/bot/v2/hook/cf68991a-0580-4d90-9ab5-2da9ed5f9308")
+--set_env("HIVE_LARK_URL", "https://open.feishu.cn/open-apis/bot/v2/hook/cf68991a-0580-4d90-9ab5-2da9ed5f9308")
+
+-----------------------------------------------------
+--数据统计开关[耗cpu,带宽,线上关闭]
+set_env("HIVE_STATIS", "0")
+--性能统计开关[耗cpu,线上关闭]
+set_env("HIVE_PERFEVAL", "0")

bin/conf/monitor.conf

@@ -21,6 +21,11 @@ set_env("HIVE_SERVICE", "monitor")
 --IP地址相关
 --------------------------------------------------------
 --monitor监听地址
-set_env("HIVE_MONITOR_HTTP", "0.0.0.0:9101")
 set_env("HIVE_MONITOR_HOST", "127.0.0.1:9201")
 
+--mongo是否分片(分片模式下过滤分片索引)
+set_env("HIVE_MONGO_SHARDING", 0)
+--是否自动构建mongo索引(线上建议手动)
+set_env("HIVE_AUTO_BUILD_MONGO_INDEX", 1)
+--mongo索引创建失败是否删除重建
+set_env("HIVE_MONGO_INDEX_REBUILD", 1)

script/agent/mongo_agent.lua

@@ -1,19 +1,20 @@
 --mongo_agent.lua
-local tunpack      = table.unpack
-local check_failed = hive.failed
-local log_err      = logger.err
-local log_info     = logger.info
-local log_warn     = logger.warn
-local mrandom      = math_ext.random
-local tequals      = table_ext.equals
-local tequal_keys  = table_ext.equal_keys
-local KernCode     = enum("KernCode")
-local router_mgr   = hive.load("router_mgr")
-local scheduler    = hive.load("scheduler")
-
-local AUTOINCCC    = environ.get("HIVE_DB_AUTOINCTB", "autoinctb")
-local MongoAgent   = singleton()
-local prop         = property(MongoAgent)
+local tunpack       = table.unpack
+local check_failed  = hive.failed
+local log_err       = logger.err
+local log_info      = logger.info
+local log_warn      = logger.warn
+local mrandom       = math_ext.random
+local tequals       = table_ext.equals
+local tequal_keys   = table_ext.equal_keys
+local KernCode      = enum("KernCode")
+local router_mgr    = hive.load("router_mgr")
+local scheduler     = hive.load("scheduler")
+local check_success = hive.success
+
+local AUTOINCCC     = environ.get("HIVE_DB_AUTOINCTB", "autoinctb")
+local MongoAgent    = singleton()
+local prop          = property(MongoAgent)
 prop:reader("service", "mongo")
 prop:reader("local_run", false) --本地线程服务
 
@@ -180,19 +181,27 @@ function MongoAgent:update_sheet(sheet_name, primary_id, primary_key, udata, db_
 end
 
 --检测是否建立了索引(only_key仅检测字段名)
-function MongoAgent:check_indexes(key, co_name, db_name, only_key)
+function MongoAgent:check_indexes(index, co_name, db_name, only_key)
     local ok, code, indexs = self:get_indexes(co_name, db_name)
     if check_failed(code, ok) then
         log_err("[MongoAgent][check_indexes] failed:{},{},ok:{},code:{},indexs:{}", co_name, db_name, ok, code, indexs)
         return false
     end
     local cmp_func = only_key and tequal_keys or tequals
     for i, v in ipairs(indexs) do
-        if cmp_func(v.key, key) then
+        if cmp_func(v.key, index.key) then
+            if index.unique and not v.unique then
+                log_warn("[MongoAgent][check_indexes] db:{},table:{},index:{} --> {},is not unique", db_name, co_name, index, v)
+                return false
+            end
+            if index.expireAfterSeconds and v.expireAfterSeconds ~= index.expireAfterSeconds then
+                log_warn("[MongoAgent][check_indexes] db:{},table:{},index:{} --> {},expireAfterSeconds is not equal", db_name, co_name, index, v)
+                return false
+            end
             return true
         end
     end
-    log_warn("[MongoAgent][check_indexes] db:{},table:{},key:{},is not exist indexes:{}", db_name, co_name, key, indexs)
+    log_warn("[MongoAgent][check_indexes] db:{},table:{},index:{},is not exist indexes:{}", db_name, co_name, index, indexs)
     return false
 end
 
@@ -221,6 +230,39 @@ function MongoAgent:push_array_data(selector, co_name, array, data, limit, hash_
     return true
 end
 
+--重建索引
+function MongoAgent:rebuild_create_index(index, table_name, db_name, rebuild)
+    --检测是否创建过索引
+    if self:check_indexes(index, table_name, db_name, false) then
+        log_info("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] is exist index", db_name, table_name, index.name)
+        return true
+    end
+    local query    = { table_name, { index } }
+    local ok, code = self:create_indexes(query, 1, db_name)
+    if check_success(code, ok) then
+        log_info("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] build index success", db_name, table_name, index.name)
+        return true
+    else
+        log_err("[MongoAgent][rebuild_create_index] db[{}],table[{}] build index[{}] fail:{}", db_name, table_name, index, code)
+        if rebuild then
+            ok, code = self:drop_indexes({ table_name, index.name }, 1, db_name)
+            if check_success(code, ok) then
+                log_warn("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] drop index success", db_name, table_name, index.name)
+                ok, code = self:create_indexes(query, 1, db_name)
+                if check_success(code, ok) then
+                    log_info("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] drop and build index success", db_name, table_name, index.name)
+                    return true
+                else
+                    log_err("[MongoAgent][rebuild_create_index] db[{}],table[{}] drop and build index[{}] fail:{}", db_name, table_name, index, code)
+                end
+            else
+                log_err("[MongoAgent][rebuild_create_index] db[{}],table[{}],key[{}] drop index fail:{}", db_name, table_name, index.name, code)
+            end
+        end
+    end
+    return false
+end
+
 hive.mongo_agent = MongoAgent()
 
 return MongoAgent

script/agent/rmsg_agent.lua

@@ -20,9 +20,9 @@ function RmsgAgent:setup()
     end
 end
 
-function RmsgAgent:build_index(sharding)
+function RmsgAgent:build_index(sharding, rebuild)
     for _, rmsg in pairs(self.rmsgs or {}) do
-        rmsg:build_index(sharding)
+        rmsg:build_index(sharding, rebuild)
     end
 end
 

script/store/reliable_msg.lua

@@ -23,7 +23,7 @@ function ReliableMsg:__init(db_name, table_name, due_day)
     self.table_name = table_name
 end
 
-function ReliableMsg:build_index(sharding)
+function ReliableMsg:generate_index(sharding)
     local indexs = {
         { key = { uuid = 1 }, name = "uuid", unique = (not sharding), background = true },
     }
@@ -33,29 +33,27 @@ function ReliableMsg:build_index(sharding)
     if self.ttl then
         tinsert(indexs, { key = { ttl = 1 }, expireAfterSeconds = 0, name = "ttl", unique = false, background = true })
     end
-    local query    = { self.table_name, indexs }
-    local ok, code = mongo_agent:create_indexes(query, nil, self.db_name)
-    if check_success(code, ok) then
-        log_info("[ReliableMsg][build_index] rmsg table:{},sharding:{} build due index success", self.table_name, sharding)
-    else
-        log_err("[ReliableMsg][build_index] rmsg table:{},sharding:{} build due index failed", self.table_name, sharding)
+    return indexs
+end
+
+function ReliableMsg:build_index(sharding, rebuild)
+    local indexs = self:generate_index(sharding)
+    for _, index in pairs(indexs) do
+        if not sharding or index.name ~= "to" then
+            mongo_agent:rebuild_create_index(index, self.table_name, self.db_name, rebuild)
+        end
     end
 end
 
 function ReliableMsg:check_indexes(sharding)
+    local indexs  = self:generate_index(sharding)
     local success = true
     local miss    = {}
-    if not mongo_agent:check_indexes({ uuid = 1 }, self.table_name, self.db_name, false) then
-        success = false
-        tinsert(miss, { db = self.db_name, co = self.table_name, key = { uuid = 1 } })
-    end
-    if not mongo_agent:check_indexes({ to = 1 }, self.table_name, self.db_name, sharding) then
-        success = false
-        tinsert(miss, { db = self.db_name, co = self.table_name, key = { to = 1 } })
-    end
-    if not mongo_agent:check_indexes({ ttl = 1 }, self.table_name, self.db_name, false) then
-        success = false
-        tinsert(miss, { db = self.db_name, co = self.table_name, key = { ttl = 1 }, expireAfterSeconds = 0 })
+    for _, index in ipairs(indexs) do
+        if not mongo_agent:check_indexes(index, self.table_name, self.db_name, sharding) then
+            success = false
+            tinsert(miss, { db = self.db_name, co = self.table_name, index = index })
+        end
     end
     return success, miss
 end

server/monitor/dbindex_mgr.lua

@@ -5,7 +5,7 @@ local tjoin         = table_ext.join
 local log_err       = logger.err
 local log_info      = logger.info
 local log_warn      = logger.warn
-local check_success = hive.success
+
 local monitor       = hive.get("monitor")
 local mongo_agent   = hive.get("mongo_agent")
 local thread_mgr    = hive.get("thread_mgr")
@@ -15,6 +15,8 @@ local config_mgr    = hive.get("config_mgr")
 
 local GMType        = enum("GMType")
 
+local dbindex_db    = config_mgr:init_table("dbindex", "db_name", "table_name", "name")
+
 local DBIndexMgr    = singleton()
 local prop          = property(DBIndexMgr)
 prop:accessor("status", 0)
@@ -57,14 +59,12 @@ end
 function DBIndexMgr:build_index(rebuild)
     log_info("[DBIndexMgr][build_index] rebuild:{}", rebuild)
     self:build_dbindex(rebuild)
-    rmsg_agent:build_index(self.sharding)
+    rmsg_agent:build_index(self.sharding, rebuild)
 end
 
 function DBIndexMgr:check_dbindexes()
-    log_info("[DBIndexMgr][check_dbindexes]")
-    local success    = true
-    local miss       = {}
-    local dbindex_db = config_mgr:init_table("dbindex", "db_name", "table_name", "name")
+    local success = true
+    local miss    = {}
     for _, conf in dbindex_db:iterator() do
         local db_name    = conf.db_name
         local table_name = conf.table_name
@@ -76,70 +76,55 @@ function DBIndexMgr:check_dbindexes()
         if self.sharding and conf.sharding then
             only_key = true
         end
-        local check_key = {}
-        for _, v in ipairs(conf.keys) do
-            check_key[v] = 1
-        end
-        if not mongo_agent:check_indexes(check_key, table_name, db_name, only_key) then
+        local index = self:generate_index(conf)
+        if not mongo_agent:check_indexes(index, table_name, db_name, only_key) then
             success = false
-            tinsert(miss, { db = db_name, co = table_name, key = check_key })
+            tinsert(miss, { db = db_name, co = table_name, index = index })
         end
     end
     local ok, res = rmsg_agent:check_indexes(self.sharding)
     if not ok then
         miss    = tjoin(res, miss)
         success = false
     end
+    log_warn("[DBIndexMgr][check_dbindexes] ret:{},miss:{}", success, miss)
     return success, miss
 end
 
+-- 检测表是否分片
+function DBIndexMgr:get_key_unique(db_name, table_name, unique)
+    if self.sharding then
+        for _, conf in dbindex_db:iterator() do
+            if db_name == conf.db_name and table_name == conf.table_name and conf.sharding then
+                return false
+            end
+        end
+    end
+    return unique
+end
+
+function DBIndexMgr:generate_index(conf)
+    local index      = {}
+    index.key        = {}
+    index.name       = conf.name
+    index.unique     = self:get_key_unique(conf.db_name, conf.table_name, conf.unique)
+    index.background = true
+    if conf.expireAfterSeconds > 0 then
+        index.expireAfterSeconds = conf.expireAfterSeconds
+    end
+    for _, v in ipairs(conf.keys) do
+        index.key[v] = 1
+    end
+    return index
+end
+
 function DBIndexMgr:build_dbindex(rebuild)
     log_info("[DBIndexMgr][build_dbindex] rebuild:{},sharding:{}", rebuild, self.sharding)
-    local dbindex_db = config_mgr:init_table("dbindex", "db_name", "table_name", "name")
     for _, conf in dbindex_db:iterator() do
-        if self.sharding and conf.sharding then
-            goto continue
-        end
-        local db_name    = conf.db_name
-        local table_name = conf.table_name
-        local index      = {}
-        index.key        = {}
-        index.name       = conf.name
-        index.unique     = (not self.sharding) and conf.unique or false
-        index.background = true
-        if conf.expireAfterSeconds > 0 then
-            index.expireAfterSeconds = conf.expireAfterSeconds
-        end
-        for _, v in ipairs(conf.keys) do
-            index.key[v] = 1
-        end
-        --检测是否创建过索引
-        if mongo_agent:check_indexes(index.key, table_name, db_name, false) then
-            log_info("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] is exist index", db_name, table_name, index.name)
-            goto continue
-        end
-        local query    = { table_name, { index } }
-        local ok, code = mongo_agent:create_indexes(query, 1, db_name)
-        if check_success(code, ok) then
-            log_info("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] build index success", db_name, table_name, index.name)
-        else
-            log_err("[DBIndexMgr][build_index] db[{}],table[{}] build index[{}] fail:{}", db_name, table_name, index, code)
-            if rebuild then
-                ok, code = mongo_agent:drop_indexes({ table_name, index.name }, 1, db_name)
-                if check_success(code, ok) then
-                    log_warn("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] drop index success", db_name, table_name, index.name)
-                    ok, code = mongo_agent:create_indexes(query, 1, db_name)
-                    if check_success(code, ok) then
-                        log_info("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] drop and build index success", db_name, table_name, index.name)
-                    else
-                        log_err("[DBIndexMgr][build_index] db[{}],table[{}] drop and build index[{}] fail:{}", db_name, table_name, index, code)
-                    end
-                else
-                    log_err("[DBIndexMgr][build_index] db[{}],table[{}],key[{}] drop index fail:{}", db_name, table_name, index.name, code)
-                end
-            end
+        if not self.sharding or not conf.sharding then
+            local index = self:generate_index(conf)
+            mongo_agent:rebuild_create_index(index, conf.table_name, conf.db_name, rebuild)
         end
-        :: continue ::
     end
 end