105 release packages can be installed in anywhere of file system (#118)

* dtcd.sh rename * conf: move to relative path. #105 * release: add libsqlparser.so to package. * release: fix bugs. * add key-hash.so * release: Reduced version. * agent: fix bugs. * agent-watchdog: fix bug. * sh: add agent.sh #102 * release: env update to ubuntu-20.04 * show tables: change layered route from L3 to L2 #98
jd-opensource · Sep 8, 2022 · 2192abe · 2192abe
1 parent 1cba51c
commit 2192abe
Show file tree

Hide file tree

Showing 34 changed files with 99 additions and 70 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -52,15 +52,23 @@ jobs:
           cp src/hwcserver/hwcserver dtc/bin/
           cp src/rule/librule.so dtc/bin/
           cp src/utils/conf-gen-utils dtc/bin/
-          cp ../src/libs/api/cc_api/libs/libdtcapi.so dtc/bin/
+          cp ../src/libs/api/cc_api/libs/libdtcapi.so dtc/bin/libdtc.so.1
+          cp ../src/libs/hsql/libs/libsqlparser.so dtc/bin/
           cp ../script/dtc.sh dtc/bin/
+          cp ../script/agent.sh dtc/bin/
+          cp ../src/libs/api/cc_api/libs/key-hash.so dtc/lib/
+
+          chmod +x dtc/bin/dtc.sh
+          chmod +x dtc/bin/agent.sh
 
           wget "https://dlcdn.apache.org/shardingsphere/5.1.2/apache-shardingsphere-5.1.2-shardingsphere-proxy-bin.tar.gz" -O dtc/sharding.tar.gz
           tar -zxvf dtc/sharding.tar.gz -C dtc
           mv dtc/apache-shardingsphere-5.1.2-shardingsphere-proxy-bin dtc/sharding
           rm dtc/sharding.tar.gz
           cp ../dockerfiles/devel/mysql-connector-java-5.1.49.jar dtc/sharding/lib/mysql-connector-java-5.1.49.jar
 
+          cp ../conf/log4cplus.conf dtc/conf/
+
           tar -zcvf dtc.tar.gz dtc/
 
       - name: Create Release

diff --git a/conf/log4cplus.conf b/conf/log4cplus.conf
@@ -7,7 +7,7 @@ log4cplus.appender.STDOUT.layout.ConversionPattern=[%T] %D{%Y-%m-%d %H:%M:%S} %-
 
 ## synchronous log properties.
 log4cplus.appender.apdPlatform=log4cplus::TimeBasedRollingFileAppender
-log4cplus.appender.apdPlatform.FilenamePattern=/var/log/dtc/dtc.%d{yyyyMMdd}.log
+log4cplus.appender.apdPlatform.FilenamePattern=../log/dtc.%d{yyyyMMdd}.log
 log4cplus.appender.apdPlatform.Append=true
 log4cplus.appender.apdPlatform.MaxHistory=999
 log4cplus.appender.apdPlatform.ImmediateFlush=false

diff --git a/dockerfiles/agent/dockerfile b/dockerfiles/agent/dockerfile
@@ -1,8 +1,8 @@
 FROM centos:centos8
 
 ARG basepath=/usr/local/dtc
-ARG confpath=/etc/dtc
-ARG logpath=/var/log/dtc
+ARG confpath=$basepath/conf
+ARG logpath=$basepath/log
 
 RUN mkdir -p $basepath/bin
 RUN mkdir -p $confpath

diff --git a/dockerfiles/devel/run.sh b/dockerfiles/devel/run.sh
@@ -5,8 +5,8 @@ sleep_count=0
 while [ $sleep_count -le 100 ]
 do
     if [ -f "/usr/local/dtc/conf/dtc.yaml" ]; then 
-        mkdir /etc/dtc
-        cp /usr/local/dtc/conf/log4cplus.conf /etc/dtc
+        mkdir /usr/local/dtc/conf
+        cp /usr/local/dtc/conf/log4cplus.conf /usr/local/dtc/conf/
         echo "Start running process: "$DTC_BIN","$DTC_ARGV
         cd /usr/local/dtc/bin/
         ./$DTC_BIN $DTC_ARGV
@@ -18,7 +18,3 @@ do
     fi
 done
 echo "Timeout waitting for dtc conf files."
-
-#cd /usr/local/lib/
-#ln -snf libdtc.so.1  libdtc.so
-#ln -snf libdtcapi.so  libdtc.so.1
diff --git a/dockerfiles/server/dockerfile b/dockerfiles/server/dockerfile
@@ -1,15 +1,13 @@
 FROM centos:centos8
 
 ARG basepath=/usr/local/dtc
-ARG confpath=/etc/dtc
-ARG logpath=/var/log/dtc
+ARG confpath=$basepath/conf
 
 RUN mkdir -p $basepath/bin
 RUN mkdir -p $basepath/data
 RUN mkdir -p $basepath/stat
 RUN mkdir -p $basepath/log
 RUN mkdir -p $confpath
-RUN mkdir -p $logpath
 
 COPY core $basepath/bin/core
 COPY hwcserver $basepath/bin/hwcserver

diff --git a/docs/configure.md b/docs/configure.md
@@ -1,11 +1,10 @@
 
 ## 配置文件
-配置文件目录：/etc/dtc<br/>
+配置文件目录：conf/<br/>
 - AGENT服务的配置文件是：
   * agent.xml 指定后端dtc server的主、备服务的地址信息和权重；
 - DTC服务的配置文件是：
-  * dtc.yaml 配置缓存模式，缓存大小等。
-  * table.yaml 表结构的定义、数据库连接信息在此配置。
+  * dtc.yaml dtc模块的配置文件，包括基础配置和表结构信息。
 
 下面就各配置文件具体配置选项做介绍：
 ### agent.xml
@@ -38,10 +37,7 @@ agent配置文件位置为当前项目的conf/agent.xml，主要配置以下字
 * sharding:
 该字段用户设置分库分表的信息，key字段用于设置依照此字段进行分片。table字段用于设置分表的信息，例如分表名为opensource_0/opensource_1.....opensouce_9，则只需要设置为{prefix: [*table, _], start: 0, last: 9}
 
-#### extension 扩展库，提供多租户功能
-在dtc的基础缓存和数据代理功能之外，还提供了扩展库，通过配置此模块能够在数据库中进行复杂查询和分库分表功能。
-
-### table.yaml
+#### table
 cache配置文件位置为当前项目的conf/table.yaml，主要配置以下字段：
 * TABLE_CONF.table_name
 * TABLE_CONF.field_count
@@ -50,4 +46,7 @@ cache配置文件位置为当前项目的conf/table.yaml，主要配置以下字
   * FIELD*ID*.field_name 字段名
   * FIELD*ID*.field_type 字段类型：1.整数型 2.无符号整数形 3.浮点型 4.字符串（大小写不明感） 5.字符串（大小写敏感）
   * FIELD*ID*.field_size 字段长度。定义了该字段的大小。例如int型可配置为4字节，long型可配置为8字节，字符串类型可根据具体使用场景配置长度，但最大长度不得超过64KB。
-  * FIELD*ID*.field_unique [可选]默认值0，可配置0或1。当为0时表示该字段的值不唯一，1时表示该字段值唯一。
+  * FIELD*ID*.field_unique [可选]默认值0，可配置0或1。当为0时表示该字段的值不唯一，1时表示该字段值唯一。
+
+#### extension 扩展库，提供多租户功能
+在dtc的基础缓存和数据代理功能之外，还提供了扩展库，通过配置此模块能够在数据库中进行复杂查询和分库分表功能。
diff --git a/docs/quickstart.md b/docs/quickstart.md
@@ -1,9 +1,9 @@
 ## 目录结构
 Bin文件目录：/usr/local/dtc<br/>
-配置文件目录：/etc/dtc<br/>
-日志目录：/var/log/<br/>
+配置文件目录：/usr/local/dtc/conf/<br/>
+日志目录：/usr/local/dtc/log/<br/>
 统计数据目录：/usr/local/dtc/stat/<br/>
-binlog数据目录：/usr/local/log/<br/>
+binlog数据目录：/usr/local/dtc/log/<br/>
 
 ## DTC模式
 
@@ -29,7 +29,7 @@ demo中定义的表名为dtc_opensource, <br/>
 为了省去配置环境的麻烦，Demo中提供docker镜像，直接运行即可启动服务端：<br/>
   ```shell
   docker pull dtc8/dtc:latest
-  docker run --rm --name dtc -p <MY_LISTENER_PORT>:12001 -v <MY_HOST_CONF_DIR>:/etc/dtc/ -e DTC_BIN=dtc -e DTC_ARGV=-ayc dtc8/dtc
+  docker run --rm --name dtc -p <MY_LISTENER_PORT>:12001 -v <MY_HOST_CONF_DIR>:/usr/local/dtc/conf/ -e DTC_BIN=dtc -e DTC_ARGV=-ayc dtc8/dtc
   ```
 
 ## 运行Client测试示例
@@ -78,8 +78,7 @@ mkdir -p basepath
 mkdir -p /usr/local/dtc/data
 mkdir -p /usr/local/dtc/stat
 mkdir -p /usr/local/dtc/log
-mkdir -p /etc/dtc
-mkdir -p /var/log/dtc
+mkdir -p /usr/local/dtc/conf
 ```
 * 将bin文件拷贝到/usr/local/dtc文件夹，并赋执行权限
 ```

diff --git a/script/agent.sh b/script/agent.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+ulimit -c unlimited
+
+DTC_BIN="agent-watchdog"
+
+if [ "$1" = "stop" ] ; then
+    pkill -9 $DTC_BIN
+    pkill -9 dtcagent
+    pkill -9 async-connector
+    pkill -9 data-lifecycle
+    ../sharding/bin/stop.sh
+elif [ "$1" = "restart" ]; then
+    pkill -9 $DTC_BIN
+    pkill -9 dtcagent
+    pkill -9 async-connector
+    pkill -9 data-lifecycle
+    ../sharding/bin/stop.sh
+    sleep 2
+    ./$DTC_BIN  >> /dev/null 2>&1
+elif [ "$1" = "start" ]; then
+    ./$DTC_BIN  >> /dev/null 2>&1 &
+    sleep 1
+else
+    echo "usage: $0 stop | start |restart"
+fi
diff --git a/src/agent-watchdog/async_conn_entry.cc b/src/agent-watchdog/async_conn_entry.cc
@@ -18,7 +18,7 @@
 #include <unistd.h>
 #include "log.h"
 
-#define ROOT_PATH "/etc/dtc/"
+#define ROOT_PATH "../conf/"
 const char *fulldata_name = "async-connector";
 extern std::map<std::string, std::string> map_dtc_conf; //key:value --> dtc addr:conf file name
 

diff --git a/src/agent-watchdog/daemons.cc b/src/agent-watchdog/daemons.cc
@@ -86,9 +86,9 @@ void close_sharding()
 {
 	char* p[1];
 	p[0] = NULL;
-	execv("/usr/local/agent/sharding/bin/stop.sh", p);
+	execv("../sharding/bin/stop.sh", p);
 
-	system("/usr/local/agent/sharding/bin/stop.sh");
+	system("../sharding/bin/stop.sh");
 }
 
 void WatchDog::run_loop()

diff --git a/src/agent-watchdog/log.h b/src/agent-watchdog/log.h
@@ -66,7 +66,7 @@ using namespace log4cplus::helpers;
 
 static Logger logger = Logger::getRoot();
 
-#define LOG4CPLUS_CONF_FILE "/etc/dtc/log4cplus.conf"
+#define LOG4CPLUS_CONF_FILE "../conf/log4cplus.conf"
 
 /*********************************************
 **TRACE：调试应用的详细步骤

diff --git a/src/agent-watchdog/main.cc b/src/agent-watchdog/main.cc
@@ -15,8 +15,8 @@
 
 extern char cache_file[256];
 extern char table_file[256];
-#define ROOT_PATH "/etc/dtc/"
-char agent_file[256] = "/etc/dtc/agent.xml";
+#define ROOT_PATH "../conf/"
+char agent_file[256] = "../conf/agent.xml";
 std::map<std::string, std::string> map_dtc_conf; //key:value --> dtc addr:conf file name
 
 #define DA_VERSION_MAJOR	1
@@ -385,7 +385,6 @@ int get_all_dtc_confs()
 		std::string addr = (*it).first;
 		std::string filename = (*it).second;
 
-		//TODO: send select dtcyaml
 		char* content = NULL;
 		int content_len = 0;
 		log4cplus_debug("addr:%s", addr.c_str());
@@ -444,7 +443,7 @@ int main(int argc, char* argv[])
 		exit(0);
 	}
 
-	if(load_agent || load_sharding || load_asyncconn)
+	if(load_agent || load_sharding || load_asyncconn || load_all)
 	{
 		if(get_all_dtc_confs() < 0)
 		{
@@ -463,7 +462,7 @@ int main(int argc, char* argv[])
 			log4cplus_error("start full-data failed.");
 	}
 
-	if (load_core || load_all) {
+	if (load_core) {
 		if(start_core(wdog, delay) < 0)
 			log4cplus_error("start core failed.");
 	}

diff --git a/src/agent-watchdog/sharding_entry.cc b/src/agent-watchdog/sharding_entry.cc
@@ -34,13 +34,13 @@ void ShardingEntry::exec()
 {
 	char *argv[4];
 
-	int ret = system("/usr/local/agent/bin/conf-gen-utils");
+	int ret = system("./conf-gen-utils");
 	if(ret == 0)
 	{
 		set_proc_title("agent_sharding");
-		argv[0] = (char *)"/usr/local/agent/sharding/bin/start.sh";
+		argv[0] = (char *)"../sharding/bin/start.sh";
 		argv[1] = "3307";
-		argv[2] = "/etc/dtc";
+		argv[2] = "../conf";
 		argv[3] = NULL;
 		execv(argv[0], argv);
 	}

diff --git a/src/agent/da.c b/src/agent/da.c
@@ -47,8 +47,8 @@
 #define DA_VERSION_DETAIL \
 	DA_VERSION_STR" built at: "__DATE__" "__TIME__
 
-#define DA_CONF_PATH "/etc/dtc/agent.xml"
-#define DA_LOG_DIR "/var/log/dtc"
+#define DA_CONF_PATH "../conf/agent.xml"
+#define DA_LOG_DIR "../log"
 
 #define DA_LOG_DEFAULT  3
 #define DA_MBUF_SIZE DEF_MBUF_SIZE
@@ -344,7 +344,7 @@ static void show_usage(void) {
 	write_stderr(
 			"  -v, --verbosity=N      		: set logging level (default: 3, min: 1, max: 7)" CRLF
 			"  -o, --output=S         		: set logging dir (default: /var/log/dtc/)" CRLF
-			"  -c, --conf-file=S      		: set configuration file (default: /etc/dtc/agent.xml)" CRLF
+			"  -c, --conf-file=S      		: set configuration file (default: ../conf/agent.xml)" CRLF
 			"  -e, --event-max-timeout=S	: set epoll max timeout(ms)(default: (30*1000)ms)" CRLF
 			"  -i, --stats_interval=S	    : set stats aggregator interval(ms)(default: (10*1000)ms)" CRLF
 			"  -p, --pid-file=S       		: set pid file (default: off)" CRLF

diff --git a/src/agent/my/my_parse.c b/src/agent/my/my_parse.c
@@ -102,8 +102,9 @@ void my_parse_req(struct msg *r)
 			p = b->last;
 			goto end;
 		}
-
+log_debug("1111111111");
 		if (r->owner->stage == CONN_STAGE_LOGGED_IN) {
+			log_debug("1111111111");
 			rc = my_get_command(p, input_packet_length, r,
 					    &command);
 			if (rc) {
@@ -163,12 +164,16 @@ void my_parse_req(struct msg *r)
 						pp++;
 				}
 
-				if(pp - dbstart > 0 && pp - dbstart < 250 && pp - dbstart != strlen("caching_sha2_password") &&
-					 memcmp(pp-dbstart, "caching_sha2_password", strlen("caching_sha2_password")) != 0)
+				if(pp - dbstart > 0 && pp - dbstart < 250)
 				{
-					memcpy(r->owner->dbname, dbstart, pp - dbstart);
-					r->owner->dbname[pp - dbstart] = '\0';
-					log_debug("client set dbname: %s", r->owner->dbname);
+					int len = pp - dbstart;
+					int len_sha2 = strlen("caching_sha2_password");
+					if(len != len_sha2 || (len == len_sha2 && memcmp(dbstart, "caching_sha2_password", len_sha2) != 0))
+					{
+						memcpy(r->owner->dbname, dbstart, len);
+						r->owner->dbname[len] = '\0';
+						log_debug("client set dbname: %s", r->owner->dbname);
+					}
 				}
 			}
 			else
@@ -660,7 +665,7 @@ int my_get_route_key(uint8_t *sql, int sql_len, int *start_offset,
 	char conf_path[260] = {0};
 	if(mid != 0)
 	{
-		sprintf(conf_path, "/etc/dtc/dtc-conf-%d.yaml", mid);
+		sprintf(conf_path, "../conf/dtc-conf-%d.yaml", mid);
 		r->mid = mid;
 	}
 

diff --git a/src/complex/main.cc b/src/complex/main.cc
@@ -33,7 +33,7 @@ static PollerBase* workerThread;
 CTransactionGroup* FullDBGroup = NULL;
 CTransactionGroup* HotDBGroup = NULL;
 
-std::string conf_path = "/etc/dtc/dtc.yaml";
+std::string conf_path = "../dtc.yaml";
 
 static int start_main_thread()
 {

diff --git a/src/daemons/helper.cc b/src/daemons/helper.cc
@@ -78,7 +78,7 @@ void WatchDogHelper::exec()
 	if (conf_ == DBHELPER_TABLE_NEW) {
 		argv[argc++] = (char *)"-t";// 4
 		char tableName[64];
-		snprintf(tableName, 64, "/etc/dtc/dtc%d.yaml", num_);
+		snprintf(tableName, 64, "../conf/dtc%d.yaml", num_);
 		argv[argc++] = tableName; // 5
 	} else if (conf_ == DBHELPER_TABLE_ORIGIN &&
 		   strcmp(daemons_table_file, TABLE_CONF_NAME)) {

diff --git a/src/data_lifecycle/data_conf.cc b/src/data_lifecycle/data_conf.cc
@@ -9,7 +9,7 @@
 
 extern DbConfig *dbConfig;
 extern char cache_file[256];
-char agent_file[256] = "/etc/dtc/agent.xml";
+char agent_file[256] = "../conf/agent.xml";
 
 DataConf::DataConf(){
 }
@@ -54,8 +54,8 @@ uint32_t DataConf::Port(){
 
 int DataConf::LoadConfig(int argc, char *argv[]){
     int c;
-    strcpy(table_file, "/etc/dtc/dtc-conf-1000.yaml");
-    strcpy(cache_file, "/etc/dtc/dtc-conf-1000.yaml");
+    strcpy(table_file, "../conf/dtc-conf-1000.yaml");
+    strcpy(cache_file, "../conf/dtc-conf-1000.yaml");
 
     while ((c = getopt(argc, argv, "df:t:hvV")) != -1) {
         switch (c) {

diff --git a/src/hwcserver/CMakeLists.txt b/src/hwcserver/CMakeLists.txt
@@ -19,6 +19,7 @@ INCLUDE_DIRECTORIES (
 )
 
 LINK_DIRECTORIES(
+    .
     ${PROJECT_SOURCE_DIR}/src/libs/api/cc_api/libs/
     ${PROJECT_SOURCE_DIR}/src/libs/mysql/libs
     ${PROJECT_SOURCE_DIR}/src/libs/zlib/lib

diff --git a/src/hwcserver/hwc_global.h b/src/hwcserver/hwc_global.h
@@ -37,7 +37,7 @@
 #define READER_SLEEP_TIME           500	//500ms
 #define READER_RETRY_COUNT          20
 
-#define SYS_CONFIG_FILE             "/etc/dtc/hbp.conf"
+#define SYS_CONFIG_FILE             "../conf/hbp.conf"
 /*
  *   err code
  */

diff --git a/src/libs/api/cc_api/libs/key-hash.so b/src/libs/api/cc_api/libs/key-hash.so
diff --git a/src/libs/common/config/dbconfig.cc b/src/libs/common/config/dbconfig.cc
@@ -592,7 +592,7 @@ int DbConfig::get_dtc_config(YAML::Node dtc_config, DTCConfig* raw, int i_server
         m->role[0].addr = STRDUP(dtc_config["primary"][layer]["real"][i]["addr"].as<string>().c_str());
         m->role[0].user = STRDUP(dtc_config["primary"][layer]["real"][i]["user"].as<string>().c_str());
         m->role[0].pass = STRDUP(dtc_config["primary"][layer]["real"][i]["pwd"].as<string>().c_str());
-        m->role[0].optfile = STRDUP("/etc/dtc/my.conf");
+        m->role[0].optfile = STRDUP("../conf/my.conf");
         log4cplus_info("addr:%s,user:%s" , m->role[0].addr , m->role[0].user);
 
         /* master DB settings */

diff --git a/src/libs/common/config/parse_cluster_config.h b/src/libs/common/config/parse_cluster_config.h
@@ -21,7 +21,7 @@
 #include <map>
 namespace ClusterConfig
 {
-#define CLUSTER_CONFIG_FILE "/etc/dtc/clusterconfig.xml"
+#define CLUSTER_CONFIG_FILE "../conf/clusterconfig.xml"
 struct ClusterNode {
 	std::string name;
 	std::string addr;