From 4439fcae20c03b95f73ce0d5e0ef405216bcb943 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 14 Mar 2023 16:00:30 +0800 Subject: [PATCH 01/25] test: fix bug from coverity scan --- src/benchTmq.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/benchTmq.c b/src/benchTmq.c index c16d2d47..1c1e7cd8 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -99,7 +99,7 @@ static int32_t data_msg_process(TAOS_RES* msg, tmqThreadInfo* pInfo, int32_t msg totalRows++; } - + free(buf); return totalRows; } @@ -122,11 +122,13 @@ static void* tmqConsume(void* arg) { tmq_res_t msgType = tmq_get_res_type(tmqMsg); if (msgType == TMQ_RES_TABLE_META) { errorPrint("consumer id %d get TMQ_RES_TABLE_META mesg.\n", pThreadInfo->id); + taos_free_result(tmqMsg); break; } else if (msgType == TMQ_RES_DATA) { totalRows += data_msg_process(tmqMsg, pThreadInfo, totalMsgs); } else if (msgType == TMQ_RES_METADATA) { errorPrint("consumer id %d get TMQ_RES_METADATA mesg.\n", pThreadInfo->id); + taos_free_result(tmqMsg); break; } From 1131fc1775eeb6e7c16206d5bcd865542a5a17d6 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 17 Mar 2023 13:43:50 +0800 Subject: [PATCH 02/25] test: save info into result file --- src/benchTmq.c | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/benchTmq.c b/src/benchTmq.c index 1c1e7cd8..3e62debd 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -169,7 +169,35 @@ static void* tmqConsume(void* arg) { return NULL; } +void printfTmqConfigIntoFile() { + if (NULL == g_arguments->fpOfInsertResult) { + return; + } + + infoPrintToFile(g_arguments->fpOfInsertResult, "%s\n", "============================================"); + + SConsumerInfo* pConsumerInfo = &g_tmqInfo.consumerInfo; + infoPrintToFile(g_arguments->fpOfInsertResult, "concurrent: %d\n", pConsumerInfo->concurrent); + infoPrintToFile(g_arguments->fpOfInsertResult, "pollDelay: %d\n", pConsumerInfo->pollDelay); + infoPrintToFile(g_arguments->fpOfInsertResult, "groupId: %s\n", pConsumerInfo->groupId); + infoPrintToFile(g_arguments->fpOfInsertResult, "clientId: %s\n", pConsumerInfo->clientId); + infoPrintToFile(g_arguments->fpOfInsertResult, "autoOffsetReset: %s\n", pConsumerInfo->autoOffsetReset); + infoPrintToFile(g_arguments->fpOfInsertResult, "enableAutoCommit: %s\n", pConsumerInfo->enableAutoCommit); + infoPrintToFile(g_arguments->fpOfInsertResult, "autoCommitIntervalMs: %d\n", pConsumerInfo->autoCommitIntervalMs); + infoPrintToFile(g_arguments->fpOfInsertResult, "enableHeartbeatBackground: %s\n", pConsumerInfo->enableHeartbeatBackground); + infoPrintToFile(g_arguments->fpOfInsertResult, "snapshotEnable: %s\n", pConsumerInfo->snapshotEnable); + infoPrintToFile(g_arguments->fpOfInsertResult, "msgWithTableName: %s\n", pConsumerInfo->msgWithTableName); + infoPrintToFile(g_arguments->fpOfInsertResult, "rowsFile: %s\n", pConsumerInfo->rowsFile); + infoPrintToFile(g_arguments->fpOfInsertResult, "expectRows: %d\n", pConsumerInfo->expectRows); + + for (int i = 0; i < pConsumerInfo->topicCount; ++i) { + infoPrintToFile(g_arguments->fpOfInsertResult, "topicName[%d]: %s\n", i, pConsumerInfo->topicName[i]); + infoPrintToFile(g_arguments->fpOfInsertResult, "topicSql[%d]: %s\n", i, pConsumerInfo->topicSql[i]); + } +} + int subscribeTestProcess() { + printfTmqConfigIntoFile(); int ret = 0; SConsumerInfo* pConsumerInfo = &g_tmqInfo.consumerInfo; if (pConsumerInfo->topicCount > 0) { @@ -184,22 +212,24 @@ int subscribeTestProcess() { tmqThreadInfo *infos = benchCalloc(pConsumerInfo->concurrent, sizeof(tmqThreadInfo), true); for (int i = 0; i < pConsumerInfo->concurrent; ++i) { - char tmpBuff[64] = {0}; + char tmpBuff[128] = {0}; tmqThreadInfo * pThreadInfo = infos + i; pThreadInfo->totalMsgs = 0; pThreadInfo->totalRows = 0; pThreadInfo->id = i; - memset(tmpBuff, 0, sizeof(tmpBuff)); - snprintf(tmpBuff, 60, "%s_%d", pConsumerInfo->rowsFile, i); - pThreadInfo->fpOfRowsFile = fopen(pConsumerInfo->rowsFile, "a"); - if (NULL == pThreadInfo->fpOfRowsFile) { - errorPrint("failed to open %s file for save rows\n", pConsumerInfo->rowsFile); - ret = -1; - goto tmq_over; + if (strlen(pConsumerInfo->rowsFile)) { + memset(tmpBuff, 0, sizeof(tmpBuff)); + snprintf(tmpBuff, 64, "%s_%d", pConsumerInfo->rowsFile, i); + pThreadInfo->fpOfRowsFile = fopen(tmpBuff, "a"); + if (NULL == pThreadInfo->fpOfRowsFile) { + errorPrint("failed to open %s file for save rows\n", pConsumerInfo->rowsFile); + ret = -1; + goto tmq_over; + } } - + tmq_conf_t * conf = tmq_conf_new(); tmq_conf_set(conf, "td.connect.user", g_arguments->user); tmq_conf_set(conf, "td.connect.pass", g_arguments->password); From 5b26c6b929c0b19cba744990f0330befcf802982 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 17 Mar 2023 15:04:46 +0800 Subject: [PATCH 03/25] test: modify case switch --- src/benchTmq.c | 17 +++++------------ tests/taosbenchmark/json/tmq.json | 2 +- tests/taosbenchmark/json/tmq_basic.json | 2 +- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/benchTmq.c b/src/benchTmq.c index 3e62debd..bcf8b022 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -75,10 +75,11 @@ static int32_t data_msg_process(TAOS_RES* msg, tmqThreadInfo* pInfo, int32_t msg // infoPrint("topic: %s\n", tmq_get_topic_name(msg)); int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); + const char* tblName = tmq_get_table_name(msg); if (pInfo->fpOfRowsFile) { fprintf(pInfo->fpOfRowsFile, "consumerId: %d, msg index:%d\n", pInfo->id, msgIndex); - fprintf(pInfo->fpOfRowsFile, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + fprintf(pInfo->fpOfRowsFile, "dbName: %s, tblname: %s, topic: %s, vgroupId: %d\n", dbName, tblName != NULL ? tblName : "invalid table", tmq_get_topic_name(msg), vgroupId); } @@ -120,14 +121,10 @@ static void* tmqConsume(void* arg) { TAOS_RES* tmqMsg = tmq_consumer_poll(pThreadInfo->tmq, consumeDelay); if (tmqMsg) { tmq_res_t msgType = tmq_get_res_type(tmqMsg); - if (msgType == TMQ_RES_TABLE_META) { - errorPrint("consumer id %d get TMQ_RES_TABLE_META mesg.\n", pThreadInfo->id); - taos_free_result(tmqMsg); - break; - } else if (msgType == TMQ_RES_DATA) { + if (msgType == TMQ_RES_DATA) { totalRows += data_msg_process(tmqMsg, pThreadInfo, totalMsgs); - } else if (msgType == TMQ_RES_METADATA) { - errorPrint("consumer id %d get TMQ_RES_METADATA mesg.\n", pThreadInfo->id); + } else { + errorPrint("consumer id %d get error msg type: %d.\n", pThreadInfo->id, msgType); taos_free_result(tmqMsg); break; } @@ -155,10 +152,6 @@ static void* tmqConsume(void* arg) { } } - if (0 == running) { - infoPrint("consumer id %d receive stop signal and not continue consume\n", pThreadInfo->id); - } - pThreadInfo->totalMsgs = totalMsgs; pThreadInfo->totalRows = totalRows; diff --git a/tests/taosbenchmark/json/tmq.json b/tests/taosbenchmark/json/tmq.json index 42446d5f..cf0eb1b2 100644 --- a/tests/taosbenchmark/json/tmq.json +++ b/tests/taosbenchmark/json/tmq.json @@ -20,7 +20,7 @@ "experimental.snapshot.enable": "true", "msg.with.table.name": "false", "rows_file": "./consumed_rows1", - "expect_rows": -1, + "expect_rows": 50, "topic_list": [ {"name": "tmq_topic_0", "sql": "select c0 from db.stb;"} ] diff --git a/tests/taosbenchmark/json/tmq_basic.json b/tests/taosbenchmark/json/tmq_basic.json index 53f08d5c..b1ecbb1d 100644 --- a/tests/taosbenchmark/json/tmq_basic.json +++ b/tests/taosbenchmark/json/tmq_basic.json @@ -17,7 +17,7 @@ "auto.commit.interval.ms": 1000, "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", - "msg.with.table.name": "false", + "msg.with.table.name": "true", "rows_file": "./consumed_rows", "expect_rows": -1, "topic_list": [ From 11ba5c8231504b45bac1ef9ec6615be933b7d482 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sat, 18 Mar 2023 10:15:03 +0800 Subject: [PATCH 04/25] test: rand generate groupid when not config --- inc/bench.h | 1 + src/benchJsonOpt.c | 6 ++ src/benchTmq.c | 99 ++++++++++++++++-------- tests/taosbenchmark/json/tmq_basic2.json | 29 +++++++ tests/taosbenchmark/json/tmq_basic3.json | 27 +++++++ tests/taosbenchmark/v3/tmq_case.py | 8 ++ 6 files changed, 139 insertions(+), 31 deletions(-) create mode 100644 tests/taosbenchmark/json/tmq_basic2.json create mode 100644 tests/taosbenchmark/json/tmq_basic3.json diff --git a/inc/bench.h b/inc/bench.h index 3663d73e..8df81d4e 100644 --- a/inc/bench.h +++ b/inc/bench.h @@ -810,6 +810,7 @@ typedef struct SConsumerInfo_S { char* clientId; char* autoOffsetReset; + char* enableManualCommit; char* enableAutoCommit; uint32_t autoCommitIntervalMs; // ms char* enableHeartbeatBackground; diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 6332fffc..f7588858 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1752,6 +1752,12 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { g_tmqInfo.consumerInfo.enableAutoCommit = enableAutoCommit->valuestring; } + tools_cJSON *enableManualCommit = tools_cJSON_GetObjectItem( + tmqInfo, "enable.manual.commit"); + if (tools_cJSON_IsString(enableManualCommit)) { + g_tmqInfo.consumerInfo.enableManualCommit = enableManualCommit->valuestring; + } + tools_cJSON *enableHeartbeatBackground = tools_cJSON_GetObjectItem( tmqInfo, "enable.heartbeat.background"); if (tools_cJSON_IsString(enableHeartbeatBackground)) { diff --git a/src/benchTmq.c b/src/benchTmq.c index bcf8b022..1fa9610f 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -23,6 +23,51 @@ typedef struct { static int running = 1; +static const char g_charset[] = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-_"; + +static void rand_string(char *str, int size) { + str[0] = 0; + if (size > 0) { + // --size; + int n; + for (n = 0; n < size; n++) { + int key = taosRandom() % (unsigned int)(sizeof(g_charset) - 1); + str[n] = g_charset[key]; + } + str[n] = 0; + } +} + + +void printfTmqConfigIntoFile() { + if (NULL == g_arguments->fpOfInsertResult) { + return; + } + + infoPrintToFile(g_arguments->fpOfInsertResult, "%s\n", "============================================"); + + SConsumerInfo* pConsumerInfo = &g_tmqInfo.consumerInfo; + infoPrintToFile(g_arguments->fpOfInsertResult, "concurrent: %d\n", pConsumerInfo->concurrent); + infoPrintToFile(g_arguments->fpOfInsertResult, "pollDelay: %d\n", pConsumerInfo->pollDelay); + infoPrintToFile(g_arguments->fpOfInsertResult, "groupId: %s\n", pConsumerInfo->groupId); + infoPrintToFile(g_arguments->fpOfInsertResult, "clientId: %s\n", pConsumerInfo->clientId); + infoPrintToFile(g_arguments->fpOfInsertResult, "autoOffsetReset: %s\n", pConsumerInfo->autoOffsetReset); + infoPrintToFile(g_arguments->fpOfInsertResult, "enableAutoCommit: %s\n", pConsumerInfo->enableAutoCommit); + infoPrintToFile(g_arguments->fpOfInsertResult, "autoCommitIntervalMs: %d\n", pConsumerInfo->autoCommitIntervalMs); + infoPrintToFile(g_arguments->fpOfInsertResult, "enableHeartbeatBackground: %s\n", pConsumerInfo->enableHeartbeatBackground); + infoPrintToFile(g_arguments->fpOfInsertResult, "snapshotEnable: %s\n", pConsumerInfo->snapshotEnable); + infoPrintToFile(g_arguments->fpOfInsertResult, "msgWithTableName: %s\n", pConsumerInfo->msgWithTableName); + infoPrintToFile(g_arguments->fpOfInsertResult, "rowsFile: %s\n", pConsumerInfo->rowsFile); + infoPrintToFile(g_arguments->fpOfInsertResult, "expectRows: %d\n", pConsumerInfo->expectRows); + + for (int i = 0; i < pConsumerInfo->topicCount; ++i) { + infoPrintToFile(g_arguments->fpOfInsertResult, "topicName[%d]: %s\n", i, pConsumerInfo->topicName[i]); + infoPrintToFile(g_arguments->fpOfInsertResult, "topicSql[%d]: %s\n", i, pConsumerInfo->topicSql[i]); + } +} + + static int create_topic() { SBenchConn* conn = initBenchConn(); if (conn == NULL) { @@ -109,9 +154,15 @@ static void* tmqConsume(void* arg) { int64_t totalMsgs = 0; int64_t totalRows = 0; + int32_t manualCommit = 0; infoPrint("consumer id %d start to loop pull msg\n", pThreadInfo->id); + if ((NULL != g_tmqInfo.consumerInfo.enableManualCommit) && (0 == strncmp("true", g_tmqInfo.consumerInfo.enableManualCommit, 4))) { + manualCommit = 1; + infoPrint("consumer id %d enable manual commit\n", pThreadInfo->id); + } + int64_t lastTotalMsgs = 0; int64_t lastTotalRows = 0; uint64_t lastPrintTime = toolsGetTimestampMs(); @@ -129,6 +180,10 @@ static void* tmqConsume(void* arg) { break; } + if (0 != manualCommit) { + tmq_commit_sync(pThreadInfo->tmq, tmqMsg); + } + taos_free_result(tmqMsg); totalMsgs++; @@ -162,33 +217,6 @@ static void* tmqConsume(void* arg) { return NULL; } -void printfTmqConfigIntoFile() { - if (NULL == g_arguments->fpOfInsertResult) { - return; - } - - infoPrintToFile(g_arguments->fpOfInsertResult, "%s\n", "============================================"); - - SConsumerInfo* pConsumerInfo = &g_tmqInfo.consumerInfo; - infoPrintToFile(g_arguments->fpOfInsertResult, "concurrent: %d\n", pConsumerInfo->concurrent); - infoPrintToFile(g_arguments->fpOfInsertResult, "pollDelay: %d\n", pConsumerInfo->pollDelay); - infoPrintToFile(g_arguments->fpOfInsertResult, "groupId: %s\n", pConsumerInfo->groupId); - infoPrintToFile(g_arguments->fpOfInsertResult, "clientId: %s\n", pConsumerInfo->clientId); - infoPrintToFile(g_arguments->fpOfInsertResult, "autoOffsetReset: %s\n", pConsumerInfo->autoOffsetReset); - infoPrintToFile(g_arguments->fpOfInsertResult, "enableAutoCommit: %s\n", pConsumerInfo->enableAutoCommit); - infoPrintToFile(g_arguments->fpOfInsertResult, "autoCommitIntervalMs: %d\n", pConsumerInfo->autoCommitIntervalMs); - infoPrintToFile(g_arguments->fpOfInsertResult, "enableHeartbeatBackground: %s\n", pConsumerInfo->enableHeartbeatBackground); - infoPrintToFile(g_arguments->fpOfInsertResult, "snapshotEnable: %s\n", pConsumerInfo->snapshotEnable); - infoPrintToFile(g_arguments->fpOfInsertResult, "msgWithTableName: %s\n", pConsumerInfo->msgWithTableName); - infoPrintToFile(g_arguments->fpOfInsertResult, "rowsFile: %s\n", pConsumerInfo->rowsFile); - infoPrintToFile(g_arguments->fpOfInsertResult, "expectRows: %d\n", pConsumerInfo->expectRows); - - for (int i = 0; i < pConsumerInfo->topicCount; ++i) { - infoPrintToFile(g_arguments->fpOfInsertResult, "topicName[%d]: %s\n", i, pConsumerInfo->topicName[i]); - infoPrintToFile(g_arguments->fpOfInsertResult, "topicSql[%d]: %s\n", i, pConsumerInfo->topicSql[i]); - } -} - int subscribeTestProcess() { printfTmqConfigIntoFile(); int ret = 0; @@ -201,6 +229,15 @@ int subscribeTestProcess() { tmq_list_t * topic_list = buildTopicList(); + char groupId[16] = {0}; + if ((NULL == pConsumerInfo->groupId) || (0 == strlen(pConsumerInfo->groupId))) { + // rand string + memset(groupId, 0, sizeof(groupId)); + rand_string(groupId, sizeof(groupId) - 1); + infoPrint("rand generate group id: %s\n", groupId); + pConsumerInfo->groupId = groupId; + } + pthread_t * pids = benchCalloc(pConsumerInfo->concurrent, sizeof(pthread_t), true); tmqThreadInfo *infos = benchCalloc(pConsumerInfo->concurrent, sizeof(tmqThreadInfo), true); @@ -277,10 +314,10 @@ int subscribeTestProcess() { for (int i = 0; i < pConsumerInfo->concurrent; i++) { tmqThreadInfo * pThreadInfo = infos + i; int32_t code; - code = tmq_unsubscribe(pThreadInfo->tmq); - if (code != 0) { - errorPrint("thread %d tmq_unsubscribe() fail, reason: %s\n", i, tmq_err2str(code)); - } + //code = tmq_unsubscribe(pThreadInfo->tmq); + //if (code != 0) { + // errorPrint("thread %d tmq_unsubscribe() fail, reason: %s\n", i, tmq_err2str(code)); + //} code = tmq_consumer_close(pThreadInfo->tmq); if (code != 0) { diff --git a/tests/taosbenchmark/json/tmq_basic2.json b/tests/taosbenchmark/json/tmq_basic2.json new file mode 100644 index 00000000..5f61d6e7 --- /dev/null +++ b/tests/taosbenchmark/json/tmq_basic2.json @@ -0,0 +1,29 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "result_file": "./tmq_result.txt", + "tmq_info": { + "concurrent": 2, + "poll_delay": 3000, + "group.id": "", + "client.id": "clientId", + "auto.offset.reset": "earliest", + "enable.manual.commit": "true", + "enable.auto.commit": "true", + "auto.commit.interval.ms": 1000, + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", + "rows_file": "./consumed_rows", + "expect_rows": -1, + "topic_list": [ + {"name": "tmq_topic_0", "sql": "select * from db.stb;"}, + {"name": "tmq_topic_1", "sql": "select * from db.stb"} + ] + } +} diff --git a/tests/taosbenchmark/json/tmq_basic3.json b/tests/taosbenchmark/json/tmq_basic3.json new file mode 100644 index 00000000..153754dd --- /dev/null +++ b/tests/taosbenchmark/json/tmq_basic3.json @@ -0,0 +1,27 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "result_file": "./tmq_result.txt", + "tmq_info": { + "concurrent": 2, + "poll_delay": 3000, + "client.id": "clientId", + "auto.offset.reset": "earliest", + "enable.auto.commit": "true", + "auto.commit.interval.ms": 1000, + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", + "rows_file": "./consumed_rows", + "expect_rows": -1, + "topic_list": [ + {"name": "tmq_topic_0", "sql": "select * from db.stb;"}, + {"name": "tmq_topic_1", "sql": "select * from db.stb"} + ] + } +} diff --git a/tests/taosbenchmark/v3/tmq_case.py b/tests/taosbenchmark/v3/tmq_case.py index 2fcf95a7..f9422b70 100644 --- a/tests/taosbenchmark/v3/tmq_case.py +++ b/tests/taosbenchmark/v3/tmq_case.py @@ -66,6 +66,14 @@ def run(self): tdLog.info("%s" % cmd) os.system("%s" % cmd) time.sleep(5) + cmd = "%s -f ./taosbenchmark/json/tmq_basic2.json " % binPath + tdLog.info("%s" % cmd) + os.system("%s" % cmd) + time.sleep(5) + cmd = "%s -f ./taosbenchmark/json/tmq_basic3.json " % binPath + tdLog.info("%s" % cmd) + os.system("%s" % cmd) + time.sleep(5) # try: # for line in os.popen("ps ax | grep taosBenchmark | grep -v grep"): # fields = line.split() From 2a0a3fb153ab1d245c533c822281c6818fc049af Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sat, 18 Mar 2023 13:10:22 +0800 Subject: [PATCH 05/25] test: share rand_string function --- inc/benchData.h | 1 + src/benchData.c | 2 +- src/benchTmq.c | 23 ++++------------------- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/inc/benchData.h b/inc/benchData.h index 0d25f3cc..b5d42086 100644 --- a/inc/benchData.h +++ b/inc/benchData.h @@ -19,6 +19,7 @@ #include /***** Global variables ******/ /***** Declare functions *****/ +void rand_string(char *str, int size, bool chinese); int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, int disorderRange); int generateRandData(SSuperTable *stbInfo, char *sampleDataBuf, diff --git a/src/benchData.c b/src/benchData.c index c04d2985..0f05ec80 100644 --- a/src/benchData.c +++ b/src/benchData.c @@ -82,7 +82,7 @@ static int usc2utf8(char *p, int unic) { return ret; } -static void rand_string(char *str, int size, bool chinese) { +void rand_string(char *str, int size, bool chinese) { if (chinese) { char *pstr = str; while (size > 0) { diff --git a/src/benchTmq.c b/src/benchTmq.c index 1fa9610f..1af06529 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -10,8 +10,9 @@ * FITNESS FOR A PARTICULAR PURPOSE. */ #include -#include - +#include +#include + typedef struct { tmq_t* tmq; int64_t totalMsgs; @@ -23,22 +24,6 @@ typedef struct { static int running = 1; -static const char g_charset[] = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-_"; - -static void rand_string(char *str, int size) { - str[0] = 0; - if (size > 0) { - // --size; - int n; - for (n = 0; n < size; n++) { - int key = taosRandom() % (unsigned int)(sizeof(g_charset) - 1); - str[n] = g_charset[key]; - } - str[n] = 0; - } -} - void printfTmqConfigIntoFile() { if (NULL == g_arguments->fpOfInsertResult) { @@ -233,7 +218,7 @@ int subscribeTestProcess() { if ((NULL == pConsumerInfo->groupId) || (0 == strlen(pConsumerInfo->groupId))) { // rand string memset(groupId, 0, sizeof(groupId)); - rand_string(groupId, sizeof(groupId) - 1); + rand_string(groupId, sizeof(groupId) - 1, 0); infoPrint("rand generate group id: %s\n", groupId); pConsumerInfo->groupId = groupId; } From 2e948a2c69385854692a9e478bfcf7dcfdedf89b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 18 Mar 2023 14:51:17 +0800 Subject: [PATCH 06/25] fix: workflow test branch --- .github/workflows/3.0-alpine.yml | 2 +- .github/workflows/3.0-windows-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/3.0-alpine.yml b/.github/workflows/3.0-alpine.yml index c8e3608c..df3a610e 100644 --- a/.github/workflows/3.0-alpine.yml +++ b/.github/workflows/3.0-alpine.yml @@ -96,7 +96,7 @@ jobs: || github.event_name == 'push' shell: alpine.sh --root {0} run: | - git clone --branch fix/sangshuduo/TD-23043-windows-socket-close --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 + git clone --branch 3.0 --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DBUILD_HTTP=false && make -j8 && make install if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taos ]] then diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml index 28de8506..83ad87ca 100644 --- a/.github/workflows/3.0-windows-build.yml +++ b/.github/workflows/3.0-windows-build.yml @@ -62,7 +62,7 @@ jobs: with: repository: 'taosdata/TDengine' path: 'TDengine' - ref: 'chore/sangshuduo/TD-19593-gcc12-support' + ref: '3.0' - name: create debug directory if: | From bc0fec3319123362121ab88dfddb144b4159822a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 28 Mar 2023 17:06:36 +0800 Subject: [PATCH 07/25] test: close tmq before thread exit --- src/benchTmq.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/benchTmq.c b/src/benchTmq.c index 1af06529..77af2fc9 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -10,9 +10,9 @@ * FITNESS FOR A PARTICULAR PURPOSE. */ #include -#include -#include - +#include +#include + typedef struct { tmq_t* tmq; int64_t totalMsgs; @@ -195,6 +195,19 @@ static void* tmqConsume(void* arg) { pThreadInfo->totalMsgs = totalMsgs; pThreadInfo->totalRows = totalRows; + int32_t code; + //code = tmq_unsubscribe(pThreadInfo->tmq); + //if (code != 0) { + // errorPrint("thread %d tmq_unsubscribe() fail, reason: %s\n", i, tmq_err2str(code)); + //} + + code = tmq_consumer_close(pThreadInfo->tmq); + if (code != 0) { + errorPrint("thread %d tmq_consumer_close() fail, reason: %s\n", + pThreadInfo->id, tmq_err2str(code)); + } + pThreadInfo->tmq = NULL; + infoPrint("consumerId: %d, consume msgs: %" PRId64 ", consume rows: %" PRId64 "\n", pThreadInfo->id, totalMsgs, totalRows); infoPrintToFile(g_arguments->fpOfInsertResult, "consumerId: %d, consume msgs: %" PRId64 ", consume rows: %" PRId64 "\n", pThreadInfo->id, totalMsgs, totalRows); @@ -298,18 +311,6 @@ int subscribeTestProcess() { for (int i = 0; i < pConsumerInfo->concurrent; i++) { tmqThreadInfo * pThreadInfo = infos + i; - int32_t code; - //code = tmq_unsubscribe(pThreadInfo->tmq); - //if (code != 0) { - // errorPrint("thread %d tmq_unsubscribe() fail, reason: %s\n", i, tmq_err2str(code)); - //} - - code = tmq_consumer_close(pThreadInfo->tmq); - if (code != 0) { - errorPrint("thread %d tmq_consumer_close() fail, reason: %s\n", - i, tmq_err2str(code)); - } - pThreadInfo->tmq = NULL; if (pThreadInfo->fpOfRowsFile) { fclose(pThreadInfo->fpOfRowsFile); From 110f826693e6c3a0f5d28b74f0a4d09d9ba1657d Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 29 Mar 2023 11:51:12 +0800 Subject: [PATCH 08/25] test: modify tmq test case by wal --- tests/taosbenchmark/v3/default_tmq_json.py | 3 +++ tests/taosbenchmark/v3/tmq_case.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tests/taosbenchmark/v3/default_tmq_json.py b/tests/taosbenchmark/v3/default_tmq_json.py index 9d74f887..7b31818e 100644 --- a/tests/taosbenchmark/v3/default_tmq_json.py +++ b/tests/taosbenchmark/v3/default_tmq_json.py @@ -61,6 +61,9 @@ def run(self): tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") + + tdSql.execute("alter database db WAL_RETENTION_PERIOD 3600000") + cmd = "%s -f ./taosbenchmark/json/tmq.json " % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) diff --git a/tests/taosbenchmark/v3/tmq_case.py b/tests/taosbenchmark/v3/tmq_case.py index f9422b70..458b50c9 100644 --- a/tests/taosbenchmark/v3/tmq_case.py +++ b/tests/taosbenchmark/v3/tmq_case.py @@ -61,11 +61,13 @@ def run(self): cmd = "%s -f ./taosbenchmark/json/default.json" % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) + tdSql.execute("alter database db WAL_RETENTION_PERIOD 3600000") tdSql.execute("reset query cache") cmd = "%s -f ./taosbenchmark/json/tmq_basic.json " % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) time.sleep(5) + cmd = "%s -f ./taosbenchmark/json/tmq_basic2.json " % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) From 2ba2b0d481a5b79046892f52ccbc559edaa8e0c9 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 12 Apr 2023 14:00:04 +0800 Subject: [PATCH 09/25] test: del snapshotEnable config --- src/benchJsonOpt.c | 6 ------ src/benchTmq.c | 2 -- 2 files changed, 8 deletions(-) diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 849ccef8..946f7d01 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1774,12 +1774,6 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { enableHeartbeatBackground->valuestring; } - tools_cJSON *snapshotEnable = tools_cJSON_GetObjectItem( - tmqInfo, "experimental.snapshot.enable"); - if (tools_cJSON_IsString(snapshotEnable)) { - g_tmqInfo.consumerInfo.snapshotEnable = snapshotEnable->valuestring; - } - tools_cJSON *msgWithTableName = tools_cJSON_GetObjectItem( tmqInfo, "msg.with.table.name"); if (tools_cJSON_IsString(msgWithTableName)) { diff --git a/src/benchTmq.c b/src/benchTmq.c index 77af2fc9..b289bcb5 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -41,7 +41,6 @@ void printfTmqConfigIntoFile() { infoPrintToFile(g_arguments->fpOfInsertResult, "enableAutoCommit: %s\n", pConsumerInfo->enableAutoCommit); infoPrintToFile(g_arguments->fpOfInsertResult, "autoCommitIntervalMs: %d\n", pConsumerInfo->autoCommitIntervalMs); infoPrintToFile(g_arguments->fpOfInsertResult, "enableHeartbeatBackground: %s\n", pConsumerInfo->enableHeartbeatBackground); - infoPrintToFile(g_arguments->fpOfInsertResult, "snapshotEnable: %s\n", pConsumerInfo->snapshotEnable); infoPrintToFile(g_arguments->fpOfInsertResult, "msgWithTableName: %s\n", pConsumerInfo->msgWithTableName); infoPrintToFile(g_arguments->fpOfInsertResult, "rowsFile: %s\n", pConsumerInfo->rowsFile); infoPrintToFile(g_arguments->fpOfInsertResult, "expectRows: %d\n", pConsumerInfo->expectRows); @@ -281,7 +280,6 @@ int subscribeTestProcess() { tmq_conf_set(conf, "auto.commit.interval.ms", tmpBuff); tmq_conf_set(conf, "enable.heartbeat.background", pConsumerInfo->enableHeartbeatBackground); - tmq_conf_set(conf, "experimental.snapshot.enable", pConsumerInfo->snapshotEnable); tmq_conf_set(conf, "msg.with.table.name", pConsumerInfo->msgWithTableName); pThreadInfo->tmq = tmq_consumer_new(conf, NULL, 0); From 26e30a603694ef2605cff2c08f11364ac99f9cc9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 1 May 2023 09:48:31 +0800 Subject: [PATCH 10/25] test: refine 3.0-non-x86 workflow (#653) * test: install official taospy only in workflows * fix: 2.x-taosbenchmark-debug.yml * test: fix 3.0-non-x64.yml --- .github/workflows/3.0-alpine.yml | 2 +- .github/workflows/3.0-non-x64.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/3.0-alpine.yml b/.github/workflows/3.0-alpine.yml index 0392ea46..94647343 100644 --- a/.github/workflows/3.0-alpine.yml +++ b/.github/workflows/3.0-alpine.yml @@ -99,7 +99,7 @@ jobs: shell: alpine.sh --root {0} run: | - git clone --branch fix/sangshuduo/taosbenchmark-stb-rest-socket-main --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 + git clone --branch main --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DBUILD_HTTP=false && make -j8 && make install if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taos ]] then diff --git a/.github/workflows/3.0-non-x64.yml b/.github/workflows/3.0-non-x64.yml index 5ecb1cb0..67eb7c50 100644 --- a/.github/workflows/3.0-non-x64.yml +++ b/.github/workflows/3.0-non-x64.yml @@ -172,7 +172,7 @@ jobs: apt install python3 python3-pip -y > /dev/null pip3 install --upgrade pip > /dev/null - pip3 install numpy psutil pandas faker toml > /dev/null + pip3 install decorator numpy psutil pandas faker toml > /dev/null pip3 install taospy > /dev/null pip3 install fabric2 > /dev/null From 50c685cabb3e9ad8e66c90c8376108fa5b0a8817 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 2 May 2023 23:03:59 +0800 Subject: [PATCH 11/25] fix: typos (#654) --- src/benchInsertMix.c | 10 +++++----- src/benchJsonOpt.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/benchInsertMix.c b/src/benchInsertMix.c index 5bd09a09..55d74dd3 100644 --- a/src/benchInsertMix.c +++ b/src/benchInsertMix.c @@ -48,13 +48,13 @@ typedef struct { // need generate count , calc from stb->insertRows * ratio uint64_t genCnt[MCNT]; - // status aleady done count + // status already done count uint64_t doneCnt[MCNT]; // task out from batch to list buffer TSKEY* buf[MCNT]; - // bufer cnt + // buffer cnt uint64_t capacity[MCNT]; // capacity size for buf uint64_t bufCnt[MCNT]; // current valid cnt in buf @@ -124,7 +124,7 @@ void mixRatioExit(SMixRatio* mix) { // --------------------- util ---------------- // -// return true can do execute delelte sql +// return true can do execute delete sql bool needExecDel(SMixRatio* mix) { if (mix->genCnt[MDEL] == 0 || mix->doneCnt[MDEL] >= mix->genCnt[MDEL]) { return false; @@ -708,7 +708,7 @@ bool checkCorrect(threadInfo* info, SDataBase* db, SSuperTable* stb, char* tbNam return false; } if (count != calcCount) { - errorPrint("checkCorrect query count unexpect, tbname=%s query=%" PRId64 " expect=%" PRId64, tbName, count, + errorPrint("checkCorrect query count unexpected, tbname=%s query=%" PRId64 " expect=%" PRId64, tbName, count, calcCount); return false; @@ -735,7 +735,7 @@ bool checkCorrect(threadInfo* info, SDataBase* db, SSuperTable* stb, char* tbNam // check count correct if (ts != lastTs) { - errorPrint("checkCorrect query last unexpect, tbname=%s query last=%" PRId64 " expect=%" PRId64, tbName, ts, + errorPrint("checkCorrect query last unexpected, tbname=%s query last=%" PRId64 " expect=%" PRId64, tbName, ts, lastTs); return false; } diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index da4de9be..41592b42 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -322,7 +322,7 @@ static int getDatabaseInfo(tools_cJSON *dbinfos, int index) { SDbCfg* cfg = benchCalloc(1, sizeof(SDbCfg), true); cfg->name = cfg_object->string; - // get duration vallue + // get duration value if (0 == strcasecmp(cfg_object->string, "duration")) { database->durMinute = getDurationVal(cfg_object); } @@ -707,7 +707,7 @@ static int getStableInfo(tools_cJSON *dbinfos, int index) { superTable->startTimestamp = toolsGetTimestamp(database->precision); superTable->useNow = true; - // fill time with now conflict wih check_sql + // fill time with now conflict with check_sql g_arguments->check_sql = false; } else { if (toolsParseTime(ts->valuestring, From 887117a5ca486ffbd3133bb6779ed7ee7d573d54 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 16 May 2023 16:39:16 +0800 Subject: [PATCH 12/25] test: add printf insert rate --- src/benchInsert.c | 7168 +++++++++++++++++++++++---------------------- 1 file changed, 3587 insertions(+), 3581 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index aa4f381c..a1660539 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1,3581 +1,3587 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the MIT license as published by the Free Software - * Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - */ - -#include -#include -#include - -#define FREE_PIDS_INFOS_RETURN_MINUS_1() \ - do { \ - tmfree(pids); \ - tmfree(infos); \ - return -1; \ - } while (0) - -#define FREE_RESOURCE() \ - do { \ - if (pThreadInfo->conn) \ - closeBenchConn(pThreadInfo->conn); \ - benchArrayDestroy(pThreadInfo->delayList); \ - tmfree(pids); \ - tmfree(infos); \ - } while (0) \ - -static int getSuperTableFromServerRest( - SDataBase* database, SSuperTable* stbInfo, char *command) { - - return -1; - // TODO(me): finish full implementation -#if 0 - int sockfd = createSockFd(); - if (sockfd < 0) { - return -1; - } - - int code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - - destroySockFd(sockfd); -#endif // 0 -} - -static int getSuperTableFromServerTaosc( - SDataBase* database, SSuperTable* stbInfo, char *command) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - return -1; - } -#endif - TAOS_RES * res; - TAOS_ROW row = NULL; - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - - res = taos_query(conn->taos, command); - int32_t code = taos_errno(res); - if (code != 0) { - printWarnCmdCodeStr(command, code, res); - infoPrint("stable %s does not exist, will create one\n", - stbInfo->stbName); - closeBenchConn(conn); - return -1; - } - infoPrint("find stable<%s>, will get meta data from server\n", - stbInfo->stbName); - benchArrayClear(stbInfo->tags); - benchArrayClear(stbInfo->cols); - int count = 0; - while ((row = taos_fetch_row(res)) != NULL) { - if (count == 0) { - count++; - continue; - } - int32_t *lengths = taos_fetch_lengths(res); - if (lengths == NULL) { - errorPrint("%s", "failed to execute taos_fetch_length\n"); - taos_free_result(res); - closeBenchConn(conn); - return -1; - } - if (strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "tag", - strlen("tag")) == 0) { - Field* tag = benchCalloc(1, sizeof(Field), true); - benchArrayPush(stbInfo->tags, tag); - tag = benchArrayGet(stbInfo->tags, stbInfo->tags->size - 1); - tag->type = convertStringToDatatype( - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); - tag->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tag->min = convertDatatypeToDefaultMin(tag->type); - tag->max = convertDatatypeToDefaultMax(tag->type); - tstrncpy(tag->name, - (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); - } else { - Field * col = benchCalloc(1, sizeof(Field), true); - benchArrayPush(stbInfo->cols, col); - col = benchArrayGet(stbInfo->cols, stbInfo->cols->size - 1); - col->type = convertStringToDatatype( - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); - col->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - col->min = convertDatatypeToDefaultMin(col->type); - col->max = convertDatatypeToDefaultMax(col->type); - tstrncpy(col->name, - (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); - } - } - taos_free_result(res); - closeBenchConn(conn); - return 0; -} - -static int getSuperTableFromServer(SDataBase* database, SSuperTable* stbInfo) { - int ret = 0; - - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "DESCRIBE `%s`.`%s`", database->dbName, - stbInfo->stbName); - - if (REST_IFACE == stbInfo->iface) { - ret = getSuperTableFromServerRest(database, stbInfo, command); - } else { - ret = getSuperTableFromServerTaosc(database, stbInfo, command); - } - - return ret; -} - -static int queryDbExec(SDataBase *database, - SSuperTable *stbInfo, char *command) { - int ret = 0; - if (REST_IFACE == stbInfo->iface) { - if (0 != convertServAddr(stbInfo->iface, false, 1)) { - errorPrint("%s", "Failed to convert server address\n"); - return -1; - } - int sockfd = createSockFd(); - if (sockfd < 0) { - ret = -1; - } else { - ret = queryDbExecRest(command, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - sockfd); - destroySockFd(sockfd); - } - } else { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - ret = -1; - } else { - ret = queryDbExecCall(conn, command); - int32_t trying = g_arguments->keep_trying; - while (ret && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then re-create " - "supertable %s\n", - g_arguments->trying_interval, stbInfo->stbName); - toolsMsleep(g_arguments->trying_interval); - ret = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - if (0 != ret) { - errorPrint("create supertable %s failed!\n\n", - stbInfo->stbName); - ret = -1; - } - closeBenchConn(conn); - } - } - - return ret; -} - -#ifdef WEBSOCKET -static void dropSuperTable(SDataBase* database, SSuperTable* stbInfo) { - if (g_arguments->supplementInsert) { - return; - } - - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - snprintf(command, sizeof(command), - g_arguments->escape_character - ? "DROP TABLE `%s`.`%s`" - : "DROP TABLE %s.%s", - database->dbName, - stbInfo->stbName); - - infoPrint("drop stable: <%s>\n", command); - queryDbExec(database, stbInfo, command); - - return; -} -#endif // WEBSOCKET - -static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { - if (g_arguments->supplementInsert) { - return 0; - } - - uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->cols->size; - char *colsBuf = benchCalloc(1, col_buffer_len, false); - char* command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - int len = 0; - - for (int colIndex = 0; colIndex < stbInfo->cols->size; colIndex++) { - Field * col = benchArrayGet(stbInfo->cols, colIndex); - int n; - if (col->type == TSDB_DATA_TYPE_BINARY || - col->type == TSDB_DATA_TYPE_NCHAR) { - n = snprintf(colsBuf + len, col_buffer_len - len, - ",%s %s(%d)", col->name, - convertDatatypeToString(col->type), col->length); - } else { - n = snprintf(colsBuf + len, col_buffer_len - len, - ",%s %s", col->name, - convertDatatypeToString(col->type)); - } - if (n < 0 || n >= col_buffer_len - len) { - errorPrint("%s() LN%d, snprintf overflow on %d\n", - __func__, __LINE__, colIndex); - break; - } else { - len += n; - } - } - - // save for creating child table - stbInfo->colsOfCreateChildTable = - (char *)benchCalloc(len + TIMESTAMP_BUFF_LEN, 1, true); - - snprintf(stbInfo->colsOfCreateChildTable, len + TIMESTAMP_BUFF_LEN, - "(ts timestamp%s)", colsBuf); - - if (stbInfo->tags->size == 0) { - free(colsBuf); - free(command); - return 0; - } - - uint32_t tag_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->tags->size; - char *tagsBuf = benchCalloc(1, tag_buffer_len, false); - int tagIndex; - len = 0; - - int n; - n = snprintf(tagsBuf + len, tag_buffer_len - len, "("); - if (n < 0 || n >= tag_buffer_len - len) { - errorPrint("%s() LN%d snprintf overflow\n", - __func__, __LINE__); - free(colsBuf); - free(command); - tmfree(tagsBuf); - return -1; - } else { - len += n; - } - for (tagIndex = 0; tagIndex < stbInfo->tags->size; tagIndex++) { - Field *tag = benchArrayGet(stbInfo->tags, tagIndex); - if (tag->type == TSDB_DATA_TYPE_BINARY || - tag->type == TSDB_DATA_TYPE_NCHAR) { - n = snprintf(tagsBuf + len, tag_buffer_len - len, - "%s %s(%d),", tag->name, - convertDatatypeToString(tag->type), tag->length); - } else if (tag->type == TSDB_DATA_TYPE_JSON) { - n = snprintf(tagsBuf + len, tag_buffer_len - len, - "%s json", tag->name); - if (n < 0 || n >= tag_buffer_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, tagIndex); - break; - } else { - len += n; - } - goto skip; - } else { - n = snprintf(tagsBuf + len, tag_buffer_len - len, - "%s %s,", tag->name, - convertDatatypeToString(tag->type)); - } - - if (n < 0 || n >= tag_buffer_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, tagIndex); - break; - } else { - len += n; - } - } - len -= 1; -skip: - snprintf(tagsBuf + len, tag_buffer_len - len, ")"); - - int length = snprintf( - command, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` (ts TIMESTAMP%s) TAGS %s" - : "CREATE TABLE %s.%s (ts TIMESTAMP%s) TAGS %s", - database->dbName, stbInfo->stbName, colsBuf, tagsBuf); - tmfree(colsBuf); - tmfree(tagsBuf); - if (stbInfo->comment != NULL) { - length += snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, - " COMMENT '%s'", stbInfo->comment); - } - if (stbInfo->delay >= 0) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, " DELAY %d", - stbInfo->delay); - } - if (stbInfo->file_factor >= 0) { - length += - snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, " FILE_FACTOR %f", - (float)stbInfo->file_factor / 100); - } - if (stbInfo->rollup != NULL) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " ROLLUP(%s)", stbInfo->rollup); - } - - if (stbInfo->max_delay != NULL) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " MAX_DELAY %s", stbInfo->max_delay); - } - - if (stbInfo->watermark != NULL) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " WATERMARK %s", stbInfo->watermark); - } - - if (stbInfo->ttl != 0) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " TTL %d", stbInfo->ttl); - } - - bool first_sma = true; - for (int i = 0; i < stbInfo->cols->size; i++) { - Field * col = benchArrayGet(stbInfo->cols, i); - if (col->sma) { - if (first_sma) { - n = snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " SMA(%s", col->name); - first_sma = false; - } else { - n = snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - ",%s", col->name); - } - - if (n < 0 || n > TSDB_MAX_ALLOWED_SQL_LEN - length) { - errorPrint("%s() LN%d snprintf overflow on %d iteral\n", - __func__, __LINE__, i); - break; - } else { - length += n; - } - } - } - if (!first_sma) { - snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, ")"); - } - infoPrint("create stable: <%s>\n", command); - - int ret = queryDbExec(database, stbInfo, command); - free(command); - return ret; -} - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 -int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { - int vgroups = 0; - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "USE `%s`" - : "USE %s", - database->dbName); - - int32_t code; - TAOS_RES *res = NULL; - - res = taos_query(conn->taos, cmd); - code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - taos_free_result(res); - - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SHOW VGROUPS"); - res = taos_query(conn->taos, cmd); - code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - - TAOS_ROW row = NULL; - while ((row = taos_fetch_row(res)) != NULL) { - vgroups++; - } - debugPrint("%s() LN%d, vgroups: %d\n", __func__, __LINE__, vgroups); - taos_free_result(res); - - database->vgroups = vgroups; - database->vgArray = benchArrayInit(vgroups, sizeof(SVGroup)); - for (int32_t v = 0; (v < vgroups - && !g_arguments->terminate); v++) { - SVGroup *vg = benchCalloc(1, sizeof(SVGroup), true); - benchArrayPush(database->vgArray, vg); - } - - res = taos_query(conn->taos, cmd); - code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - - int32_t vgItem = 0; - while (((row = taos_fetch_row(res)) != NULL) - && !g_arguments->terminate) { - SVGroup *vg = benchArrayGet(database->vgArray, vgItem); - vg->vgId = *(int32_t*)row[0]; - vgItem++; - } - taos_free_result(res); - - return vgroups; -} -#endif // TD_VER_COMPATIBLE_3_0_0_0 - -int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { - int dataLen = 0; - int n; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto || (-1 != g_arguments->inputted_vgroups)) { - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s` VGROUPS %d" - : "CREATE DATABASE IF NOT EXISTS %s VGROUPS %d", - database->dbName, - (-1 != g_arguments->inputted_vgroups)? - g_arguments->inputted_vgroups: - min(remainVnodes, toolsGetNumberOfCores())); - } else { - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s`" - : "CREATE DATABASE IF NOT EXISTS %s", - database->dbName); - } -#else - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s`" - : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); -#endif // TD_VER_COMPATIBLE_3_0_0_0 - if (n < 0 || n >= SHORT_1K_SQL_BUFF_LEN - dataLen) { - errorPrint("%s() LN%d snprintf overflow\n", - __func__, __LINE__); - return -1; - } else { - dataLen += n; - } - - if (database->cfgs) { - for (int i = 0; i < database->cfgs->size; i++) { - SDbCfg* cfg = benchArrayGet(database->cfgs, i); - if (cfg->valuestring) { - n = snprintf(command + dataLen, - TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " %s %s", cfg->name, cfg->valuestring); - } else { - n = snprintf(command + dataLen, - TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " %s %d", cfg->name, cfg->valueint); - } - if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - dataLen) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, i); - break; - } else { - dataLen += n; - } - } - } - - switch (database->precision) { - case TSDB_TIME_PRECISION_MILLI: - snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " PRECISION \'ms\';"); - break; - case TSDB_TIME_PRECISION_MICRO: - snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " PRECISION \'us\';"); - break; - case TSDB_TIME_PRECISION_NANO: - snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " PRECISION \'ns\';"); - break; - } - - return dataLen; -} - -int createDatabaseRest(SDataBase* database) { - int32_t code = 0; - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - - int sockfd = createSockFd(); - if (sockfd < 0) { - return -1; - } - - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "DROP DATABASE IF EXISTS `%s`;" - : "DROP DATABASE IF EXISTS %s;", - database->dbName); - code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - if (code != 0) { - errorPrint("Failed to drop database %s\n", database->dbName); - } else { - int remainVnodes = INT_MAX; - geneDbCreateCmd(database, command, remainVnodes); - code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - int32_t trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then " - "re-create database %s\n", - g_arguments->trying_interval, database->dbName); - toolsMsleep(g_arguments->trying_interval); - code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - if (trying != -1) { - trying--; - } - } - } - destroySockFd(sockfd); - return code; -} - -int32_t getRemainVnodes(SBenchConn *conn) { - int remainVnodes = 0; - char command[SHORT_1K_SQL_BUFF_LEN] = "SHOW DNODES"; - - TAOS_RES *res = taos_query(conn->taos, command); - int32_t code = taos_errno(res); - if (code) { - printErrCmdCodeStr(command, code, res); - closeBenchConn(conn); - return -1; - } - TAOS_ROW row = NULL; - while ((row = taos_fetch_row(res)) != NULL) { - remainVnodes += (*(int16_t*)(row[3]) - *(int16_t*)(row[2])); - } - debugPrint("%s() LN%d, remainVnodes: %d\n", - __func__, __LINE__, remainVnodes); - taos_free_result(res); - return remainVnodes; -} - -int createDatabaseTaosc(SDataBase* database) { - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - if (g_arguments->taosc_version == 3) { - for (int i = 0; i < g_arguments->streams->size; i++) { - SSTREAM* stream = benchArrayGet(g_arguments->streams, i); - if (stream->drop) { - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "DROP STREAM IF EXISTS %s;", - stream->stream_name); - if (queryDbExecCall(conn, command)) { - closeBenchConn(conn); - return -1; - } - infoPrint("%s\n", command); - memset(command, 0, SHORT_1K_SQL_BUFF_LEN); - } - } - } - - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "DROP DATABASE IF EXISTS `%s`;": - "DROP DATABASE IF EXISTS %s;", - database->dbName); - if (0 != queryDbExecCall(conn, command)) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - warnPrint("%s", "TDengine cloud normal users have no privilege " - "to drop database! DROP DATABASE failure is ignored!\n"); - } else { -#endif - closeBenchConn(conn); - return -1; -#ifdef WEBSOCKET - } -#endif - } - - int remainVnodes = INT_MAX; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - remainVnodes = getRemainVnodes(conn); - if (0 >= remainVnodes) { - errorPrint("Remain vnodes %d, failed to create database\n", - remainVnodes); - return -1; - } - } -#endif - geneDbCreateCmd(database, command, remainVnodes); - - int32_t code = queryDbExecCall(conn, command); - int32_t trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then " - "re-create database %s\n", - g_arguments->trying_interval, database->dbName); - toolsMsleep(g_arguments->trying_interval); - code = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - - if (code) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - warnPrint("%s", "TDengine cloud normal users have no privilege " - "to create database! CREATE DATABASE " - "failure is ignored!\n"); - } else { -#endif - - closeBenchConn(conn); - errorPrint("\ncreate database %s failed!\n\n", - database->dbName); - return -1; -#ifdef WEBSOCKET - } -#endif - } - infoPrint("command to create database: <%s>\n", command); - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->superTbls) { - if (g_arguments->nthreads_auto) { - int32_t vgroups = getVgroupsOfDb(conn, database); - if (vgroups <=0) { - closeBenchConn(conn); - errorPrint("Database %s's vgroups is %d\n", - database->dbName, vgroups); - return -1; - } - } - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 - - closeBenchConn(conn); - return 0; -} - -int createDatabase(SDataBase* database) { - int ret = 0; - if (REST_IFACE == g_arguments->iface) { - ret = createDatabaseRest(database); - } else { - ret = createDatabaseTaosc(database); - } -#if 0 -#ifdef LINUX - infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); - sleep(10); - infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); -#elif defined(DARWIN) - sleep(2); -#else - Sleep(2); -#endif -#endif - - return ret; -} - -static int generateChildTblName(int len, char *buffer, SDataBase *database, - SSuperTable *stbInfo, uint64_t i, - char *ttl) { - if (0 == len) { - memset(buffer, 0, TSDB_MAX_ALLOWED_SQL_LEN); - len += snprintf(buffer + len, - TSDB_MAX_ALLOWED_SQL_LEN - len, "CREATE TABLE "); - } - - len += snprintf( - buffer + len, TSDB_MAX_ALLOWED_SQL_LEN - len, - g_arguments->escape_character - ? "`%s`.`%s%" PRIu64 "` USING `%s`.`%s` TAGS (%s) %s " - : "%s.%s%" PRIu64 " USING %s.%s TAGS (%s) %s ", - database->dbName, stbInfo->childTblPrefix, i, database->dbName, - stbInfo->stbName, - stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); - - return len; -} - -static int getBatchOfTblCreating(threadInfo *pThreadInfo, - SSuperTable *stbInfo) { - BArray *batchArray = stbInfo->batchTblCreatingNumbersArray; - if (batchArray) { - int *batch = benchArrayGet( - batchArray, pThreadInfo->posOfTblCreatingBatch); - pThreadInfo->posOfTblCreatingBatch++; - if (pThreadInfo->posOfTblCreatingBatch == batchArray->size) { - pThreadInfo->posOfTblCreatingBatch = 0; - } - return *batch; - } - return 0; -} - -static int getIntervalOfTblCreating(threadInfo *pThreadInfo, - SSuperTable *stbInfo) { - BArray *intervalArray = stbInfo->batchTblCreatingIntervalsArray; - if (intervalArray) { - int *interval = benchArrayGet( - intervalArray, pThreadInfo->posOfTblCreatingInterval); - pThreadInfo->posOfTblCreatingInterval++; - if (pThreadInfo->posOfTblCreatingInterval == intervalArray->size) { - pThreadInfo->posOfTblCreatingInterval = 0; - } - return *interval; - } - return 0; -} - -static void *createTable(void *sarg) { - if (g_arguments->supplementInsert) { - return NULL; - } - - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; -#ifdef LINUX - prctl(PR_SET_NAME, "createTable"); -#endif - uint64_t lastPrintTime = toolsGetTimestampMs(); - pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - int len = 0; - int batchNum = 0; - infoPrint( - "thread[%d] start creating table from %" PRIu64 " to %" PRIu64 - "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to); - - char ttl[SMALL_BUFF_LEN] = ""; - if (stbInfo->ttl != 0) { - snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); - } - - int smallBatchCount = 0; - for (uint64_t i = pThreadInfo->start_table_from + stbInfo->childTblFrom; - (i <= (pThreadInfo->end_table_to + stbInfo->childTblFrom) - && !g_arguments->terminate); i++) { - if (g_arguments->terminate) { - goto create_table_end; - } - if (!stbInfo->use_metric || stbInfo->tags->size == 0) { - if (stbInfo->childTblCount == 1) { - snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` %s;" - : "CREATE TABLE %s.%s %s;", - database->dbName, stbInfo->stbName, - stbInfo->colsOfCreateChildTable); - } else { - snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` %s;" - : "CREATE TABLE %s.%s %s;", - database->dbName, - stbInfo->childTblArray[i]->name, - stbInfo->colsOfCreateChildTable); - } - batchNum++; - } else { - if (0 == len) { - batchNum = 0; - } - len = generateChildTblName(len, pThreadInfo->buffer, - database, stbInfo, i, ttl); - - batchNum++; - smallBatchCount++; - - int smallBatch = getBatchOfTblCreating(pThreadInfo, stbInfo); - if ((!smallBatch || (smallBatchCount == smallBatch)) - && (batchNum < stbInfo->batchTblCreatingNum) - && ((TSDB_MAX_ALLOWED_SQL_LEN - len) >= - (stbInfo->lenOfTags + EXTRA_SQL_LEN))) { - continue; - } else { - smallBatchCount = 0; - } - } - - len = 0; - - int ret = 0; - debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, - pThreadInfo->buffer); - if (REST_IFACE == stbInfo->iface) { - ret = queryDbExecRest(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - pThreadInfo->sockfd); - } else { - ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - int32_t trying = g_arguments->keep_trying; - while (ret && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then re-create " - "table %s\n", - g_arguments->trying_interval, pThreadInfo->buffer); - toolsMsleep(g_arguments->trying_interval); - ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - if (trying != -1) { - trying--; - } - } - } - - if (0 != ret) { - g_fail = true; - goto create_table_end; - } - uint64_t intervalOfTblCreating = getIntervalOfTblCreating(pThreadInfo, - stbInfo); - if (intervalOfTblCreating) { - debugPrint("will sleep %"PRIu64" milliseconds " - "for table creating interval\n", intervalOfTblCreating); - toolsMsleep(intervalOfTblCreating); - } - - pThreadInfo->tables_created += batchNum; - batchNum = 0; - uint64_t currentPrintTime = toolsGetTimestampMs(); - if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { - infoPrint( - "thread[%d] already created %" PRId64 " tables\n", - pThreadInfo->threadID, pThreadInfo->tables_created); - lastPrintTime = currentPrintTime; - } - } - - if (0 != len) { - int ret = 0; - debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, - pThreadInfo->buffer); - if (REST_IFACE == stbInfo->iface) { - ret = queryDbExecRest(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - pThreadInfo->sockfd); - } else { - ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - } - if (0 != ret) { - g_fail = true; - goto create_table_end; - } - pThreadInfo->tables_created += batchNum; - debugPrint("thread[%d] already created %" PRId64 " tables\n", - pThreadInfo->threadID, pThreadInfo->tables_created); - } -create_table_end: - tmfree(pThreadInfo->buffer); - pThreadInfo->buffer = NULL; - return NULL; -} - -static int startMultiThreadCreateChildTable( - SDataBase* database, SSuperTable* stbInfo) { - int code = -1; - int threads = g_arguments->table_threads; - int64_t ntables; - if (stbInfo->childTblTo > 0) { - ntables = stbInfo->childTblTo - stbInfo->childTblFrom; - } else { - ntables = stbInfo->childTblCount; - } - pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), false); - threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), false); - uint64_t tableFrom = 0; - if (threads < 1) { - threads = 1; - } - - int64_t a = ntables / threads; - if (a < 1) { - threads = (int)ntables; - a = 1; - } - - if (ntables == 0) { - errorPrint("failed to create child table, childTblCount: %"PRId64"\n", - ntables); - goto over; - } - int64_t b = ntables % threads; - - for (uint32_t i = 0; (i < threads && !g_arguments->terminate); i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - pThreadInfo->stbInfo = stbInfo; - pThreadInfo->dbInfo = database; - if (REST_IFACE == stbInfo->iface) { - int sockfd = createSockFd(); - if (sockfd < 0) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); - } - pThreadInfo->sockfd = sockfd; - } else { - pThreadInfo->conn = initBenchConn(); - if (NULL == pThreadInfo->conn) { - goto over; - } - } - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - pThreadInfo->tables_created = 0; - pthread_create(pids + i, NULL, createTable, pThreadInfo); - } - - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - pthread_join(pids[i], NULL); - } - - if (g_arguments->terminate) toolsMsleep(100); - - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - g_arguments->actualChildTables += pThreadInfo->tables_created; - - if ((REST_IFACE != stbInfo->iface) && pThreadInfo->conn) { - closeBenchConn(pThreadInfo->conn); - } - } - - if (g_fail) { - goto over; - } - code = 0; -over: - free(pids); - free(infos); - return code; -} - -static int createChildTables() { - int32_t code; - infoPrint("start creating %" PRId64 " table(s) with %d thread(s)\n", - g_arguments->totalChildTables, g_arguments->table_threads); - if (g_arguments->fpOfInsertResult) { - infoPrintToFile(g_arguments->fpOfInsertResult, - "start creating %" PRId64 " table(s) with %d thread(s)\n", - g_arguments->totalChildTables, g_arguments->table_threads); - } - double start = (double)toolsGetTimestampMs(); - - for (int i = 0; (i < g_arguments->databases->size - && !g_arguments->terminate); i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (int j = 0; (j < database->superTbls->size - && !g_arguments->terminate); j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - if (stbInfo->autoTblCreating || stbInfo->iface == SML_IFACE - || stbInfo->iface == SML_REST_IFACE) { - g_arguments->autoCreatedChildTables += - stbInfo->childTblCount; - continue; - } - if (stbInfo->childTblExists) { - g_arguments->existedChildTables += - stbInfo->childTblCount; - continue; - } - debugPrint("colsOfCreateChildTable: %s\n", - stbInfo->colsOfCreateChildTable); - - code = startMultiThreadCreateChildTable(database, stbInfo); - if (code && !g_arguments->terminate) { - return code; - } - } - } - } - - double end = (double)toolsGetTimestampMs(); - succPrint( - "Spent %.4f seconds to create %" PRId64 - " table(s) with %d thread(s), already exist %" PRId64 - " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 - " table(s) will be auto created\n", - (end - start) / 1000.0, g_arguments->totalChildTables, - g_arguments->table_threads, g_arguments->existedChildTables, - g_arguments->actualChildTables, - g_arguments->autoCreatedChildTables); - return 0; -} - -static void freeChildTable(SChildTable *childTbl, int colsSize) { - if (childTbl->useOwnSample) { - if (childTbl->childCols) { - for (int col = 0; col < colsSize; col++) { - ChildField *childCol = - benchArrayGet(childTbl->childCols, col); - if (childCol) { - tmfree(childCol->stmtData.data); - tmfree(childCol->stmtData.is_null); - } - } - benchArrayDestroy(childTbl->childCols); - } - tmfree(childTbl->sampleDataBuf); - } - tmfree(childTbl); -} - -void postFreeResource() { - if (!g_arguments->terminate) { - tmfclose(g_arguments->fpOfInsertResult); - } - - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->cfgs) { - for (int c = 0; c < database->cfgs->size; c++) { - SDbCfg *cfg = benchArrayGet(database->cfgs, c); - if ((NULL == root) && (0 == strcmp(cfg->name, "replica"))) { - tmfree(cfg->name); - cfg->name = NULL; - } - } - benchArrayDestroy(database->cfgs); - } - if (database->superTbls) { - for (uint64_t j = 0; j < database->superTbls->size; j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - tmfree(stbInfo->colsOfCreateChildTable); - stbInfo->colsOfCreateChildTable = NULL; - tmfree(stbInfo->sampleDataBuf); - stbInfo->sampleDataBuf = NULL; - tmfree(stbInfo->tagDataBuf); - stbInfo->tagDataBuf = NULL; - tmfree(stbInfo->partialColNameBuf); - stbInfo->partialColNameBuf = NULL; - benchArrayDestroy(stbInfo->batchTblCreatingNumbersArray); - benchArrayDestroy(stbInfo->batchTblCreatingIntervalsArray); - for (int k = 0; k < stbInfo->tags->size; k++) { - Field * tag = benchArrayGet(stbInfo->tags, k); - tmfree(tag->stmtData.data); - tag->stmtData.data = NULL; - } - benchArrayDestroy(stbInfo->tags); - - for (int k = 0; k < stbInfo->cols->size; k++) { - Field * col = benchArrayGet(stbInfo->cols, k); - tmfree(col->stmtData.data); - col->stmtData.data = NULL; - tmfree(col->stmtData.is_null); - col->stmtData.is_null = NULL; - } - if (g_arguments->test_mode == INSERT_TEST) { - if (stbInfo->childTblArray) { - for (int64_t child = 0; child < stbInfo->childTblCount; - child++) { - SChildTable *childTbl = - stbInfo->childTblArray[child]; - if (childTbl) { - freeChildTable(childTbl, stbInfo->cols->size); - } - } - } - } - benchArrayDestroy(stbInfo->cols); - tmfree(stbInfo->childTblArray); - stbInfo->childTblArray = NULL; - benchArrayDestroy(stbInfo->tsmas); -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - tmfree(vg->childTblArray); - vg->childTblArray = NULL; - } - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 - } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->vgArray) - benchArrayDestroy(database->vgArray); -#endif // TD_VER_COMPATIBLE_3_0_0_0 - benchArrayDestroy(database->superTbls); - } - } - benchArrayDestroy(g_arguments->databases); - benchArrayDestroy(g_arguments->streams); - tools_cJSON_Delete(root); -} - -int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - TAOS_RES * res = NULL; - int32_t code = 0; - uint16_t iface = stbInfo->iface; - - int32_t trying = (stbInfo->keep_trying)? - stbInfo->keep_trying:g_arguments->keep_trying; - int32_t trying_interval = stbInfo->trying_interval? - stbInfo->trying_interval:g_arguments->trying_interval; - int protocol = stbInfo->lineProtocol; - - switch (iface) { - case TAOSC_IFACE: - debugPrint("buffer: %s\n", pThreadInfo->buffer); - code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - while (code && trying && !g_arguments->terminate) { - infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", - trying_interval); - toolsMsleep(trying_interval); - code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - if (trying != -1) { - trying--; - } - } - break; - - case REST_IFACE: - debugPrint("buffer: %s\n", pThreadInfo->buffer); - code = postProceSql(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, - pThreadInfo->filePath); - while (code && trying && !g_arguments->terminate) { - infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", - trying_interval); - toolsMsleep(trying_interval); - code = postProceSql(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, - pThreadInfo->filePath); - if (trying != -1) { - trying--; - } - } - break; - - case STMT_IFACE: - code = taos_stmt_execute(pThreadInfo->conn->stmt); - if (code) { - errorPrint( - "failed to execute insert statement. reason: %s\n", - taos_stmt_errstr(pThreadInfo->conn->stmt)); - code = -1; - } - break; - - case SML_IFACE: - res = taos_schemaless_insert( - pThreadInfo->conn->taos, pThreadInfo->lines, - (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) - ? 0 : k, - (SML_JSON_TAOS_FORMAT == protocol) - ? TSDB_SML_JSON_PROTOCOL : protocol, - (TSDB_SML_LINE_PROTOCOL == protocol) - ? database->sml_precision - : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); - code = taos_errno(res); - trying = stbInfo->keep_trying; - while (code && trying && !g_arguments->terminate) { - taos_free_result(res); - infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", - trying_interval); - toolsMsleep(trying_interval); - res = taos_schemaless_insert( - pThreadInfo->conn->taos, pThreadInfo->lines, - (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) - ? 0 : k, - (SML_JSON_TAOS_FORMAT == protocol) - ? TSDB_SML_JSON_PROTOCOL : protocol, - (TSDB_SML_LINE_PROTOCOL == protocol) - ? database->sml_precision - : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); - code = taos_errno(res); - if (trying != -1) { - trying--; - } - } - - if (code != TSDB_CODE_SUCCESS && !g_arguments->terminate) { - debugPrint("Failed to execute " - "schemaless insert content: %s\n\n", - pThreadInfo->lines?(pThreadInfo->lines[0]? - pThreadInfo->lines[0]:""):""); - errorPrint( - "failed to execute schemaless insert. " - "code: 0x%08x reason: %s\n\n", - code, taos_errstr(res)); - } - taos_free_result(res); - break; - - case SML_REST_IFACE: { - if (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) { - code = postProceSql(pThreadInfo->lines[0], database->dbName, - database->precision, stbInfo->iface, - protocol, g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, pThreadInfo->filePath); - } else { - int len = 0; - for (int i = 0; i < k; i++) { - if (strlen(pThreadInfo->lines[i]) != 0) { - int n; - if (TSDB_SML_TELNET_PROTOCOL == protocol - && stbInfo->tcpTransfer) { - n = snprintf(pThreadInfo->buffer + len, - TSDB_MAX_ALLOWED_SQL_LEN - len, - "put %s\n", pThreadInfo->lines[i]); - } else { - n = snprintf(pThreadInfo->buffer + len, - TSDB_MAX_ALLOWED_SQL_LEN - len, - "%s\n", - pThreadInfo->lines[i]); - } - if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, i); - break; - } else { - len += n; - } - } else { - break; - } - } - if (g_arguments->terminate) { - break; - } - code = postProceSql(pThreadInfo->buffer, database->dbName, - database->precision, - stbInfo->iface, protocol, - g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, pThreadInfo->filePath); - } - break; - } - } - return code; -} - -static int smartContinueIfFail(threadInfo *pThreadInfo, - SChildTable *childTbl, - int64_t i, - char *ttl) { - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - char *buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - snprintf( - buffer, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character ? - "CREATE TABLE `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s " - : "CREATE TABLE %s.%s USING %s.%s TAGS (%s) %s ", - database->dbName, childTbl->name, database->dbName, - stbInfo->stbName, - stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); - debugPrint("creating table: %s\n", buffer); - int ret; - if (REST_IFACE == stbInfo->iface) { - ret = queryDbExecRest(buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - pThreadInfo->sockfd); - } else { - ret = queryDbExecCall(pThreadInfo->conn, buffer); - int32_t trying = g_arguments->keep_trying; - while (ret && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then " - "re-create table %s\n", - g_arguments->trying_interval, buffer); - toolsMsleep(g_arguments->trying_interval); - ret = queryDbExecCall(pThreadInfo->conn, buffer); - if (trying != -1) { - trying--; - } - } - } - tmfree(buffer); - - return ret; -} - -static void cleanupAndPrint(threadInfo *pThreadInfo, char *mode) { - if (pThreadInfo) { - if (pThreadInfo->json_array) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - if (0 == pThreadInfo->totalDelay) { - pThreadInfo->totalDelay = 1; - } - succPrint( - "thread[%d] %s mode, completed total inserted rows: %" PRIu64 - ", %.2f records/second\n", - pThreadInfo->threadID, - mode, - pThreadInfo->totalInsertRows, - (double)(pThreadInfo->totalInsertRows / - ((double)pThreadInfo->totalDelay / 1E6))); - } -} - -static int64_t getDisorderTs(SSuperTable *stbInfo, int *disorderRange) { - int64_t disorderTs = 0; - int64_t startTimestamp = stbInfo->startTimestamp; - if (stbInfo->disorderRatio > 0) { - int rand_num = taosRandom() % 100; - if (rand_num < stbInfo->disorderRatio) { - (*disorderRange)--; - if (0 == *disorderRange) { - *disorderRange = stbInfo->disorderRange; - } - disorderTs = startTimestamp - *disorderRange; - debugPrint("rand_num: %d, < disorderRatio: %d, " - "disorderTs: %"PRId64"\n", - rand_num, stbInfo->disorderRatio, - disorderTs); - } - } - return disorderTs; -} - -static void *syncWriteInterlace(void *sarg) { - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - infoPrint( - "thread[%d] start interlace inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to); - - int64_t insertRows = stbInfo->insertRows; - int32_t interlaceRows = stbInfo->interlaceRows; - int64_t pos = 0; - uint32_t batchPerTblTimes = g_arguments->reqPerReq / interlaceRows; - uint64_t lastPrintTime = toolsGetTimestampMs(); - int64_t startTs = toolsGetTimestampUs(); - int64_t endTs; - uint64_t tableSeq = pThreadInfo->start_table_from; - int disorderRange = stbInfo->disorderRange; - - while (insertRows > 0) { - int64_t tmp_total_insert_rows = 0; - uint32_t generated = 0; - if (insertRows <= interlaceRows) { - interlaceRows = insertRows; - } - for (int i = 0; i < batchPerTblTimes; i++) { - if (g_arguments->terminate) { - goto free_of_interlace; - } - int64_t timestamp = pThreadInfo->start_time; - SChildTable *childTbl = stbInfo->childTblArray[tableSeq]; - char * tableName = - stbInfo->childTblArray[tableSeq]->name; - char *sampleDataBuf = childTbl->useOwnSample? - childTbl->sampleDataBuf: - stbInfo->sampleDataBuf; - char ttl[SMALL_BUFF_LEN] = ""; - if (stbInfo->ttl != 0) { - snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); - } - switch (stbInfo->iface) { - case REST_IFACE: - case TAOSC_IFACE: { - char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; - if (g_arguments->escape_character) { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "`%s`", - tableName); - } else { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "%s", - tableName); - } - if (i == 0) { - ds_add_str(&pThreadInfo->buffer, STR_INSERT_INTO); - } - if (stbInfo->partialColNum == stbInfo->cols->size) { - if (stbInfo->autoTblCreating) { - ds_add_strs(&pThreadInfo->buffer, 8, - escapedTbName, - " USING `", - stbInfo->stbName, - "` TAGS (", - stbInfo->tagDataBuf - + stbInfo->lenOfTags * tableSeq, - ") ", ttl, " VALUES "); - } else { - ds_add_strs(&pThreadInfo->buffer, 2, - escapedTbName, " VALUES "); - } - } else { - if (stbInfo->autoTblCreating) { - ds_add_strs(&pThreadInfo->buffer, 10, - escapedTbName, - " (", - stbInfo->partialColNameBuf, - ") USING `", - stbInfo->stbName, - "` TAGS (", - stbInfo->tagDataBuf - + stbInfo->lenOfTags * tableSeq, - ") ", ttl, " VALUES "); - } else { - ds_add_strs(&pThreadInfo->buffer, 4, - escapedTbName, - "(", - stbInfo->partialColNameBuf, - ") VALUES "); - } - } - - for (int64_t j = 0; j < interlaceRows; j++) { - int64_t disorderTs = getDisorderTs(stbInfo, - &disorderRange); - char time_string[BIGINT_BUFF_LEN]; - snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", - disorderTs?disorderTs:timestamp); - ds_add_strs(&pThreadInfo->buffer, 5, - "(", - time_string, - ",", - sampleDataBuf + pos * stbInfo->lenOfCols, - ") "); - if (ds_len(pThreadInfo->buffer) - > stbInfo->max_sql_len) { - errorPrint("sql buffer length (%"PRIu64") " - "is larger than max sql length " - "(%"PRId64")\n", - ds_len(pThreadInfo->buffer), - stbInfo->max_sql_len); - goto free_of_interlace; - } - generated++; - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - timestamp += stbInfo->timestamp_step; - } - break; - } - case STMT_IFACE: { - char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; - if (g_arguments->escape_character) { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, - "`%s`", tableName); - } else { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", - tableName); - } - if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, - escapedTbName)) { - errorPrint( - "taos_stmt_set_tbname(%s) failed, reason: %s\n", - tableName, - taos_stmt_errstr(pThreadInfo->conn->stmt)); - g_fail = true; - goto free_of_interlace; - } - generated = - bindParamBatch(pThreadInfo, interlaceRows, - timestamp, childTbl); - break; - } - case SML_REST_IFACE: - case SML_IFACE: { - int protocol = stbInfo->lineProtocol; - for (int64_t j = 0; j < interlaceRows; j++) { - int64_t disorderTs = getDisorderTs(stbInfo, - &disorderRange); - if (TSDB_SML_JSON_PROTOCOL == protocol) { - tools_cJSON *tag = tools_cJSON_Duplicate( - tools_cJSON_GetArrayItem( - pThreadInfo->sml_json_tags, - (int)tableSeq - - pThreadInfo->start_table_from), - true); - generateSmlJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, - disorderTs?disorderTs:timestamp); - } else if (SML_JSON_TAOS_FORMAT == protocol) { - tools_cJSON *tag = tools_cJSON_Duplicate( - tools_cJSON_GetArrayItem( - pThreadInfo->sml_json_tags, - (int)tableSeq - - pThreadInfo->start_table_from), - true); - generateSmlTaosJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, - disorderTs?disorderTs:timestamp); - } else if (TSDB_SML_LINE_PROTOCOL == protocol) { - snprintf( - pThreadInfo->lines[generated], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %s %" PRId64 "", - pThreadInfo - ->sml_tags[(int)tableSeq - - pThreadInfo->start_table_from], - sampleDataBuf + pos * stbInfo->lenOfCols, - disorderTs?disorderTs:timestamp); - } else { - snprintf( - pThreadInfo->lines[generated], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %" PRId64 " %s %s", stbInfo->stbName, - disorderTs?disorderTs:timestamp, - sampleDataBuf + pos * stbInfo->lenOfCols, - pThreadInfo - ->sml_tags[(int)tableSeq - - pThreadInfo->start_table_from]); - } - generated++; - timestamp += stbInfo->timestamp_step; - } - if (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) { - pThreadInfo->lines[0] = - tools_cJSON_PrintUnformatted( - pThreadInfo->json_array); - } - break; - } - } - tableSeq++; - tmp_total_insert_rows += interlaceRows; - if (tableSeq > pThreadInfo->end_table_to) { - tableSeq = pThreadInfo->start_table_from; - pThreadInfo->start_time += - interlaceRows * stbInfo->timestamp_step; - if (!stbInfo->non_stop) { - insertRows -= interlaceRows; - } - if (stbInfo->insert_interval > 0) { - debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", - __func__, __LINE__, stbInfo->insert_interval); - perfPrint("sleep %" PRIu64 " ms\n", - stbInfo->insert_interval); - toolsMsleep((int32_t)stbInfo->insert_interval); - } - break; - } - } - - startTs = toolsGetTimestampUs(); - if (execInsert(pThreadInfo, generated)) { - g_fail = true; - goto free_of_interlace; - } - endTs = toolsGetTimestampUs(); - - pThreadInfo->totalInsertRows += tmp_total_insert_rows; - - if (g_arguments->terminate) { - goto free_of_interlace; - } - - int protocol = stbInfo->lineProtocol; - switch (stbInfo->iface) { - case TAOSC_IFACE: - case REST_IFACE: - debugPrint("pThreadInfo->buffer: %s\n", - pThreadInfo->buffer); - free_ds(&pThreadInfo->buffer); - pThreadInfo->buffer = new_ds(0); - break; - case SML_REST_IFACE: - memset(pThreadInfo->buffer, 0, - g_arguments->reqPerReq * (pThreadInfo->max_sql_len + 1)); - case SML_IFACE: - if (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) { - debugPrint("pThreadInfo->lines[0]: %s\n", - pThreadInfo->lines[0]); - if (pThreadInfo->json_array && !g_arguments->terminate) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - pThreadInfo->json_array = tools_cJSON_CreateArray(); - if (pThreadInfo->lines && pThreadInfo->lines[0]) { - tmfree(pThreadInfo->lines[0]); - pThreadInfo->lines[0] = NULL; - } - } else { - for (int j = 0; j < generated; j++) { - if (pThreadInfo && pThreadInfo->lines - && !g_arguments->terminate) { - debugPrint("pThreadInfo->lines[%d]: %s\n", j, - pThreadInfo->lines[j]); - memset(pThreadInfo->lines[j], 0, - pThreadInfo->max_sql_len); - } - } - } - break; - case STMT_IFACE: - break; - } - - int64_t delay = endTs - startTs; - if (delay <=0) { - debugPrint("thread[%d]: startTS: %"PRId64", endTS: %"PRId64"\n", - pThreadInfo->threadID, startTs, endTs); - } else { - perfPrint("insert execution time is %10.2f ms\n", - delay / 1E6); - - int64_t * pdelay = benchCalloc(1, sizeof(int64_t), false); - *pdelay = delay; - if (benchArrayPush(pThreadInfo->delayList, pdelay) == NULL) { - tmfree(pdelay); - } - pThreadInfo->totalDelay += delay; - } - - int64_t currentPrintTime = toolsGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30 * 1000) { - infoPrint( - "thread[%d] has currently inserted rows: %" PRIu64 - "\n", - pThreadInfo->threadID, pThreadInfo->totalInsertRows); - lastPrintTime = currentPrintTime; - } - } -free_of_interlace: - cleanupAndPrint(pThreadInfo, "interlace"); - return NULL; -} - -static int32_t prepareProgressDataStmt( - threadInfo *pThreadInfo, - SChildTable *childTbl, - int64_t *timestamp, uint64_t i, char *ttl) { - SSuperTable *stbInfo = pThreadInfo->stbInfo; - char escapedTbName[TSDB_TABLE_NAME_LEN + 2] = "\0"; - if (g_arguments->escape_character) { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN + 2, - "`%s`", childTbl->name); - } else { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", - childTbl->name); - } - if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, - escapedTbName)) { - errorPrint( - "taos_stmt_set_tbname(%s) failed," - "reason: %s\n", escapedTbName, - taos_stmt_errstr(pThreadInfo->conn->stmt)); - return -1; - } - int32_t generated = bindParamBatch( - pThreadInfo, - (g_arguments->reqPerReq > (stbInfo->insertRows - i)) - ? (stbInfo->insertRows - i) - : g_arguments->reqPerReq, - *timestamp, childTbl); - *timestamp += generated * stbInfo->timestamp_step; - return generated; -} - -static void makeTimestampDisorder( - int64_t *timestamp, SSuperTable *stbInfo) { - int64_t startTimestamp = stbInfo->startTimestamp; - int disorderRange = stbInfo->disorderRange; - int rand_num = taosRandom() % 100; - if (rand_num < stbInfo->disorderRatio) { - disorderRange--; - if (0 == disorderRange) { - disorderRange = stbInfo->disorderRange; - } - *timestamp = startTimestamp - disorderRange; - debugPrint("rand_num: %d, < disorderRatio: %d" - ", ts: %"PRId64"\n", - rand_num, - stbInfo->disorderRatio, - *timestamp); - } -} - -static int32_t prepareProgressDataSmlJsonText( - threadInfo *pThreadInfo, - uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { - // prepareProgressDataSmlJsonText - SSuperTable *stbInfo = pThreadInfo->stbInfo; - int32_t generated = 0; - - int len = 0; - - char *line = pThreadInfo->lines[0]; - uint32_t line_buf_len = pThreadInfo->line_buf_len; - - strncat(line + len, "[", 2); - len += 1; - - int32_t pos = 0; - for (int j = 0; (j < g_arguments->reqPerReq) - && !g_arguments->terminate; j++) { - strncat(line + len, "{", 2); - len += 1; - int n; - n = snprintf(line + len, line_buf_len - len, - "\"timestamp\":%"PRId64",", *timestamp); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - - n = snprintf(line + len, line_buf_len - len, "%s", - pThreadInfo->sml_json_value_array[tableSeq]); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - n = snprintf(line + len, line_buf_len - len, "\"tags\":%s,", - pThreadInfo->sml_tags_json_array[tableSeq]); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - n = snprintf(line + len, line_buf_len - len, - "\"metric\":\"%s\"}", stbInfo->stbName); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - *timestamp += stbInfo->timestamp_step; - if (stbInfo->disorderRatio > 0) { - makeTimestampDisorder(timestamp, stbInfo); - } - generated++; - if (i + generated >= stbInfo->insertRows) { - break; - } - if ((j+1) < g_arguments->reqPerReq) { - strncat(line + len, ",", 2); - len += 1; - } - } - strncat(line + len, "]", 2); - - debugPrint("%s() LN%d, lines[0]: %s\n", - __func__, __LINE__, pThreadInfo->lines[0]); - return generated; -} - -static int32_t prepareProgressDataSmlJson( - threadInfo *pThreadInfo, - uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { - // prepareProgressDataSmlJson - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - int32_t generated = 0; - - int32_t pos = 0; - int protocol = stbInfo->lineProtocol; - for (int j = 0; (j < g_arguments->reqPerReq) - && !g_arguments->terminate; j++) { - tools_cJSON *tag = tools_cJSON_Duplicate( - tools_cJSON_GetArrayItem( - pThreadInfo->sml_json_tags, - (int)tableSeq - - pThreadInfo->start_table_from), - true); - debugPrintJsonNoTime(tag); - if (TSDB_SML_JSON_PROTOCOL == protocol) { - generateSmlJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, *timestamp); - } else { - generateSmlTaosJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, *timestamp); - } - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - *timestamp += stbInfo->timestamp_step; - if (stbInfo->disorderRatio > 0) { - makeTimestampDisorder(timestamp, stbInfo); - } - generated++; - if (i + generated >= stbInfo->insertRows) { - break; - } - } - - tmfree(pThreadInfo->lines[0]); - pThreadInfo->lines[0] = NULL; - pThreadInfo->lines[0] = - tools_cJSON_PrintUnformatted( - pThreadInfo->json_array); - debugPrint("pThreadInfo->lines[0]: %s\n", - pThreadInfo->lines[0]); - - return generated; -} - -static int32_t prepareProgressDataSmlLineOrTelnet( - threadInfo *pThreadInfo, uint64_t tableSeq, char *sampleDataBuf, - int64_t *timestamp, uint64_t i, char *ttl, int protocol) { - // prepareProgressDataSmlLine - SSuperTable *stbInfo = pThreadInfo->stbInfo; - int32_t generated = 0; - - int32_t pos = 0; - for (int j = 0; (j < g_arguments->reqPerReq) - && !g_arguments->terminate; j++) { - if (TSDB_SML_LINE_PROTOCOL == protocol) { - snprintf( - pThreadInfo->lines[j], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %s %" PRId64 "", - pThreadInfo->sml_tags[tableSeq - - pThreadInfo->start_table_from], - sampleDataBuf + pos * stbInfo->lenOfCols, - *timestamp); - } else { - snprintf( - pThreadInfo->lines[j], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %" PRId64 " %s %s", stbInfo->stbName, - *timestamp, - sampleDataBuf - + pos * stbInfo->lenOfCols, - pThreadInfo->sml_tags[tableSeq - -pThreadInfo->start_table_from]); - } - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - *timestamp += stbInfo->timestamp_step; - if (stbInfo->disorderRatio > 0) { - makeTimestampDisorder(timestamp, stbInfo); - } - generated++; - if (i + generated >= stbInfo->insertRows) { - break; - } - } - return generated; -} - -static int32_t prepareProgressDataSml( - threadInfo *pThreadInfo, - SChildTable *childTbl, - uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { - // prepareProgressDataSml - SSuperTable *stbInfo = pThreadInfo->stbInfo; - - char *sampleDataBuf; - if (childTbl->useOwnSample) { - sampleDataBuf = childTbl->sampleDataBuf; - } else { - sampleDataBuf = stbInfo->sampleDataBuf; - } - int protocol = stbInfo->lineProtocol; - int32_t generated = -1; - switch (protocol) { - case TSDB_SML_LINE_PROTOCOL: - case TSDB_SML_TELNET_PROTOCOL: - generated = prepareProgressDataSmlLineOrTelnet( - pThreadInfo, - tableSeq, - sampleDataBuf, - timestamp, i, ttl, protocol); - break; - case TSDB_SML_JSON_PROTOCOL: - generated = prepareProgressDataSmlJsonText( - pThreadInfo, - tableSeq - pThreadInfo->start_table_from, - timestamp, i, ttl); - break; - case SML_JSON_TAOS_FORMAT: - generated = prepareProgressDataSmlJson( - pThreadInfo, - tableSeq, - timestamp, i, ttl); - break; - default: - errorPrint("%s() LN%d: unknown protcolor: %d\n", - __func__, __LINE__, protocol); - break; - } - - return generated; -} - -static int32_t prepareProgressDataSql( - threadInfo *pThreadInfo, - SChildTable *childTbl, uint64_t tableSeq, - char *sampleDataBuf, - int64_t *timestamp, uint64_t i, char *ttl, - int32_t *pos, uint64_t *len) { - // prepareProgressDataSql - int32_t generated = 0; - SDataBase *database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - char * pstr = pThreadInfo->buffer; - int disorderRange = stbInfo->disorderRange; - - if (stbInfo->partialColNum == stbInfo->cols->size) { - if (stbInfo->autoTblCreating) { - *len = - snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s VALUES " - : "%s %s.%s USING %s.%s TAGS (%s) %s VALUES ", - STR_INSERT_INTO, database->dbName, - childTbl->name, database->dbName, - stbInfo->stbName, - stbInfo->tagDataBuf + - stbInfo->lenOfTags * tableSeq, ttl); - } else { - *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` VALUES " - : "%s %s.%s VALUES ", - STR_INSERT_INTO, - database->dbName, childTbl->name); - } - } else { - if (stbInfo->autoTblCreating) { - *len = snprintf( - pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` (%s) USING `%s`.`%s` TAGS (%s) %s VALUES " - : "%s %s.%s (%s) USING %s.%s TAGS (%s) %s VALUES ", - STR_INSERT_INTO, database->dbName, - childTbl->name, - stbInfo->partialColNameBuf, - database->dbName, stbInfo->stbName, - stbInfo->tagDataBuf + - stbInfo->lenOfTags * tableSeq, ttl); - } else { - *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` (%s) VALUES " - : "%s %s.%s (%s) VALUES ", - STR_INSERT_INTO, database->dbName, - childTbl->name, - stbInfo->partialColNameBuf); - } - } - - char *ownSampleDataBuf; - if (childTbl->useOwnSample) { - debugPrint("%s is using own sample data\n", - childTbl->name); - ownSampleDataBuf = childTbl->sampleDataBuf; - } else { - ownSampleDataBuf = stbInfo->sampleDataBuf; - } - for (int j = 0; j < g_arguments->reqPerReq; j++) { - if (stbInfo->useSampleTs - && (!stbInfo->random_data_source)) { - *len += - snprintf(pstr + *len, - TSDB_MAX_ALLOWED_SQL_LEN - *len, "(%s)", - sampleDataBuf + - *pos * stbInfo->lenOfCols); - } else { - int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); - *len += snprintf(pstr + *len, - TSDB_MAX_ALLOWED_SQL_LEN - *len, - "(%" PRId64 ",%s)", - disorderTs?disorderTs:*timestamp, - ownSampleDataBuf + - *pos * stbInfo->lenOfCols); - } - *pos += 1; - if (*pos >= g_arguments->prepared_rand) { - *pos = 0; - } - *timestamp += stbInfo->timestamp_step; - generated++; - if (*len > (TSDB_MAX_ALLOWED_SQL_LEN - - stbInfo->lenOfCols)) { - break; - } - if (i + generated >= stbInfo->insertRows) { - break; - } - } - - return generated; -} - -void *syncWriteProgressive(void *sarg) { - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - - // special deal flow for TAOSC_IFACE - if (insertDataMix(pThreadInfo, database, stbInfo)) { - // request be dealt by this function , so return - return NULL; - } - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - if (0 == pThreadInfo->vg->tbCountPerVgId) { - return NULL; - } - } else { - infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); - } -#else - infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); -#endif - uint64_t lastPrintTime = toolsGetTimestampMs(); - int64_t startTs = toolsGetTimestampUs(); - int64_t endTs; - - for (uint64_t tableSeq = pThreadInfo->start_table_from; - tableSeq <= pThreadInfo->end_table_to; tableSeq++) { - char *sampleDataBuf; - SChildTable *childTbl; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - childTbl = pThreadInfo->vg->childTblArray[tableSeq]; - } else { - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; - } -#else - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; -#endif - if (childTbl->useOwnSample) { - sampleDataBuf = childTbl->sampleDataBuf; - } else { - sampleDataBuf = stbInfo->sampleDataBuf; - } - - int64_t timestamp = pThreadInfo->start_time; - uint64_t len = 0; - int32_t pos = 0; - if (stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating) { - taos_stmt_close(pThreadInfo->conn->stmt); - pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); - if (NULL == pThreadInfo->conn->stmt) { - errorPrint("taos_stmt_init() failed, reason: %s\n", - taos_errstr(NULL)); - g_fail = true; - goto free_of_progressive; - } - - if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tableSeq)) { - g_fail = true; - goto free_of_progressive; - } - } - - char ttl[SMALL_BUFF_LEN] = ""; - if (stbInfo->ttl != 0) { - snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); - } - for (uint64_t i = 0; i < stbInfo->insertRows;) { - if (g_arguments->terminate) { - goto free_of_progressive; - } - int32_t generated = 0; - switch (stbInfo->iface) { - case TAOSC_IFACE: - case REST_IFACE: - generated = prepareProgressDataSql( - pThreadInfo, - childTbl, - tableSeq, - sampleDataBuf, - ×tamp, i, ttl, &pos, &len); - break; - case STMT_IFACE: { - generated = prepareProgressDataStmt( - pThreadInfo, - childTbl, ×tamp, i, ttl); - break; - } - case SML_REST_IFACE: - case SML_IFACE: - generated = prepareProgressDataSml( - pThreadInfo, - childTbl, - tableSeq, ×tamp, i, ttl); - break; - default: - break; - } - if (generated < 0) { - g_fail = true; - goto free_of_progressive; - } - if (!stbInfo->non_stop) { - i += generated; - } - // only measure insert - startTs = toolsGetTimestampUs(); - int code = execInsert(pThreadInfo, generated); - if (code) { - if (NO_IF_FAILED == stbInfo->continueIfFail) { - warnPrint("The super table parameter " - "continueIfFail: %d, STOP insertion!\n", - stbInfo->continueIfFail); - g_fail = true; - goto free_of_progressive; - } else if (YES_IF_FAILED == stbInfo->continueIfFail) { - infoPrint("The super table parameter " - "continueIfFail: %d, " - "will continue to insert ..\n", - stbInfo->continueIfFail); - } else if (SMART_IF_FAILED == stbInfo->continueIfFail) { - warnPrint("The super table parameter " - "continueIfFail: %d, will create table " - "then insert ..\n", - stbInfo->continueIfFail); - int ret = smartContinueIfFail( - pThreadInfo, - childTbl, i, ttl); - if (0 != ret) { - g_fail = true; - goto free_of_progressive; - } - - code = execInsert(pThreadInfo, generated); - if (code) { - g_fail = true; - goto free_of_progressive; - } - } else { - warnPrint("Unknown super table parameter " - "continueIfFail: %d\n", - stbInfo->continueIfFail); - g_fail = true; - goto free_of_progressive; - } - } - endTs = toolsGetTimestampUs()+1; - - if (stbInfo->insert_interval > 0) { - debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", - __func__, __LINE__, stbInfo->insert_interval); - perfPrint("sleep %" PRIu64 " ms\n", - stbInfo->insert_interval); - toolsMsleep((int32_t)stbInfo->insert_interval); - } - - pThreadInfo->totalInsertRows += generated; - - if (g_arguments->terminate) { - goto free_of_progressive; - } - int protocol = stbInfo->lineProtocol; - switch (stbInfo->iface) { - case REST_IFACE: - case TAOSC_IFACE: - memset(pThreadInfo->buffer, 0, pThreadInfo->max_sql_len); - break; - case SML_REST_IFACE: - memset(pThreadInfo->buffer, 0, - g_arguments->reqPerReq * - (pThreadInfo->max_sql_len + 1)); - case SML_IFACE: - if (TSDB_SML_JSON_PROTOCOL == protocol) { - memset(pThreadInfo->lines[0], 0, - pThreadInfo->line_buf_len); - } else if (SML_JSON_TAOS_FORMAT == protocol) { - if (pThreadInfo->lines && pThreadInfo->lines[0]) { - tmfree(pThreadInfo->lines[0]); - pThreadInfo->lines[0] = NULL; - } - if (pThreadInfo->json_array) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - pThreadInfo->json_array = tools_cJSON_CreateArray(); - } else { - for (int j = 0; j < generated; j++) { - debugPrint("pThreadInfo->lines[%d]: %s\n", - j, pThreadInfo->lines[j]); - memset(pThreadInfo->lines[j], 0, - pThreadInfo->max_sql_len); - } - } - break; - case STMT_IFACE: - break; - } - - int64_t delay = endTs - startTs; - if (delay <= 0) { - debugPrint("thread[%d]: startTs: %"PRId64", endTs: %"PRId64"\n", - pThreadInfo->threadID, startTs, endTs); - } else { - perfPrint("insert execution time is %.6f s\n", - delay / 1E6); - - int64_t * pDelay = benchCalloc(1, sizeof(int64_t), false); - *pDelay = delay; - if (benchArrayPush(pThreadInfo->delayList, pDelay) == NULL) { - tmfree(pDelay); - } - pThreadInfo->totalDelay += delay; - } - - int64_t currentPrintTime = toolsGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30 * 1000) { - infoPrint( - "thread[%d] has currently inserted rows: " - "%" PRId64 "\n", - pThreadInfo->threadID, pThreadInfo->totalInsertRows); - lastPrintTime = currentPrintTime; - } - if (i >= stbInfo->insertRows) { - break; - } - } // insertRows - } // tableSeq -free_of_progressive: - cleanupAndPrint(pThreadInfo, "progressive"); - return NULL; -} - -static int initStmtDataValue(SSuperTable *stbInfo, SChildTable *childTbl) { - int32_t columnCount = stbInfo->cols->size; - - char *sampleDataBuf; - if (childTbl) { - sampleDataBuf = childTbl->sampleDataBuf; - } else { - sampleDataBuf = stbInfo->sampleDataBuf; - } - int64_t lenOfOneRow = stbInfo->lenOfCols; - - if (stbInfo->useSampleTs) { - columnCount += 1; // for skipping first column - } - for (int i=0; i < g_arguments->prepared_rand; i++) { - int cursor = 0; - - for (int c = 0; c < columnCount; c++) { - char *restStr = sampleDataBuf - + lenOfOneRow * i + cursor; - int lengthOfRest = strlen(restStr); - - int index = 0; - for (index = 0; index < lengthOfRest; index++) { - if (restStr[index] == ',') { - break; - } - } - - cursor += index + 1; // skip ',' too - if ((0 == c) && stbInfo->useSampleTs) { - continue; - } - - char *tmpStr = calloc(1, index + 1); - if (NULL == tmpStr) { - errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, index + 1); - return -1; - } - Field *col = benchArrayGet(stbInfo->cols, - (stbInfo->useSampleTs?c-1:c)); - char dataType = col->type; - - StmtData *stmtData; - if (childTbl) { - ChildField *childCol = - benchArrayGet(childTbl->childCols, - (stbInfo->useSampleTs?c-1:c)); - stmtData = &childCol->stmtData; - } else { - stmtData = &col->stmtData; - } - - strncpy(tmpStr, restStr, index); - - if (0 == strcmp(tmpStr, "NULL")) { - *(stmtData->is_null + i) = true; - } else { - switch (dataType) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *((int32_t*)stmtData->data + i) = atoi(tmpStr); - break; - case TSDB_DATA_TYPE_FLOAT: - *((float*)stmtData->data +i) = (float)atof(tmpStr); - break; - case TSDB_DATA_TYPE_DOUBLE: - *((double*)stmtData->data + i) = atof(tmpStr); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *((int16_t*)stmtData->data + i) = (int16_t)atoi(tmpStr); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); - break; - case TSDB_DATA_TYPE_BOOL: - *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - { - size_t tmpLen = strlen(tmpStr); - debugPrint("%s() LN%d, index: %d, " - "tmpStr len: %"PRIu64", col->length: %d\n", - __func__, __LINE__, - i, (uint64_t)tmpLen, col->length); - if (tmpLen-2 > col->length) { - errorPrint("data length %"PRIu64" " - "is larger than column length %d\n", - (uint64_t)tmpLen, col->length); - } - if (tmpLen > 2) { - strncpy((char *)stmtData->data - + i * col->length, - tmpStr+1, - min(col->length, tmpLen - 2)); - } else { - strncpy((char *)stmtData->data - + i*col->length, - "", 1); - } - } - break; - default: - break; - } - } - free(tmpStr); - } - } - return 0; -} - -static void initStmtData(char dataType, void **data, uint32_t length) { - char *tmpP = NULL; - - switch (dataType) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - tmpP = calloc(1, sizeof(int) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - tmpP = calloc(1, sizeof(int16_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_BOOL: - tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_FLOAT: - tmpP = calloc(1, sizeof(float) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_DOUBLE: - tmpP = calloc(1, sizeof(double) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - tmpP = calloc(1, g_arguments->prepared_rand * length); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_TIMESTAMP: - tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - default: - errorPrint("Unknown data type: %s\n", - convertDatatypeToString(dataType)); - exit(EXIT_FAILURE); - } -} - -static int parseBufferToStmtBatchChildTbl(SSuperTable *stbInfo, - SChildTable* childTbl) { - int32_t columnCount = stbInfo->cols->size; - - for (int c = 0; c < columnCount; c++) { - Field *col = benchArrayGet(stbInfo->cols, c); - ChildField *childCol = benchArrayGet(childTbl->childCols, c); - char dataType = col->type; - - char *is_null = benchCalloc( - 1, sizeof(char) *g_arguments->prepared_rand, false); - - tmfree(childCol->stmtData.is_null); - childCol->stmtData.is_null = is_null; - - initStmtData(dataType, &(childCol->stmtData.data), col->length); - } - - return initStmtDataValue(stbInfo, childTbl); -} - -static int parseBufferToStmtBatch(SSuperTable* stbInfo) { - int32_t columnCount = stbInfo->cols->size; - - for (int c = 0; c < columnCount; c++) { - Field *col = benchArrayGet(stbInfo->cols, c); - char dataType = col->type; - - char *is_null = benchCalloc( - 1, sizeof(char) *g_arguments->prepared_rand, false); - tmfree(col->stmtData.is_null); - col->stmtData.is_null = is_null; - - initStmtData(dataType, &(col->stmtData.data), col->length); - } - - return initStmtDataValue(stbInfo, NULL); -} - -static int64_t fillChildTblNameByCount(SSuperTable *stbInfo) { - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - snprintf(stbInfo->childTblArray[i]->name, - TSDB_TABLE_NAME_LEN, - "%s%" PRIu64 "", - stbInfo->childTblPrefix, i); - debugPrint("%s(): %s\n", __func__, - stbInfo->childTblArray[i]->name); - } - - return stbInfo->childTblCount; -} - -static int64_t fillChildTblNameByFromTo(SDataBase *database, - SSuperTable* stbInfo) { - for (int64_t i = stbInfo->childTblFrom; i < stbInfo->childTblTo; i++) { - snprintf(stbInfo->childTblArray[i-stbInfo->childTblFrom]->name, - TSDB_TABLE_NAME_LEN, - "%s%" PRIu64 "", - stbInfo->childTblPrefix, i); - } - - return (stbInfo->childTblTo-stbInfo->childTblFrom); -} - -static int64_t fillChildTblNameByLimitOffset(SDataBase *database, - SSuperTable* stbInfo) { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - if (g_arguments->taosc_version == 3) { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT DISTINCT(TBNAME) FROM %s.`%s` LIMIT %" PRId64 - " OFFSET %" PRIu64 "", - database->dbName, stbInfo->stbName, stbInfo->childTblLimit, - stbInfo->childTblOffset); - } else { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT TBNAME FROM %s.`%s` LIMIT %" PRId64 - " OFFSET %" PRIu64 "", - database->dbName, stbInfo->stbName, stbInfo->childTblLimit, - stbInfo->childTblOffset); - } - debugPrint("cmd: %s\n", cmd); - TAOS_RES *res = taos_query(conn->taos, cmd); - int32_t code = taos_errno(res); - int64_t count = 0; - if (code) { - printErrCmdCodeStr(cmd, code, res); - closeBenchConn(conn); - return -1; - } - TAOS_ROW row = NULL; - while ((row = taos_fetch_row(res)) != NULL) { - int *lengths = taos_fetch_lengths(res); - strncpy(stbInfo->childTblArray[count]->name, row[0], lengths[0]); - stbInfo->childTblArray[count]->name[lengths[0] + 1] = '\0'; - debugPrint("stbInfo->childTblArray[%" PRId64 "]->name: %s\n", - count, stbInfo->childTblArray[count]->name); - count++; - } - taos_free_result(res); - closeBenchConn(conn); - return count; -} - -static void preProcessArgument(SSuperTable *stbInfo) { - if (stbInfo->interlaceRows > g_arguments->reqPerReq) { - infoPrint( - "interlaceRows(%d) is larger than record per request(%u), which " - "will be set to %u\n", - stbInfo->interlaceRows, g_arguments->reqPerReq, - g_arguments->reqPerReq); - stbInfo->interlaceRows = g_arguments->reqPerReq; - } - - if (stbInfo->interlaceRows > stbInfo->insertRows) { - infoPrint( - "interlaceRows larger than insertRows %d > %" PRId64 "\n", - stbInfo->interlaceRows, stbInfo->insertRows); - infoPrint("%s", "interlaceRows will be set to 0\n"); - stbInfo->interlaceRows = 0; - } - - if (stbInfo->interlaceRows == 0 - && g_arguments->reqPerReq > stbInfo->insertRows) { - infoPrint("record per request (%u) is larger than " - "insert rows (%"PRIu64")" - " in progressive mode, which will be set to %"PRIu64"\n", - g_arguments->reqPerReq, stbInfo->insertRows, - stbInfo->insertRows); - g_arguments->reqPerReq = stbInfo->insertRows; - } - - if (stbInfo->interlaceRows > 0 && stbInfo->iface == STMT_IFACE - && stbInfo->autoTblCreating) { - infoPrint("%s", - "not support autocreate table with interlace row in stmt " - "insertion, will change to progressive mode\n"); - stbInfo->interlaceRows = 0; - } -} - -static int printTotalDelay(SDataBase *database, - int64_t totalDelay, - BArray *total_delay_list, - int threads, - int64_t totalInsertRows, - int64_t start, int64_t end) { - succPrint("Spent %.6f seconds to insert rows: %" PRIu64 - " with %d thread(s) into %s %.2f records/second\n", - (end - start)/1E6, totalInsertRows, threads, - database->dbName, - (double)(totalInsertRows / ((end - start)/1E6))); - if (!total_delay_list->size) { - return -1; - } - - succPrint("insert delay, " - "min: %.4fms, " - "avg: %.4fms, " - "p90: %.4fms, " - "p95: %.4fms, " - "p99: %.4fms, " - "max: %.4fms\n", - *(int64_t *)(benchArrayGet(total_delay_list, 0))/1E3, - (double)totalDelay/total_delay_list->size/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - * 0.9)))/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - * 0.95)))/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - * 0.99)))/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - - 1)))/1E3); - return 0; -} - -static int64_t fillChildTblNameImp(SDataBase *database, SSuperTable *stbInfo) { - int64_t ntables; - if (stbInfo->childTblLimit) { - ntables = fillChildTblNameByLimitOffset(database, stbInfo); - } else if (stbInfo->childTblFrom || stbInfo->childTblTo) { - ntables = fillChildTblNameByFromTo(database, stbInfo); - } else { - ntables = fillChildTblNameByCount(stbInfo); - } - return ntables; -} - -static int64_t fillChildTblName(SDataBase *database, SSuperTable *stbInfo) { - int64_t ntables = stbInfo->childTblCount; - stbInfo->childTblArray = benchCalloc(stbInfo->childTblCount, - sizeof(SChildTable*), true); - for (int64_t child = 0; child < stbInfo->childTblCount; child++) { - stbInfo->childTblArray[child] = - benchCalloc(1, sizeof(SChildTable), true); - } - - if (stbInfo->childTblCount == 1 && stbInfo->tags->size == 0) { - // Normal table - snprintf(stbInfo->childTblArray[0]->name, TSDB_TABLE_NAME_LEN, - "%s", stbInfo->stbName); - } else if ((stbInfo->iface != SML_IFACE - && stbInfo->iface != SML_REST_IFACE) - && stbInfo->childTblExists) { - ntables = fillChildTblNameImp(database, stbInfo); - } else { - ntables = fillChildTblNameByCount(stbInfo); - } - - return ntables; -} - -static int startMultiThreadInsertData(SDataBase* database, - SSuperTable* stbInfo) { - if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) - && !stbInfo->use_metric) { - errorPrint("%s", "schemaless cannot work without stable\n"); - return -1; - } - - preProcessArgument(stbInfo); - - int64_t ntables; - if (stbInfo->childTblTo > 0) { - ntables = stbInfo->childTblTo - stbInfo->childTblFrom; - } else if (stbInfo->childTblLimit) { - ntables = stbInfo->childTblLimit; - } else { - ntables = stbInfo->childTblCount; - } - if (ntables == 0) { - return 0; - } - - uint64_t tableFrom = 0; - int32_t threads = g_arguments->nthreads; - int64_t a = 0, b = 0; - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->tbCountPerVgId++; - } - } - } - - threads = 0; - for (int v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - infoPrint("Total %"PRId64" tables on bb %s's vgroup %d (id: %d)\n", - vg->tbCountPerVgId, database->dbName, v, vg->vgId); - if (vg->tbCountPerVgId) { - threads++; - } else { - continue; - } - vg->childTblArray = benchCalloc( - vg->tbCountPerVgId, sizeof(SChildTable*), true); - vg->tbOffset = 0; - } - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); - - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->childTblArray[vg->tbOffset] = - stbInfo->childTblArray[i]; - vg->tbOffset++; - } - } - } - closeBenchConn(conn); - } else { - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; - } - b = 0; - if (threads != 0) { - b = ntables % threads; - } - } - - int32_t vgFrom = 0; -#else - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; - } - b = 0; - if (threads != 0) { - b = ntables % threads; - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 - pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), true); - threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), true); - - for (int32_t i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - pThreadInfo->dbInfo = database; - pThreadInfo->stbInfo = stbInfo; - pThreadInfo->start_time = stbInfo->startTimestamp; - pThreadInfo->totalInsertRows = 0; - pThreadInfo->samplePos = 0; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - int32_t j; - for (j = vgFrom; i < database->vgroups; j++) { - SVGroup *vg = benchArrayGet(database->vgArray, j); - if (0 == vg->tbCountPerVgId) { - continue; - } - pThreadInfo->vg = vg; - pThreadInfo->start_table_from = 0; - pThreadInfo->ntables = vg->tbCountPerVgId; - pThreadInfo->end_table_to = vg->tbCountPerVgId-1; - break; - } - vgFrom = j + 1; - } else { - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; - } -#else - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; -#endif // TD_VER_COMPATIBLE_3_0_0_0 - pThreadInfo->delayList = benchArrayInit(1, sizeof(int64_t)); - switch (stbInfo->iface) { - case REST_IFACE: { - if (stbInfo->interlaceRows > 0) { - pThreadInfo->buffer = new_ds(0); - } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); - } - int sockfd = createSockFd(); - if (sockfd < 0) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); - } - pThreadInfo->sockfd = sockfd; - break; - } - case STMT_IFACE: { - pThreadInfo->conn = initBenchConn(); - if (NULL == pThreadInfo->conn) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); - } - pThreadInfo->conn->stmt = - taos_stmt_init(pThreadInfo->conn->taos); - if (NULL == pThreadInfo->conn->stmt) { - errorPrint("taos_stmt_init() failed, reason: %s\n", - taos_errstr(NULL)); - FREE_RESOURCE(); - return -1; - } - if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; - } - if (!stbInfo->autoTblCreating) { - if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, 0)) { - FREE_RESOURCE(); - return -1; - } - } - - pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); - pThreadInfo->bind_ts_array = - benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, - true); - pThreadInfo->bindParams = benchCalloc( - 1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), - true); - pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, - true); - parseBufferToStmtBatch(stbInfo); - for (int64_t child = 0; - child < stbInfo->childTblCount; child++) { - SChildTable *childTbl = stbInfo->childTblArray[child]; - if (childTbl->useOwnSample) { - parseBufferToStmtBatchChildTbl(stbInfo, childTbl); - } - } - - break; - } - case SML_REST_IFACE: { - int sockfd = createSockFd(); - if (sockfd < 0) { - free(pids); - free(infos); - return -1; - } - pThreadInfo->sockfd = sockfd; - } - case SML_IFACE: { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - errorPrint("%s() init connection failed\n", __func__); - FREE_RESOURCE(); - return -1; - } - if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; - } - pThreadInfo->max_sql_len = - stbInfo->lenOfCols + stbInfo->lenOfTags; - if (stbInfo->iface == SML_REST_IFACE) { - pThreadInfo->buffer = - benchCalloc(1, g_arguments->reqPerReq * - (1 + pThreadInfo->max_sql_len), true); - } - int protocol = stbInfo->lineProtocol; - if (TSDB_SML_JSON_PROTOCOL != protocol - && SML_JSON_TAOS_FORMAT != protocol) { - pThreadInfo->sml_tags = - (char **)benchCalloc(pThreadInfo->ntables, - sizeof(char *), true); - for (int t = 0; t < pThreadInfo->ntables; t++) { - pThreadInfo->sml_tags[t] = - benchCalloc(1, stbInfo->lenOfTags, true); - } - - for (int t = 0; t < pThreadInfo->ntables; t++) { - if (generateRandData( - stbInfo, pThreadInfo->sml_tags[t], - stbInfo->lenOfTags, - stbInfo->lenOfCols + stbInfo->lenOfTags, - stbInfo->tags, 1, true, NULL)) { - return -1; - } - debugPrint("pThreadInfo->sml_tags[%d]: %s\n", t, - pThreadInfo->sml_tags[t]); - } - pThreadInfo->lines = - benchCalloc(g_arguments->reqPerReq, - sizeof(char *), true); - - for (int j = 0; (j < g_arguments->reqPerReq - && !g_arguments->terminate); j++) { - pThreadInfo->lines[j] = - benchCalloc(1, pThreadInfo->max_sql_len, true); - } - } else { - pThreadInfo->json_array = tools_cJSON_CreateArray(); - pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); - pThreadInfo->sml_tags_json_array = (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); - for (int t = 0; t < pThreadInfo->ntables; t++) { - if (stbInfo->lineProtocol == TSDB_SML_JSON_PROTOCOL) { - generateSmlJsonTags( - pThreadInfo->sml_json_tags, - pThreadInfo->sml_tags_json_array, - stbInfo, - pThreadInfo->start_table_from, t); - } else { - generateSmlTaosJsonTags( - pThreadInfo->sml_json_tags, stbInfo, - pThreadInfo->start_table_from, t); - } - } - pThreadInfo->lines = (char **)benchCalloc( - 1, sizeof(char *), true); - if ((0 == stbInfo->interlaceRows) - && (TSDB_SML_JSON_PROTOCOL == protocol)) { - pThreadInfo->line_buf_len = - g_arguments->reqPerReq * - accumulateRowLen(pThreadInfo->stbInfo->tags, - pThreadInfo->stbInfo->iface); - debugPrint("%s() LN%d, line_buf_len=%d\n", - __func__, __LINE__, pThreadInfo->line_buf_len); - pThreadInfo->lines[0] = benchCalloc( - 1, pThreadInfo->line_buf_len, true); - pThreadInfo->sml_json_value_array = - (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); - for (int t = 0; t < pThreadInfo->ntables; t++) { - generateSmlJsonValues( - pThreadInfo->sml_json_value_array, stbInfo, t); - } - } - } - break; - } - case TAOSC_IFACE: { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - errorPrint("%s() failed to connect\n", __func__); - FREE_RESOURCE(); - return -1; - } - char* command = benchCalloc(1, SHORT_1K_SQL_BUFF_LEN, false); - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "USE `%s`" - : "USE %s", - database->dbName); - if (queryDbExecCall(pThreadInfo->conn, command)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - tmfree(command); - return -1; - } - tmfree(command); - command = NULL; - - if (stbInfo->interlaceRows > 0) { - pThreadInfo->buffer = new_ds(0); - } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); - if (g_arguments->check_sql) { - pThreadInfo->csql = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); - memset(pThreadInfo->csql, 0, TSDB_MAX_ALLOWED_SQL_LEN); - } - } - - break; - } - default: - break; - } - } - - infoPrint("Estimate memory usage: %.2fMB\n", - (double)g_memoryUsage / 1048576); - prompt(0); - - // create threads - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - threadInfo *pThreadInfo = infos + i; - if (stbInfo->interlaceRows > 0) { - pthread_create(pids + i, NULL, - syncWriteInterlace, pThreadInfo); - } else { - pthread_create(pids + i, NULL, - syncWriteProgressive, pThreadInfo); - } - } - - int64_t start = toolsGetTimestampUs(); - - // wait threads - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - pthread_join(pids[i], NULL); - } - - int64_t end = toolsGetTimestampUs()+1; - - if (g_arguments->terminate) toolsMsleep(100); - - BArray * total_delay_list = benchArrayInit(1, sizeof(int64_t)); - int64_t totalDelay = 0; - uint64_t totalInsertRows = 0; - - // free threads resource - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - // free check sql - if (pThreadInfo->csql) { - tmfree(pThreadInfo->csql); - pThreadInfo->csql = NULL; - } - - int protocol = stbInfo->lineProtocol; - switch (stbInfo->iface) { - case REST_IFACE: - if (g_arguments->terminate) - toolsMsleep(100); - destroySockFd(pThreadInfo->sockfd); - if (stbInfo->interlaceRows > 0) { - free_ds(&pThreadInfo->buffer); - } else { - tmfree(pThreadInfo->buffer); - pThreadInfo->buffer = NULL; - } - break; - case SML_REST_IFACE: - if (g_arguments->terminate) - toolsMsleep(100); - tmfree(pThreadInfo->buffer); - // on-purpose no break here - case SML_IFACE: - if (TSDB_SML_JSON_PROTOCOL != protocol - && SML_JSON_TAOS_FORMAT != protocol) { - for (int t = 0; t < pThreadInfo->ntables; t++) { - tmfree(pThreadInfo->sml_tags[t]); - } - for (int j = 0; j < g_arguments->reqPerReq; j++) { - tmfree(pThreadInfo->lines[j]); - } - tmfree(pThreadInfo->sml_tags); - pThreadInfo->sml_tags = NULL; - } else { - for (int t = 0; t < pThreadInfo->ntables; t++) { - tmfree(pThreadInfo->sml_tags_json_array[t]); - } - tmfree(pThreadInfo->sml_tags_json_array); - pThreadInfo->sml_tags_json_array = NULL; - if (pThreadInfo->sml_json_tags) { - tools_cJSON_Delete(pThreadInfo->sml_json_tags); - pThreadInfo->sml_json_tags = NULL; - } - if (pThreadInfo->json_array) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - } - if (pThreadInfo->lines) { - if ((0 == stbInfo->interlaceRows) - && (TSDB_SML_JSON_PROTOCOL == protocol)) { - tmfree(pThreadInfo->lines[0]); - for (int t = 0; t < pThreadInfo->ntables; t++) { - tmfree(pThreadInfo->sml_json_value_array[t]); - } - tmfree(pThreadInfo->sml_json_value_array); - } - tmfree(pThreadInfo->lines); - pThreadInfo->lines = NULL; - } - break; - - case STMT_IFACE: - taos_stmt_close(pThreadInfo->conn->stmt); - tmfree(pThreadInfo->bind_ts); - tmfree(pThreadInfo->bind_ts_array); - tmfree(pThreadInfo->bindParams); - tmfree(pThreadInfo->is_null); - break; - - case TAOSC_IFACE: - if (stbInfo->interlaceRows > 0) { - free_ds(&pThreadInfo->buffer); - } else { - tmfree(pThreadInfo->buffer); - pThreadInfo->buffer = NULL; - } - break; - - default: - break; - } - totalInsertRows += pThreadInfo->totalInsertRows; - totalDelay += pThreadInfo->totalDelay; - benchArrayAddBatch(total_delay_list, pThreadInfo->delayList->pData, - pThreadInfo->delayList->size); - tmfree(pThreadInfo->delayList); - pThreadInfo->delayList = NULL; - // free conn - if (pThreadInfo->conn) { - closeBenchConn(pThreadInfo->conn); - pThreadInfo->conn = NULL; - } - } - - // calculate result - qsort(total_delay_list->pData, total_delay_list->size, - total_delay_list->elemSize, compare); - - if (g_arguments->terminate) toolsMsleep(100); - - free(pids); - free(infos); - - int ret = printTotalDelay(database, totalDelay, - total_delay_list, threads, - totalInsertRows, start, end); - benchArrayDestroy(total_delay_list); - if (g_fail || ret) { - return -1; - } - return 0; -} - -static int getStbInsertedRows(char* dbName, char* stbName, TAOS* taos) { - int rows = 0; - char command[SHORT_1K_SQL_BUFF_LEN]; - snprintf(command, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(*) FROM %s.%s", - dbName, stbName); - TAOS_RES* res = taos_query(taos, command); - int code = taos_errno(res); - if (code != 0) { - printErrCmdCodeStr(command, code, res); - return -1; - } - TAOS_ROW row = taos_fetch_row(res); - if (row == NULL) { - rows = 0; - } else { - rows = (int)*(int64_t*)row[0]; - } - taos_free_result(res); - return rows; -} - -static void create_tsma(TSMA* tsma, SBenchConn* conn, char* stbName) { - char command[SHORT_1K_SQL_BUFF_LEN]; - int len = snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "CREATE sma INDEX %s ON %s function(%s) " - "INTERVAL (%s) SLIDING (%s)", - tsma->name, stbName, tsma->func, - tsma->interval, tsma->sliding); - if (tsma->custom) { - snprintf(command + len, SHORT_1K_SQL_BUFF_LEN - len, - " %s", tsma->custom); - } - int code = queryDbExecCall(conn, command); - if (code == 0) { - infoPrint("successfully create tsma with command <%s>\n", command); - } -} - -static void* create_tsmas(void* args) { - tsmaThreadInfo* pThreadInfo = (tsmaThreadInfo*) args; - int inserted_rows = 0; - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return NULL; - } - int finished = 0; - if (taos_select_db(conn->taos, pThreadInfo->dbName)) { - errorPrint("failed to use database (%s)\n", pThreadInfo->dbName); - closeBenchConn(conn); - return NULL; - } - while (finished < pThreadInfo->tsmas->size && inserted_rows >= 0) { - inserted_rows = (int)getStbInsertedRows( - pThreadInfo->dbName, pThreadInfo->stbName, conn->taos); - for (int i = 0; i < pThreadInfo->tsmas->size; i++) { - TSMA* tsma = benchArrayGet(pThreadInfo->tsmas, i); - if (!tsma->done && inserted_rows >= tsma->start_when_inserted) { - create_tsma(tsma, conn, pThreadInfo->stbName); - tsma->done = true; - finished++; - break; - } - } - toolsMsleep(10); - } - benchArrayDestroy(pThreadInfo->tsmas); - closeBenchConn(conn); - return NULL; -} - -static int32_t createStream(SSTREAM* stream) { - int32_t code = -1; - char * command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, "DROP STREAM IF EXISTS %s", - stream->stream_name); - infoPrint("%s\n", command); - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - goto END_STREAM; - } - - code = queryDbExecCall(conn, command); - int32_t trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then re-drop stream %s\n", - g_arguments->trying_interval, stream->stream_name); - toolsMsleep(g_arguments->trying_interval); - code = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - - if (code) { - closeBenchConn(conn); - goto END_STREAM; - } - - memset(command, 0, TSDB_MAX_ALLOWED_SQL_LEN); - int pos = snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, - "CREATE STREAM IF NOT EXISTS %s ", stream->stream_name); - if (stream->trigger_mode[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "TRIGGER %s ", stream->trigger_mode); - } - if (stream->watermark[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "WATERMARK %s ", stream->watermark); - } - if (stream->ignore_update[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "IGNORE UPDATE %s ", stream->ignore_update); - } - if (stream->ignore_expired[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "IGNORE EXPIRED %s ", stream->ignore_expired); - } - if (stream->fill_history[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "FILL_HISTORY %s ", stream->fill_history); - } - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "INTO %s ", stream->stream_stb); - if (stream->stream_stb_field[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "%s ", stream->stream_stb_field); - } - if (stream->stream_tag_field[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "TAGS%s ", stream->stream_tag_field); - } - if (stream->subtable[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "SUBTABLE%s ", stream->subtable); - } - snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "as %s", stream->source_sql); - infoPrint("%s\n", command); - - code = queryDbExecCall(conn, command); - trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds " - "then re-create stream %s\n", - g_arguments->trying_interval, stream->stream_name); - toolsMsleep(g_arguments->trying_interval); - code = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - - closeBenchConn(conn); -END_STREAM: - tmfree(command); - return code; -} - -int insertTestProcess() { - prompt(0); - - encodeAuthBase64(); - for (int i = 0; i < g_arguments->databases->size; i++) { - if (REST_IFACE == g_arguments->iface) { - if (0 != convertServAddr(g_arguments->iface, - false, - 1)) { - return -1; - } - } - SDataBase * database = benchArrayGet(g_arguments->databases, i); - - if (database->drop && !(g_arguments->supplementInsert)) { - if (database->superTbls) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, 0); - if (stbInfo && (REST_IFACE == stbInfo->iface)) { - if (0 != convertServAddr(stbInfo->iface, - stbInfo->tcpTransfer, - stbInfo->lineProtocol)) { - return -1; - } - } - } - if (createDatabase(database)) { - errorPrint("failed to create database (%s)\n", - database->dbName); - return -1; - } - succPrint("created database (%s)\n", database->dbName); - } - } - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (int j = 0; j < database->superTbls->size; j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - if (stbInfo->iface != SML_IFACE - && stbInfo->iface != SML_REST_IFACE - && !stbInfo->childTblExists) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - dropSuperTable(database, stbInfo); - } -#endif - if (getSuperTableFromServer(database, stbInfo) != 0) { - if (createSuperTable(database, stbInfo)) { - return -1; - } - } - } - fillChildTblName(database, stbInfo); - if (0 != prepareSampleData(database, stbInfo)) { - return -1; - } - } - } - } - - if (g_arguments->taosc_version == 3) { - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase* database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (int j = 0; (j < database->superTbls->size - && !g_arguments->terminate); j++) { - SSuperTable* stbInfo = - benchArrayGet(database->superTbls, j); - if (stbInfo->tsmas == NULL) { - continue; - } - if (stbInfo->tsmas->size > 0) { - tsmaThreadInfo* pThreadInfo = - benchCalloc(1, sizeof(tsmaThreadInfo), true); - pthread_t tsmas_pid = {0}; - pThreadInfo->dbName = database->dbName; - pThreadInfo->stbName = stbInfo->stbName; - pThreadInfo->tsmas = stbInfo->tsmas; - pthread_create(&tsmas_pid, NULL, - create_tsmas, pThreadInfo); - } - } - } - } - } - - if (createChildTables()) return -1; - - if (g_arguments->taosc_version == 3) { - for (int j = 0; j < g_arguments->streams->size; j++) { - SSTREAM * stream = benchArrayGet(g_arguments->streams, j); - if (stream->drop) { - if (createStream(stream)) { - return -1; - } - } - } - } - - // create sub threads for inserting data - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (uint64_t j = 0; j < database->superTbls->size; j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - if (stbInfo->insertRows == 0) { - continue; - } - prompt(stbInfo->non_stop); - if (startMultiThreadInsertData(database, stbInfo)) { - return -1; - } - } - } - } - return 0; -} +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the MIT license as published by the Free Software + * Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + */ + +#include +#include +#include + +#define FREE_PIDS_INFOS_RETURN_MINUS_1() \ + do { \ + tmfree(pids); \ + tmfree(infos); \ + return -1; \ + } while (0) + +#define FREE_RESOURCE() \ + do { \ + if (pThreadInfo->conn) \ + closeBenchConn(pThreadInfo->conn); \ + benchArrayDestroy(pThreadInfo->delayList); \ + tmfree(pids); \ + tmfree(infos); \ + } while (0) \ + +static int getSuperTableFromServerRest( + SDataBase* database, SSuperTable* stbInfo, char *command) { + + return -1; + // TODO(me): finish full implementation +#if 0 + int sockfd = createSockFd(); + if (sockfd < 0) { + return -1; + } + + int code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + + destroySockFd(sockfd); +#endif // 0 +} + +static int getSuperTableFromServerTaosc( + SDataBase* database, SSuperTable* stbInfo, char *command) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + return -1; + } +#endif + TAOS_RES * res; + TAOS_ROW row = NULL; + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + + res = taos_query(conn->taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printWarnCmdCodeStr(command, code, res); + infoPrint("stable %s does not exist, will create one\n", + stbInfo->stbName); + closeBenchConn(conn); + return -1; + } + infoPrint("find stable<%s>, will get meta data from server\n", + stbInfo->stbName); + benchArrayClear(stbInfo->tags); + benchArrayClear(stbInfo->cols); + int count = 0; + while ((row = taos_fetch_row(res)) != NULL) { + if (count == 0) { + count++; + continue; + } + int32_t *lengths = taos_fetch_lengths(res); + if (lengths == NULL) { + errorPrint("%s", "failed to execute taos_fetch_length\n"); + taos_free_result(res); + closeBenchConn(conn); + return -1; + } + if (strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "tag", + strlen("tag")) == 0) { + Field* tag = benchCalloc(1, sizeof(Field), true); + benchArrayPush(stbInfo->tags, tag); + tag = benchArrayGet(stbInfo->tags, stbInfo->tags->size - 1); + tag->type = convertStringToDatatype( + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); + tag->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tag->min = convertDatatypeToDefaultMin(tag->type); + tag->max = convertDatatypeToDefaultMax(tag->type); + tstrncpy(tag->name, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); + } else { + Field * col = benchCalloc(1, sizeof(Field), true); + benchArrayPush(stbInfo->cols, col); + col = benchArrayGet(stbInfo->cols, stbInfo->cols->size - 1); + col->type = convertStringToDatatype( + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); + col->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + col->min = convertDatatypeToDefaultMin(col->type); + col->max = convertDatatypeToDefaultMax(col->type); + tstrncpy(col->name, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); + } + } + taos_free_result(res); + closeBenchConn(conn); + return 0; +} + +static int getSuperTableFromServer(SDataBase* database, SSuperTable* stbInfo) { + int ret = 0; + + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "DESCRIBE `%s`.`%s`", database->dbName, + stbInfo->stbName); + + if (REST_IFACE == stbInfo->iface) { + ret = getSuperTableFromServerRest(database, stbInfo, command); + } else { + ret = getSuperTableFromServerTaosc(database, stbInfo, command); + } + + return ret; +} + +static int queryDbExec(SDataBase *database, + SSuperTable *stbInfo, char *command) { + int ret = 0; + if (REST_IFACE == stbInfo->iface) { + if (0 != convertServAddr(stbInfo->iface, false, 1)) { + errorPrint("%s", "Failed to convert server address\n"); + return -1; + } + int sockfd = createSockFd(); + if (sockfd < 0) { + ret = -1; + } else { + ret = queryDbExecRest(command, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + sockfd); + destroySockFd(sockfd); + } + } else { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + ret = -1; + } else { + ret = queryDbExecCall(conn, command); + int32_t trying = g_arguments->keep_trying; + while (ret && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then re-create " + "supertable %s\n", + g_arguments->trying_interval, stbInfo->stbName); + toolsMsleep(g_arguments->trying_interval); + ret = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + if (0 != ret) { + errorPrint("create supertable %s failed!\n\n", + stbInfo->stbName); + ret = -1; + } + closeBenchConn(conn); + } + } + + return ret; +} + +#ifdef WEBSOCKET +static void dropSuperTable(SDataBase* database, SSuperTable* stbInfo) { + if (g_arguments->supplementInsert) { + return; + } + + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + snprintf(command, sizeof(command), + g_arguments->escape_character + ? "DROP TABLE `%s`.`%s`" + : "DROP TABLE %s.%s", + database->dbName, + stbInfo->stbName); + + infoPrint("drop stable: <%s>\n", command); + queryDbExec(database, stbInfo, command); + + return; +} +#endif // WEBSOCKET + +static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { + if (g_arguments->supplementInsert) { + return 0; + } + + uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->cols->size; + char *colsBuf = benchCalloc(1, col_buffer_len, false); + char* command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + int len = 0; + + for (int colIndex = 0; colIndex < stbInfo->cols->size; colIndex++) { + Field * col = benchArrayGet(stbInfo->cols, colIndex); + int n; + if (col->type == TSDB_DATA_TYPE_BINARY || + col->type == TSDB_DATA_TYPE_NCHAR) { + n = snprintf(colsBuf + len, col_buffer_len - len, + ",%s %s(%d)", col->name, + convertDatatypeToString(col->type), col->length); + } else { + n = snprintf(colsBuf + len, col_buffer_len - len, + ",%s %s", col->name, + convertDatatypeToString(col->type)); + } + if (n < 0 || n >= col_buffer_len - len) { + errorPrint("%s() LN%d, snprintf overflow on %d\n", + __func__, __LINE__, colIndex); + break; + } else { + len += n; + } + } + + // save for creating child table + stbInfo->colsOfCreateChildTable = + (char *)benchCalloc(len + TIMESTAMP_BUFF_LEN, 1, true); + + snprintf(stbInfo->colsOfCreateChildTable, len + TIMESTAMP_BUFF_LEN, + "(ts timestamp%s)", colsBuf); + + if (stbInfo->tags->size == 0) { + free(colsBuf); + free(command); + return 0; + } + + uint32_t tag_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->tags->size; + char *tagsBuf = benchCalloc(1, tag_buffer_len, false); + int tagIndex; + len = 0; + + int n; + n = snprintf(tagsBuf + len, tag_buffer_len - len, "("); + if (n < 0 || n >= tag_buffer_len - len) { + errorPrint("%s() LN%d snprintf overflow\n", + __func__, __LINE__); + free(colsBuf); + free(command); + tmfree(tagsBuf); + return -1; + } else { + len += n; + } + for (tagIndex = 0; tagIndex < stbInfo->tags->size; tagIndex++) { + Field *tag = benchArrayGet(stbInfo->tags, tagIndex); + if (tag->type == TSDB_DATA_TYPE_BINARY || + tag->type == TSDB_DATA_TYPE_NCHAR) { + n = snprintf(tagsBuf + len, tag_buffer_len - len, + "%s %s(%d),", tag->name, + convertDatatypeToString(tag->type), tag->length); + } else if (tag->type == TSDB_DATA_TYPE_JSON) { + n = snprintf(tagsBuf + len, tag_buffer_len - len, + "%s json", tag->name); + if (n < 0 || n >= tag_buffer_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, tagIndex); + break; + } else { + len += n; + } + goto skip; + } else { + n = snprintf(tagsBuf + len, tag_buffer_len - len, + "%s %s,", tag->name, + convertDatatypeToString(tag->type)); + } + + if (n < 0 || n >= tag_buffer_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, tagIndex); + break; + } else { + len += n; + } + } + len -= 1; +skip: + snprintf(tagsBuf + len, tag_buffer_len - len, ")"); + + int length = snprintf( + command, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "CREATE TABLE `%s`.`%s` (ts TIMESTAMP%s) TAGS %s" + : "CREATE TABLE %s.%s (ts TIMESTAMP%s) TAGS %s", + database->dbName, stbInfo->stbName, colsBuf, tagsBuf); + tmfree(colsBuf); + tmfree(tagsBuf); + if (stbInfo->comment != NULL) { + length += snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, + " COMMENT '%s'", stbInfo->comment); + } + if (stbInfo->delay >= 0) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, " DELAY %d", + stbInfo->delay); + } + if (stbInfo->file_factor >= 0) { + length += + snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, " FILE_FACTOR %f", + (float)stbInfo->file_factor / 100); + } + if (stbInfo->rollup != NULL) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " ROLLUP(%s)", stbInfo->rollup); + } + + if (stbInfo->max_delay != NULL) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " MAX_DELAY %s", stbInfo->max_delay); + } + + if (stbInfo->watermark != NULL) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " WATERMARK %s", stbInfo->watermark); + } + + if (stbInfo->ttl != 0) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " TTL %d", stbInfo->ttl); + } + + bool first_sma = true; + for (int i = 0; i < stbInfo->cols->size; i++) { + Field * col = benchArrayGet(stbInfo->cols, i); + if (col->sma) { + if (first_sma) { + n = snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " SMA(%s", col->name); + first_sma = false; + } else { + n = snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + ",%s", col->name); + } + + if (n < 0 || n > TSDB_MAX_ALLOWED_SQL_LEN - length) { + errorPrint("%s() LN%d snprintf overflow on %d iteral\n", + __func__, __LINE__, i); + break; + } else { + length += n; + } + } + } + if (!first_sma) { + snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, ")"); + } + infoPrint("create stable: <%s>\n", command); + + int ret = queryDbExec(database, stbInfo, command); + free(command); + return ret; +} + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 +int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { + int vgroups = 0; + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "USE `%s`" + : "USE %s", + database->dbName); + + int32_t code; + TAOS_RES *res = NULL; + + res = taos_query(conn->taos, cmd); + code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + return -1; + } + taos_free_result(res); + + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SHOW VGROUPS"); + res = taos_query(conn->taos, cmd); + code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + return -1; + } + + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res)) != NULL) { + vgroups++; + } + debugPrint("%s() LN%d, vgroups: %d\n", __func__, __LINE__, vgroups); + taos_free_result(res); + + database->vgroups = vgroups; + database->vgArray = benchArrayInit(vgroups, sizeof(SVGroup)); + for (int32_t v = 0; (v < vgroups + && !g_arguments->terminate); v++) { + SVGroup *vg = benchCalloc(1, sizeof(SVGroup), true); + benchArrayPush(database->vgArray, vg); + } + + res = taos_query(conn->taos, cmd); + code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + return -1; + } + + int32_t vgItem = 0; + while (((row = taos_fetch_row(res)) != NULL) + && !g_arguments->terminate) { + SVGroup *vg = benchArrayGet(database->vgArray, vgItem); + vg->vgId = *(int32_t*)row[0]; + vgItem++; + } + taos_free_result(res); + + return vgroups; +} +#endif // TD_VER_COMPATIBLE_3_0_0_0 + +int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { + int dataLen = 0; + int n; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto || (-1 != g_arguments->inputted_vgroups)) { + n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, + g_arguments->escape_character + ? "CREATE DATABASE IF NOT EXISTS `%s` VGROUPS %d" + : "CREATE DATABASE IF NOT EXISTS %s VGROUPS %d", + database->dbName, + (-1 != g_arguments->inputted_vgroups)? + g_arguments->inputted_vgroups: + min(remainVnodes, toolsGetNumberOfCores())); + } else { + n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, + g_arguments->escape_character + ? "CREATE DATABASE IF NOT EXISTS `%s`" + : "CREATE DATABASE IF NOT EXISTS %s", + database->dbName); + } +#else + n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, + g_arguments->escape_character + ? "CREATE DATABASE IF NOT EXISTS `%s`" + : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); +#endif // TD_VER_COMPATIBLE_3_0_0_0 + if (n < 0 || n >= SHORT_1K_SQL_BUFF_LEN - dataLen) { + errorPrint("%s() LN%d snprintf overflow\n", + __func__, __LINE__); + return -1; + } else { + dataLen += n; + } + + if (database->cfgs) { + for (int i = 0; i < database->cfgs->size; i++) { + SDbCfg* cfg = benchArrayGet(database->cfgs, i); + if (cfg->valuestring) { + n = snprintf(command + dataLen, + TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " %s %s", cfg->name, cfg->valuestring); + } else { + n = snprintf(command + dataLen, + TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " %s %d", cfg->name, cfg->valueint); + } + if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - dataLen) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, i); + break; + } else { + dataLen += n; + } + } + } + + switch (database->precision) { + case TSDB_TIME_PRECISION_MILLI: + snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " PRECISION \'ms\';"); + break; + case TSDB_TIME_PRECISION_MICRO: + snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " PRECISION \'us\';"); + break; + case TSDB_TIME_PRECISION_NANO: + snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " PRECISION \'ns\';"); + break; + } + + return dataLen; +} + +int createDatabaseRest(SDataBase* database) { + int32_t code = 0; + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + + int sockfd = createSockFd(); + if (sockfd < 0) { + return -1; + } + + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "DROP DATABASE IF EXISTS `%s`;" + : "DROP DATABASE IF EXISTS %s;", + database->dbName); + code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + if (code != 0) { + errorPrint("Failed to drop database %s\n", database->dbName); + } else { + int remainVnodes = INT_MAX; + geneDbCreateCmd(database, command, remainVnodes); + code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + int32_t trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then " + "re-create database %s\n", + g_arguments->trying_interval, database->dbName); + toolsMsleep(g_arguments->trying_interval); + code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + if (trying != -1) { + trying--; + } + } + } + destroySockFd(sockfd); + return code; +} + +int32_t getRemainVnodes(SBenchConn *conn) { + int remainVnodes = 0; + char command[SHORT_1K_SQL_BUFF_LEN] = "SHOW DNODES"; + + TAOS_RES *res = taos_query(conn->taos, command); + int32_t code = taos_errno(res); + if (code) { + printErrCmdCodeStr(command, code, res); + closeBenchConn(conn); + return -1; + } + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res)) != NULL) { + remainVnodes += (*(int16_t*)(row[3]) - *(int16_t*)(row[2])); + } + debugPrint("%s() LN%d, remainVnodes: %d\n", + __func__, __LINE__, remainVnodes); + taos_free_result(res); + return remainVnodes; +} + +int createDatabaseTaosc(SDataBase* database) { + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + if (g_arguments->taosc_version == 3) { + for (int i = 0; i < g_arguments->streams->size; i++) { + SSTREAM* stream = benchArrayGet(g_arguments->streams, i); + if (stream->drop) { + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "DROP STREAM IF EXISTS %s;", + stream->stream_name); + if (queryDbExecCall(conn, command)) { + closeBenchConn(conn); + return -1; + } + infoPrint("%s\n", command); + memset(command, 0, SHORT_1K_SQL_BUFF_LEN); + } + } + } + + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "DROP DATABASE IF EXISTS `%s`;": + "DROP DATABASE IF EXISTS %s;", + database->dbName); + if (0 != queryDbExecCall(conn, command)) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + warnPrint("%s", "TDengine cloud normal users have no privilege " + "to drop database! DROP DATABASE failure is ignored!\n"); + } else { +#endif + closeBenchConn(conn); + return -1; +#ifdef WEBSOCKET + } +#endif + } + + int remainVnodes = INT_MAX; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto) { + remainVnodes = getRemainVnodes(conn); + if (0 >= remainVnodes) { + errorPrint("Remain vnodes %d, failed to create database\n", + remainVnodes); + return -1; + } + } +#endif + geneDbCreateCmd(database, command, remainVnodes); + + int32_t code = queryDbExecCall(conn, command); + int32_t trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then " + "re-create database %s\n", + g_arguments->trying_interval, database->dbName); + toolsMsleep(g_arguments->trying_interval); + code = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + + if (code) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + warnPrint("%s", "TDengine cloud normal users have no privilege " + "to create database! CREATE DATABASE " + "failure is ignored!\n"); + } else { +#endif + + closeBenchConn(conn); + errorPrint("\ncreate database %s failed!\n\n", + database->dbName); + return -1; +#ifdef WEBSOCKET + } +#endif + } + infoPrint("command to create database: <%s>\n", command); + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (database->superTbls) { + if (g_arguments->nthreads_auto) { + int32_t vgroups = getVgroupsOfDb(conn, database); + if (vgroups <=0) { + closeBenchConn(conn); + errorPrint("Database %s's vgroups is %d\n", + database->dbName, vgroups); + return -1; + } + } + } +#endif // TD_VER_COMPATIBLE_3_0_0_0 + + closeBenchConn(conn); + return 0; +} + +int createDatabase(SDataBase* database) { + int ret = 0; + if (REST_IFACE == g_arguments->iface) { + ret = createDatabaseRest(database); + } else { + ret = createDatabaseTaosc(database); + } +#if 0 +#ifdef LINUX + infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); + sleep(10); + infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); +#elif defined(DARWIN) + sleep(2); +#else + Sleep(2); +#endif +#endif + + return ret; +} + +static int generateChildTblName(int len, char *buffer, SDataBase *database, + SSuperTable *stbInfo, uint64_t i, + char *ttl) { + if (0 == len) { + memset(buffer, 0, TSDB_MAX_ALLOWED_SQL_LEN); + len += snprintf(buffer + len, + TSDB_MAX_ALLOWED_SQL_LEN - len, "CREATE TABLE "); + } + + len += snprintf( + buffer + len, TSDB_MAX_ALLOWED_SQL_LEN - len, + g_arguments->escape_character + ? "`%s`.`%s%" PRIu64 "` USING `%s`.`%s` TAGS (%s) %s " + : "%s.%s%" PRIu64 " USING %s.%s TAGS (%s) %s ", + database->dbName, stbInfo->childTblPrefix, i, database->dbName, + stbInfo->stbName, + stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); + + return len; +} + +static int getBatchOfTblCreating(threadInfo *pThreadInfo, + SSuperTable *stbInfo) { + BArray *batchArray = stbInfo->batchTblCreatingNumbersArray; + if (batchArray) { + int *batch = benchArrayGet( + batchArray, pThreadInfo->posOfTblCreatingBatch); + pThreadInfo->posOfTblCreatingBatch++; + if (pThreadInfo->posOfTblCreatingBatch == batchArray->size) { + pThreadInfo->posOfTblCreatingBatch = 0; + } + return *batch; + } + return 0; +} + +static int getIntervalOfTblCreating(threadInfo *pThreadInfo, + SSuperTable *stbInfo) { + BArray *intervalArray = stbInfo->batchTblCreatingIntervalsArray; + if (intervalArray) { + int *interval = benchArrayGet( + intervalArray, pThreadInfo->posOfTblCreatingInterval); + pThreadInfo->posOfTblCreatingInterval++; + if (pThreadInfo->posOfTblCreatingInterval == intervalArray->size) { + pThreadInfo->posOfTblCreatingInterval = 0; + } + return *interval; + } + return 0; +} + +static void *createTable(void *sarg) { + if (g_arguments->supplementInsert) { + return NULL; + } + + threadInfo * pThreadInfo = (threadInfo *)sarg; + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; +#ifdef LINUX + prctl(PR_SET_NAME, "createTable"); +#endif + uint64_t lastPrintTime = toolsGetTimestampMs(); + pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + int len = 0; + int batchNum = 0; + infoPrint( + "thread[%d] start creating table from %" PRIu64 " to %" PRIu64 + "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to); + + char ttl[SMALL_BUFF_LEN] = ""; + if (stbInfo->ttl != 0) { + snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); + } + + int smallBatchCount = 0; + for (uint64_t i = pThreadInfo->start_table_from + stbInfo->childTblFrom; + (i <= (pThreadInfo->end_table_to + stbInfo->childTblFrom) + && !g_arguments->terminate); i++) { + if (g_arguments->terminate) { + goto create_table_end; + } + if (!stbInfo->use_metric || stbInfo->tags->size == 0) { + if (stbInfo->childTblCount == 1) { + snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "CREATE TABLE `%s`.`%s` %s;" + : "CREATE TABLE %s.%s %s;", + database->dbName, stbInfo->stbName, + stbInfo->colsOfCreateChildTable); + } else { + snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "CREATE TABLE `%s`.`%s` %s;" + : "CREATE TABLE %s.%s %s;", + database->dbName, + stbInfo->childTblArray[i]->name, + stbInfo->colsOfCreateChildTable); + } + batchNum++; + } else { + if (0 == len) { + batchNum = 0; + } + len = generateChildTblName(len, pThreadInfo->buffer, + database, stbInfo, i, ttl); + + batchNum++; + smallBatchCount++; + + int smallBatch = getBatchOfTblCreating(pThreadInfo, stbInfo); + if ((!smallBatch || (smallBatchCount == smallBatch)) + && (batchNum < stbInfo->batchTblCreatingNum) + && ((TSDB_MAX_ALLOWED_SQL_LEN - len) >= + (stbInfo->lenOfTags + EXTRA_SQL_LEN))) { + continue; + } else { + smallBatchCount = 0; + } + } + + len = 0; + + int ret = 0; + debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, + pThreadInfo->buffer); + if (REST_IFACE == stbInfo->iface) { + ret = queryDbExecRest(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + pThreadInfo->sockfd); + } else { + ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + int32_t trying = g_arguments->keep_trying; + while (ret && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then re-create " + "table %s\n", + g_arguments->trying_interval, pThreadInfo->buffer); + toolsMsleep(g_arguments->trying_interval); + ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + if (trying != -1) { + trying--; + } + } + } + + if (0 != ret) { + g_fail = true; + goto create_table_end; + } + uint64_t intervalOfTblCreating = getIntervalOfTblCreating(pThreadInfo, + stbInfo); + if (intervalOfTblCreating) { + debugPrint("will sleep %"PRIu64" milliseconds " + "for table creating interval\n", intervalOfTblCreating); + toolsMsleep(intervalOfTblCreating); + } + + pThreadInfo->tables_created += batchNum; + batchNum = 0; + uint64_t currentPrintTime = toolsGetTimestampMs(); + if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { + infoPrint( + "thread[%d] already created %" PRId64 " tables\n", + pThreadInfo->threadID, pThreadInfo->tables_created); + lastPrintTime = currentPrintTime; + } + } + + if (0 != len) { + int ret = 0; + debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, + pThreadInfo->buffer); + if (REST_IFACE == stbInfo->iface) { + ret = queryDbExecRest(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + pThreadInfo->sockfd); + } else { + ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + } + if (0 != ret) { + g_fail = true; + goto create_table_end; + } + pThreadInfo->tables_created += batchNum; + debugPrint("thread[%d] already created %" PRId64 " tables\n", + pThreadInfo->threadID, pThreadInfo->tables_created); + } +create_table_end: + tmfree(pThreadInfo->buffer); + pThreadInfo->buffer = NULL; + return NULL; +} + +static int startMultiThreadCreateChildTable( + SDataBase* database, SSuperTable* stbInfo) { + int code = -1; + int threads = g_arguments->table_threads; + int64_t ntables; + if (stbInfo->childTblTo > 0) { + ntables = stbInfo->childTblTo - stbInfo->childTblFrom; + } else { + ntables = stbInfo->childTblCount; + } + pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), false); + threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), false); + uint64_t tableFrom = 0; + if (threads < 1) { + threads = 1; + } + + int64_t a = ntables / threads; + if (a < 1) { + threads = (int)ntables; + a = 1; + } + + if (ntables == 0) { + errorPrint("failed to create child table, childTblCount: %"PRId64"\n", + ntables); + goto over; + } + int64_t b = ntables % threads; + + for (uint32_t i = 0; (i < threads && !g_arguments->terminate); i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + pThreadInfo->stbInfo = stbInfo; + pThreadInfo->dbInfo = database; + if (REST_IFACE == stbInfo->iface) { + int sockfd = createSockFd(); + if (sockfd < 0) { + FREE_PIDS_INFOS_RETURN_MINUS_1(); + } + pThreadInfo->sockfd = sockfd; + } else { + pThreadInfo->conn = initBenchConn(); + if (NULL == pThreadInfo->conn) { + goto over; + } + } + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = i < b ? a + 1 : a; + pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->tables_created = 0; + pthread_create(pids + i, NULL, createTable, pThreadInfo); + } + + for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + pthread_join(pids[i], NULL); + } + + if (g_arguments->terminate) toolsMsleep(100); + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + g_arguments->actualChildTables += pThreadInfo->tables_created; + + if ((REST_IFACE != stbInfo->iface) && pThreadInfo->conn) { + closeBenchConn(pThreadInfo->conn); + } + } + + if (g_fail) { + goto over; + } + code = 0; +over: + free(pids); + free(infos); + return code; +} + +static int createChildTables() { + int32_t code; + infoPrint("start creating %" PRId64 " table(s) with %d thread(s)\n", + g_arguments->totalChildTables, g_arguments->table_threads); + if (g_arguments->fpOfInsertResult) { + infoPrintToFile(g_arguments->fpOfInsertResult, + "start creating %" PRId64 " table(s) with %d thread(s)\n", + g_arguments->totalChildTables, g_arguments->table_threads); + } + double start = (double)toolsGetTimestampMs(); + + for (int i = 0; (i < g_arguments->databases->size + && !g_arguments->terminate); i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (int j = 0; (j < database->superTbls->size + && !g_arguments->terminate); j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + if (stbInfo->autoTblCreating || stbInfo->iface == SML_IFACE + || stbInfo->iface == SML_REST_IFACE) { + g_arguments->autoCreatedChildTables += + stbInfo->childTblCount; + continue; + } + if (stbInfo->childTblExists) { + g_arguments->existedChildTables += + stbInfo->childTblCount; + continue; + } + debugPrint("colsOfCreateChildTable: %s\n", + stbInfo->colsOfCreateChildTable); + + code = startMultiThreadCreateChildTable(database, stbInfo); + if (code && !g_arguments->terminate) { + return code; + } + } + } + } + + double end = (double)toolsGetTimestampMs(); + succPrint( + "Spent %.4f seconds to create %" PRId64 + " table(s) with %d thread(s), already exist %" PRId64 + " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 + " table(s) will be auto created\n", + (end - start) / 1000.0, g_arguments->totalChildTables, + g_arguments->table_threads, g_arguments->existedChildTables, + g_arguments->actualChildTables, + g_arguments->autoCreatedChildTables); + return 0; +} + +static void freeChildTable(SChildTable *childTbl, int colsSize) { + if (childTbl->useOwnSample) { + if (childTbl->childCols) { + for (int col = 0; col < colsSize; col++) { + ChildField *childCol = + benchArrayGet(childTbl->childCols, col); + if (childCol) { + tmfree(childCol->stmtData.data); + tmfree(childCol->stmtData.is_null); + } + } + benchArrayDestroy(childTbl->childCols); + } + tmfree(childTbl->sampleDataBuf); + } + tmfree(childTbl); +} + +void postFreeResource() { + if (!g_arguments->terminate) { + tmfclose(g_arguments->fpOfInsertResult); + } + + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->cfgs) { + for (int c = 0; c < database->cfgs->size; c++) { + SDbCfg *cfg = benchArrayGet(database->cfgs, c); + if ((NULL == root) && (0 == strcmp(cfg->name, "replica"))) { + tmfree(cfg->name); + cfg->name = NULL; + } + } + benchArrayDestroy(database->cfgs); + } + if (database->superTbls) { + for (uint64_t j = 0; j < database->superTbls->size; j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + tmfree(stbInfo->colsOfCreateChildTable); + stbInfo->colsOfCreateChildTable = NULL; + tmfree(stbInfo->sampleDataBuf); + stbInfo->sampleDataBuf = NULL; + tmfree(stbInfo->tagDataBuf); + stbInfo->tagDataBuf = NULL; + tmfree(stbInfo->partialColNameBuf); + stbInfo->partialColNameBuf = NULL; + benchArrayDestroy(stbInfo->batchTblCreatingNumbersArray); + benchArrayDestroy(stbInfo->batchTblCreatingIntervalsArray); + for (int k = 0; k < stbInfo->tags->size; k++) { + Field * tag = benchArrayGet(stbInfo->tags, k); + tmfree(tag->stmtData.data); + tag->stmtData.data = NULL; + } + benchArrayDestroy(stbInfo->tags); + + for (int k = 0; k < stbInfo->cols->size; k++) { + Field * col = benchArrayGet(stbInfo->cols, k); + tmfree(col->stmtData.data); + col->stmtData.data = NULL; + tmfree(col->stmtData.is_null); + col->stmtData.is_null = NULL; + } + if (g_arguments->test_mode == INSERT_TEST) { + if (stbInfo->childTblArray) { + for (int64_t child = 0; child < stbInfo->childTblCount; + child++) { + SChildTable *childTbl = + stbInfo->childTblArray[child]; + if (childTbl) { + freeChildTable(childTbl, stbInfo->cols->size); + } + } + } + } + benchArrayDestroy(stbInfo->cols); + tmfree(stbInfo->childTblArray); + stbInfo->childTblArray = NULL; + benchArrayDestroy(stbInfo->tsmas); +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if ((0 == stbInfo->interlaceRows) + && (g_arguments->nthreads_auto)) { + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + tmfree(vg->childTblArray); + vg->childTblArray = NULL; + } + } +#endif // TD_VER_COMPATIBLE_3_0_0_0 + } +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (database->vgArray) + benchArrayDestroy(database->vgArray); +#endif // TD_VER_COMPATIBLE_3_0_0_0 + benchArrayDestroy(database->superTbls); + } + } + benchArrayDestroy(g_arguments->databases); + benchArrayDestroy(g_arguments->streams); + tools_cJSON_Delete(root); +} + +int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + TAOS_RES * res = NULL; + int32_t code = 0; + uint16_t iface = stbInfo->iface; + + int32_t trying = (stbInfo->keep_trying)? + stbInfo->keep_trying:g_arguments->keep_trying; + int32_t trying_interval = stbInfo->trying_interval? + stbInfo->trying_interval:g_arguments->trying_interval; + int protocol = stbInfo->lineProtocol; + + switch (iface) { + case TAOSC_IFACE: + debugPrint("buffer: %s\n", pThreadInfo->buffer); + code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + while (code && trying && !g_arguments->terminate) { + infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", + trying_interval); + toolsMsleep(trying_interval); + code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + if (trying != -1) { + trying--; + } + } + break; + + case REST_IFACE: + debugPrint("buffer: %s\n", pThreadInfo->buffer); + code = postProceSql(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, + pThreadInfo->filePath); + while (code && trying && !g_arguments->terminate) { + infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", + trying_interval); + toolsMsleep(trying_interval); + code = postProceSql(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, + pThreadInfo->filePath); + if (trying != -1) { + trying--; + } + } + break; + + case STMT_IFACE: + code = taos_stmt_execute(pThreadInfo->conn->stmt); + if (code) { + errorPrint( + "failed to execute insert statement. reason: %s\n", + taos_stmt_errstr(pThreadInfo->conn->stmt)); + code = -1; + } + break; + + case SML_IFACE: + res = taos_schemaless_insert( + pThreadInfo->conn->taos, pThreadInfo->lines, + (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) + ? 0 : k, + (SML_JSON_TAOS_FORMAT == protocol) + ? TSDB_SML_JSON_PROTOCOL : protocol, + (TSDB_SML_LINE_PROTOCOL == protocol) + ? database->sml_precision + : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(res); + trying = stbInfo->keep_trying; + while (code && trying && !g_arguments->terminate) { + taos_free_result(res); + infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", + trying_interval); + toolsMsleep(trying_interval); + res = taos_schemaless_insert( + pThreadInfo->conn->taos, pThreadInfo->lines, + (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) + ? 0 : k, + (SML_JSON_TAOS_FORMAT == protocol) + ? TSDB_SML_JSON_PROTOCOL : protocol, + (TSDB_SML_LINE_PROTOCOL == protocol) + ? database->sml_precision + : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(res); + if (trying != -1) { + trying--; + } + } + + if (code != TSDB_CODE_SUCCESS && !g_arguments->terminate) { + debugPrint("Failed to execute " + "schemaless insert content: %s\n\n", + pThreadInfo->lines?(pThreadInfo->lines[0]? + pThreadInfo->lines[0]:""):""); + errorPrint( + "failed to execute schemaless insert. " + "code: 0x%08x reason: %s\n\n", + code, taos_errstr(res)); + } + taos_free_result(res); + break; + + case SML_REST_IFACE: { + if (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) { + code = postProceSql(pThreadInfo->lines[0], database->dbName, + database->precision, stbInfo->iface, + protocol, g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, pThreadInfo->filePath); + } else { + int len = 0; + for (int i = 0; i < k; i++) { + if (strlen(pThreadInfo->lines[i]) != 0) { + int n; + if (TSDB_SML_TELNET_PROTOCOL == protocol + && stbInfo->tcpTransfer) { + n = snprintf(pThreadInfo->buffer + len, + TSDB_MAX_ALLOWED_SQL_LEN - len, + "put %s\n", pThreadInfo->lines[i]); + } else { + n = snprintf(pThreadInfo->buffer + len, + TSDB_MAX_ALLOWED_SQL_LEN - len, + "%s\n", + pThreadInfo->lines[i]); + } + if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, i); + break; + } else { + len += n; + } + } else { + break; + } + } + if (g_arguments->terminate) { + break; + } + code = postProceSql(pThreadInfo->buffer, database->dbName, + database->precision, + stbInfo->iface, protocol, + g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, pThreadInfo->filePath); + } + break; + } + } + return code; +} + +static int smartContinueIfFail(threadInfo *pThreadInfo, + SChildTable *childTbl, + int64_t i, + char *ttl) { + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + char *buffer = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + snprintf( + buffer, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character ? + "CREATE TABLE `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s " + : "CREATE TABLE %s.%s USING %s.%s TAGS (%s) %s ", + database->dbName, childTbl->name, database->dbName, + stbInfo->stbName, + stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); + debugPrint("creating table: %s\n", buffer); + int ret; + if (REST_IFACE == stbInfo->iface) { + ret = queryDbExecRest(buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + pThreadInfo->sockfd); + } else { + ret = queryDbExecCall(pThreadInfo->conn, buffer); + int32_t trying = g_arguments->keep_trying; + while (ret && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then " + "re-create table %s\n", + g_arguments->trying_interval, buffer); + toolsMsleep(g_arguments->trying_interval); + ret = queryDbExecCall(pThreadInfo->conn, buffer); + if (trying != -1) { + trying--; + } + } + } + tmfree(buffer); + + return ret; +} + +static void cleanupAndPrint(threadInfo *pThreadInfo, char *mode) { + if (pThreadInfo) { + if (pThreadInfo->json_array) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + if (0 == pThreadInfo->totalDelay) { + pThreadInfo->totalDelay = 1; + } + succPrint( + "thread[%d] %s mode, completed total inserted rows: %" PRIu64 + ", %.2f records/second\n", + pThreadInfo->threadID, + mode, + pThreadInfo->totalInsertRows, + (double)(pThreadInfo->totalInsertRows / + ((double)pThreadInfo->totalDelay / 1E6))); + } +} + +static int64_t getDisorderTs(SSuperTable *stbInfo, int *disorderRange) { + int64_t disorderTs = 0; + int64_t startTimestamp = stbInfo->startTimestamp; + if (stbInfo->disorderRatio > 0) { + int rand_num = taosRandom() % 100; + if (rand_num < stbInfo->disorderRatio) { + (*disorderRange)--; + if (0 == *disorderRange) { + *disorderRange = stbInfo->disorderRange; + } + disorderTs = startTimestamp - *disorderRange; + debugPrint("rand_num: %d, < disorderRatio: %d, " + "disorderTs: %"PRId64"\n", + rand_num, stbInfo->disorderRatio, + disorderTs); + } + } + return disorderTs; +} + +static void *syncWriteInterlace(void *sarg) { + threadInfo * pThreadInfo = (threadInfo *)sarg; + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + infoPrint( + "thread[%d] start interlace inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to); + + int64_t insertRows = stbInfo->insertRows; + int32_t interlaceRows = stbInfo->interlaceRows; + int64_t pos = 0; + uint32_t batchPerTblTimes = g_arguments->reqPerReq / interlaceRows; + uint64_t lastPrintTime = toolsGetTimestampMs(); + uint64_t lastTotalInsertRows = 0; + int64_t startTs = toolsGetTimestampUs(); + int64_t endTs; + uint64_t tableSeq = pThreadInfo->start_table_from; + int disorderRange = stbInfo->disorderRange; + + while (insertRows > 0) { + int64_t tmp_total_insert_rows = 0; + uint32_t generated = 0; + if (insertRows <= interlaceRows) { + interlaceRows = insertRows; + } + for (int i = 0; i < batchPerTblTimes; i++) { + if (g_arguments->terminate) { + goto free_of_interlace; + } + int64_t timestamp = pThreadInfo->start_time; + SChildTable *childTbl = stbInfo->childTblArray[tableSeq]; + char * tableName = + stbInfo->childTblArray[tableSeq]->name; + char *sampleDataBuf = childTbl->useOwnSample? + childTbl->sampleDataBuf: + stbInfo->sampleDataBuf; + char ttl[SMALL_BUFF_LEN] = ""; + if (stbInfo->ttl != 0) { + snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); + } + switch (stbInfo->iface) { + case REST_IFACE: + case TAOSC_IFACE: { + char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; + if (g_arguments->escape_character) { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "`%s`", + tableName); + } else { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "%s", + tableName); + } + if (i == 0) { + ds_add_str(&pThreadInfo->buffer, STR_INSERT_INTO); + } + if (stbInfo->partialColNum == stbInfo->cols->size) { + if (stbInfo->autoTblCreating) { + ds_add_strs(&pThreadInfo->buffer, 8, + escapedTbName, + " USING `", + stbInfo->stbName, + "` TAGS (", + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, + ") ", ttl, " VALUES "); + } else { + ds_add_strs(&pThreadInfo->buffer, 2, + escapedTbName, " VALUES "); + } + } else { + if (stbInfo->autoTblCreating) { + ds_add_strs(&pThreadInfo->buffer, 10, + escapedTbName, + " (", + stbInfo->partialColNameBuf, + ") USING `", + stbInfo->stbName, + "` TAGS (", + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, + ") ", ttl, " VALUES "); + } else { + ds_add_strs(&pThreadInfo->buffer, 4, + escapedTbName, + "(", + stbInfo->partialColNameBuf, + ") VALUES "); + } + } + + for (int64_t j = 0; j < interlaceRows; j++) { + int64_t disorderTs = getDisorderTs(stbInfo, + &disorderRange); + char time_string[BIGINT_BUFF_LEN]; + snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", + disorderTs?disorderTs:timestamp); + ds_add_strs(&pThreadInfo->buffer, 5, + "(", + time_string, + ",", + sampleDataBuf + pos * stbInfo->lenOfCols, + ") "); + if (ds_len(pThreadInfo->buffer) + > stbInfo->max_sql_len) { + errorPrint("sql buffer length (%"PRIu64") " + "is larger than max sql length " + "(%"PRId64")\n", + ds_len(pThreadInfo->buffer), + stbInfo->max_sql_len); + goto free_of_interlace; + } + generated++; + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + timestamp += stbInfo->timestamp_step; + } + break; + } + case STMT_IFACE: { + char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; + if (g_arguments->escape_character) { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, + "`%s`", tableName); + } else { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", + tableName); + } + if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, + escapedTbName)) { + errorPrint( + "taos_stmt_set_tbname(%s) failed, reason: %s\n", + tableName, + taos_stmt_errstr(pThreadInfo->conn->stmt)); + g_fail = true; + goto free_of_interlace; + } + generated = + bindParamBatch(pThreadInfo, interlaceRows, + timestamp, childTbl); + break; + } + case SML_REST_IFACE: + case SML_IFACE: { + int protocol = stbInfo->lineProtocol; + for (int64_t j = 0; j < interlaceRows; j++) { + int64_t disorderTs = getDisorderTs(stbInfo, + &disorderRange); + if (TSDB_SML_JSON_PROTOCOL == protocol) { + tools_cJSON *tag = tools_cJSON_Duplicate( + tools_cJSON_GetArrayItem( + pThreadInfo->sml_json_tags, + (int)tableSeq - + pThreadInfo->start_table_from), + true); + generateSmlJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, + disorderTs?disorderTs:timestamp); + } else if (SML_JSON_TAOS_FORMAT == protocol) { + tools_cJSON *tag = tools_cJSON_Duplicate( + tools_cJSON_GetArrayItem( + pThreadInfo->sml_json_tags, + (int)tableSeq - + pThreadInfo->start_table_from), + true); + generateSmlTaosJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, + disorderTs?disorderTs:timestamp); + } else if (TSDB_SML_LINE_PROTOCOL == protocol) { + snprintf( + pThreadInfo->lines[generated], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %s %" PRId64 "", + pThreadInfo + ->sml_tags[(int)tableSeq - + pThreadInfo->start_table_from], + sampleDataBuf + pos * stbInfo->lenOfCols, + disorderTs?disorderTs:timestamp); + } else { + snprintf( + pThreadInfo->lines[generated], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %" PRId64 " %s %s", stbInfo->stbName, + disorderTs?disorderTs:timestamp, + sampleDataBuf + pos * stbInfo->lenOfCols, + pThreadInfo + ->sml_tags[(int)tableSeq - + pThreadInfo->start_table_from]); + } + generated++; + timestamp += stbInfo->timestamp_step; + } + if (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) { + pThreadInfo->lines[0] = + tools_cJSON_PrintUnformatted( + pThreadInfo->json_array); + } + break; + } + } + tableSeq++; + tmp_total_insert_rows += interlaceRows; + if (tableSeq > pThreadInfo->end_table_to) { + tableSeq = pThreadInfo->start_table_from; + pThreadInfo->start_time += + interlaceRows * stbInfo->timestamp_step; + if (!stbInfo->non_stop) { + insertRows -= interlaceRows; + } + if (stbInfo->insert_interval > 0) { + debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", + __func__, __LINE__, stbInfo->insert_interval); + perfPrint("sleep %" PRIu64 " ms\n", + stbInfo->insert_interval); + toolsMsleep((int32_t)stbInfo->insert_interval); + } + break; + } + } + + startTs = toolsGetTimestampUs(); + if (execInsert(pThreadInfo, generated)) { + g_fail = true; + goto free_of_interlace; + } + endTs = toolsGetTimestampUs(); + + pThreadInfo->totalInsertRows += tmp_total_insert_rows; + + if (g_arguments->terminate) { + goto free_of_interlace; + } + + int protocol = stbInfo->lineProtocol; + switch (stbInfo->iface) { + case TAOSC_IFACE: + case REST_IFACE: + debugPrint("pThreadInfo->buffer: %s\n", + pThreadInfo->buffer); + free_ds(&pThreadInfo->buffer); + pThreadInfo->buffer = new_ds(0); + break; + case SML_REST_IFACE: + memset(pThreadInfo->buffer, 0, + g_arguments->reqPerReq * (pThreadInfo->max_sql_len + 1)); + case SML_IFACE: + if (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) { + debugPrint("pThreadInfo->lines[0]: %s\n", + pThreadInfo->lines[0]); + if (pThreadInfo->json_array && !g_arguments->terminate) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + pThreadInfo->json_array = tools_cJSON_CreateArray(); + if (pThreadInfo->lines && pThreadInfo->lines[0]) { + tmfree(pThreadInfo->lines[0]); + pThreadInfo->lines[0] = NULL; + } + } else { + for (int j = 0; j < generated; j++) { + if (pThreadInfo && pThreadInfo->lines + && !g_arguments->terminate) { + debugPrint("pThreadInfo->lines[%d]: %s\n", j, + pThreadInfo->lines[j]); + memset(pThreadInfo->lines[j], 0, + pThreadInfo->max_sql_len); + } + } + } + break; + case STMT_IFACE: + break; + } + + int64_t delay = endTs - startTs; + if (delay <=0) { + debugPrint("thread[%d]: startTS: %"PRId64", endTS: %"PRId64"\n", + pThreadInfo->threadID, startTs, endTs); + } else { + perfPrint("insert execution time is %10.2f ms\n", + delay / 1E6); + + int64_t * pdelay = benchCalloc(1, sizeof(int64_t), false); + *pdelay = delay; + if (benchArrayPush(pThreadInfo->delayList, pdelay) == NULL) { + tmfree(pdelay); + } + pThreadInfo->totalDelay += delay; + } + + int64_t currentPrintTime = toolsGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30 * 1000) { + infoPrint( + "thread[%d] has currently inserted rows: %" PRIu64 + ", peroid insert rate: %.3f rows/s \n", + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); + lastPrintTime = currentPrintTime; + lastTotalInsertRows = pThreadInfo->totalInsertRows; + } + } +free_of_interlace: + cleanupAndPrint(pThreadInfo, "interlace"); + return NULL; +} + +static int32_t prepareProgressDataStmt( + threadInfo *pThreadInfo, + SChildTable *childTbl, + int64_t *timestamp, uint64_t i, char *ttl) { + SSuperTable *stbInfo = pThreadInfo->stbInfo; + char escapedTbName[TSDB_TABLE_NAME_LEN + 2] = "\0"; + if (g_arguments->escape_character) { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN + 2, + "`%s`", childTbl->name); + } else { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", + childTbl->name); + } + if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, + escapedTbName)) { + errorPrint( + "taos_stmt_set_tbname(%s) failed," + "reason: %s\n", escapedTbName, + taos_stmt_errstr(pThreadInfo->conn->stmt)); + return -1; + } + int32_t generated = bindParamBatch( + pThreadInfo, + (g_arguments->reqPerReq > (stbInfo->insertRows - i)) + ? (stbInfo->insertRows - i) + : g_arguments->reqPerReq, + *timestamp, childTbl); + *timestamp += generated * stbInfo->timestamp_step; + return generated; +} + +static void makeTimestampDisorder( + int64_t *timestamp, SSuperTable *stbInfo) { + int64_t startTimestamp = stbInfo->startTimestamp; + int disorderRange = stbInfo->disorderRange; + int rand_num = taosRandom() % 100; + if (rand_num < stbInfo->disorderRatio) { + disorderRange--; + if (0 == disorderRange) { + disorderRange = stbInfo->disorderRange; + } + *timestamp = startTimestamp - disorderRange; + debugPrint("rand_num: %d, < disorderRatio: %d" + ", ts: %"PRId64"\n", + rand_num, + stbInfo->disorderRatio, + *timestamp); + } +} + +static int32_t prepareProgressDataSmlJsonText( + threadInfo *pThreadInfo, + uint64_t tableSeq, + int64_t *timestamp, uint64_t i, char *ttl) { + // prepareProgressDataSmlJsonText + SSuperTable *stbInfo = pThreadInfo->stbInfo; + int32_t generated = 0; + + int len = 0; + + char *line = pThreadInfo->lines[0]; + uint32_t line_buf_len = pThreadInfo->line_buf_len; + + strncat(line + len, "[", 2); + len += 1; + + int32_t pos = 0; + for (int j = 0; (j < g_arguments->reqPerReq) + && !g_arguments->terminate; j++) { + strncat(line + len, "{", 2); + len += 1; + int n; + n = snprintf(line + len, line_buf_len - len, + "\"timestamp\":%"PRId64",", *timestamp); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + + n = snprintf(line + len, line_buf_len - len, "%s", + pThreadInfo->sml_json_value_array[tableSeq]); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + n = snprintf(line + len, line_buf_len - len, "\"tags\":%s,", + pThreadInfo->sml_tags_json_array[tableSeq]); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + n = snprintf(line + len, line_buf_len - len, + "\"metric\":\"%s\"}", stbInfo->stbName); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + *timestamp += stbInfo->timestamp_step; + if (stbInfo->disorderRatio > 0) { + makeTimestampDisorder(timestamp, stbInfo); + } + generated++; + if (i + generated >= stbInfo->insertRows) { + break; + } + if ((j+1) < g_arguments->reqPerReq) { + strncat(line + len, ",", 2); + len += 1; + } + } + strncat(line + len, "]", 2); + + debugPrint("%s() LN%d, lines[0]: %s\n", + __func__, __LINE__, pThreadInfo->lines[0]); + return generated; +} + +static int32_t prepareProgressDataSmlJson( + threadInfo *pThreadInfo, + uint64_t tableSeq, + int64_t *timestamp, uint64_t i, char *ttl) { + // prepareProgressDataSmlJson + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + int32_t generated = 0; + + int32_t pos = 0; + int protocol = stbInfo->lineProtocol; + for (int j = 0; (j < g_arguments->reqPerReq) + && !g_arguments->terminate; j++) { + tools_cJSON *tag = tools_cJSON_Duplicate( + tools_cJSON_GetArrayItem( + pThreadInfo->sml_json_tags, + (int)tableSeq - + pThreadInfo->start_table_from), + true); + debugPrintJsonNoTime(tag); + if (TSDB_SML_JSON_PROTOCOL == protocol) { + generateSmlJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, *timestamp); + } else { + generateSmlTaosJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, *timestamp); + } + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + *timestamp += stbInfo->timestamp_step; + if (stbInfo->disorderRatio > 0) { + makeTimestampDisorder(timestamp, stbInfo); + } + generated++; + if (i + generated >= stbInfo->insertRows) { + break; + } + } + + tmfree(pThreadInfo->lines[0]); + pThreadInfo->lines[0] = NULL; + pThreadInfo->lines[0] = + tools_cJSON_PrintUnformatted( + pThreadInfo->json_array); + debugPrint("pThreadInfo->lines[0]: %s\n", + pThreadInfo->lines[0]); + + return generated; +} + +static int32_t prepareProgressDataSmlLineOrTelnet( + threadInfo *pThreadInfo, uint64_t tableSeq, char *sampleDataBuf, + int64_t *timestamp, uint64_t i, char *ttl, int protocol) { + // prepareProgressDataSmlLine + SSuperTable *stbInfo = pThreadInfo->stbInfo; + int32_t generated = 0; + + int32_t pos = 0; + for (int j = 0; (j < g_arguments->reqPerReq) + && !g_arguments->terminate; j++) { + if (TSDB_SML_LINE_PROTOCOL == protocol) { + snprintf( + pThreadInfo->lines[j], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %s %" PRId64 "", + pThreadInfo->sml_tags[tableSeq + - pThreadInfo->start_table_from], + sampleDataBuf + pos * stbInfo->lenOfCols, + *timestamp); + } else { + snprintf( + pThreadInfo->lines[j], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %" PRId64 " %s %s", stbInfo->stbName, + *timestamp, + sampleDataBuf + + pos * stbInfo->lenOfCols, + pThreadInfo->sml_tags[tableSeq + -pThreadInfo->start_table_from]); + } + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + *timestamp += stbInfo->timestamp_step; + if (stbInfo->disorderRatio > 0) { + makeTimestampDisorder(timestamp, stbInfo); + } + generated++; + if (i + generated >= stbInfo->insertRows) { + break; + } + } + return generated; +} + +static int32_t prepareProgressDataSml( + threadInfo *pThreadInfo, + SChildTable *childTbl, + uint64_t tableSeq, + int64_t *timestamp, uint64_t i, char *ttl) { + // prepareProgressDataSml + SSuperTable *stbInfo = pThreadInfo->stbInfo; + + char *sampleDataBuf; + if (childTbl->useOwnSample) { + sampleDataBuf = childTbl->sampleDataBuf; + } else { + sampleDataBuf = stbInfo->sampleDataBuf; + } + int protocol = stbInfo->lineProtocol; + int32_t generated = -1; + switch (protocol) { + case TSDB_SML_LINE_PROTOCOL: + case TSDB_SML_TELNET_PROTOCOL: + generated = prepareProgressDataSmlLineOrTelnet( + pThreadInfo, + tableSeq, + sampleDataBuf, + timestamp, i, ttl, protocol); + break; + case TSDB_SML_JSON_PROTOCOL: + generated = prepareProgressDataSmlJsonText( + pThreadInfo, + tableSeq - pThreadInfo->start_table_from, + timestamp, i, ttl); + break; + case SML_JSON_TAOS_FORMAT: + generated = prepareProgressDataSmlJson( + pThreadInfo, + tableSeq, + timestamp, i, ttl); + break; + default: + errorPrint("%s() LN%d: unknown protcolor: %d\n", + __func__, __LINE__, protocol); + break; + } + + return generated; +} + +static int32_t prepareProgressDataSql( + threadInfo *pThreadInfo, + SChildTable *childTbl, uint64_t tableSeq, + char *sampleDataBuf, + int64_t *timestamp, uint64_t i, char *ttl, + int32_t *pos, uint64_t *len) { + // prepareProgressDataSql + int32_t generated = 0; + SDataBase *database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + char * pstr = pThreadInfo->buffer; + int disorderRange = stbInfo->disorderRange; + + if (stbInfo->partialColNum == stbInfo->cols->size) { + if (stbInfo->autoTblCreating) { + *len = + snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s VALUES " + : "%s %s.%s USING %s.%s TAGS (%s) %s VALUES ", + STR_INSERT_INTO, database->dbName, + childTbl->name, database->dbName, + stbInfo->stbName, + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, ttl); + } else { + *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` VALUES " + : "%s %s.%s VALUES ", + STR_INSERT_INTO, + database->dbName, childTbl->name); + } + } else { + if (stbInfo->autoTblCreating) { + *len = snprintf( + pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` (%s) USING `%s`.`%s` TAGS (%s) %s VALUES " + : "%s %s.%s (%s) USING %s.%s TAGS (%s) %s VALUES ", + STR_INSERT_INTO, database->dbName, + childTbl->name, + stbInfo->partialColNameBuf, + database->dbName, stbInfo->stbName, + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, ttl); + } else { + *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` (%s) VALUES " + : "%s %s.%s (%s) VALUES ", + STR_INSERT_INTO, database->dbName, + childTbl->name, + stbInfo->partialColNameBuf); + } + } + + char *ownSampleDataBuf; + if (childTbl->useOwnSample) { + debugPrint("%s is using own sample data\n", + childTbl->name); + ownSampleDataBuf = childTbl->sampleDataBuf; + } else { + ownSampleDataBuf = stbInfo->sampleDataBuf; + } + for (int j = 0; j < g_arguments->reqPerReq; j++) { + if (stbInfo->useSampleTs + && (!stbInfo->random_data_source)) { + *len += + snprintf(pstr + *len, + TSDB_MAX_ALLOWED_SQL_LEN - *len, "(%s)", + sampleDataBuf + + *pos * stbInfo->lenOfCols); + } else { + int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); + *len += snprintf(pstr + *len, + TSDB_MAX_ALLOWED_SQL_LEN - *len, + "(%" PRId64 ",%s)", + disorderTs?disorderTs:*timestamp, + ownSampleDataBuf + + *pos * stbInfo->lenOfCols); + } + *pos += 1; + if (*pos >= g_arguments->prepared_rand) { + *pos = 0; + } + *timestamp += stbInfo->timestamp_step; + generated++; + if (*len > (TSDB_MAX_ALLOWED_SQL_LEN + - stbInfo->lenOfCols)) { + break; + } + if (i + generated >= stbInfo->insertRows) { + break; + } + } + + return generated; +} + +void *syncWriteProgressive(void *sarg) { + threadInfo * pThreadInfo = (threadInfo *)sarg; + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + + // special deal flow for TAOSC_IFACE + if (insertDataMix(pThreadInfo, database, stbInfo)) { + // request be dealt by this function , so return + return NULL; + } + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto) { + if (0 == pThreadInfo->vg->tbCountPerVgId) { + return NULL; + } + } else { + infoPrint( + "thread[%d] start progressive inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to + 1); + } +#else + infoPrint( + "thread[%d] start progressive inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to + 1); +#endif + uint64_t lastPrintTime = toolsGetTimestampMs(); + uint64_t lastTotalInsertRows = 0; + int64_t startTs = toolsGetTimestampUs(); + int64_t endTs; + + for (uint64_t tableSeq = pThreadInfo->start_table_from; + tableSeq <= pThreadInfo->end_table_to; tableSeq++) { + char *sampleDataBuf; + SChildTable *childTbl; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto) { + childTbl = pThreadInfo->vg->childTblArray[tableSeq]; + } else { + childTbl = stbInfo->childTblArray[ + stbInfo->childTblExists? + tableSeq: + stbInfo->childTblFrom + tableSeq]; + } +#else + childTbl = stbInfo->childTblArray[ + stbInfo->childTblExists? + tableSeq: + stbInfo->childTblFrom + tableSeq]; +#endif + if (childTbl->useOwnSample) { + sampleDataBuf = childTbl->sampleDataBuf; + } else { + sampleDataBuf = stbInfo->sampleDataBuf; + } + + int64_t timestamp = pThreadInfo->start_time; + uint64_t len = 0; + int32_t pos = 0; + if (stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating) { + taos_stmt_close(pThreadInfo->conn->stmt); + pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + if (NULL == pThreadInfo->conn->stmt) { + errorPrint("taos_stmt_init() failed, reason: %s\n", + taos_errstr(NULL)); + g_fail = true; + goto free_of_progressive; + } + + if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tableSeq)) { + g_fail = true; + goto free_of_progressive; + } + } + + char ttl[SMALL_BUFF_LEN] = ""; + if (stbInfo->ttl != 0) { + snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); + } + for (uint64_t i = 0; i < stbInfo->insertRows;) { + if (g_arguments->terminate) { + goto free_of_progressive; + } + int32_t generated = 0; + switch (stbInfo->iface) { + case TAOSC_IFACE: + case REST_IFACE: + generated = prepareProgressDataSql( + pThreadInfo, + childTbl, + tableSeq, + sampleDataBuf, + ×tamp, i, ttl, &pos, &len); + break; + case STMT_IFACE: { + generated = prepareProgressDataStmt( + pThreadInfo, + childTbl, ×tamp, i, ttl); + break; + } + case SML_REST_IFACE: + case SML_IFACE: + generated = prepareProgressDataSml( + pThreadInfo, + childTbl, + tableSeq, ×tamp, i, ttl); + break; + default: + break; + } + if (generated < 0) { + g_fail = true; + goto free_of_progressive; + } + if (!stbInfo->non_stop) { + i += generated; + } + // only measure insert + startTs = toolsGetTimestampUs(); + int code = execInsert(pThreadInfo, generated); + if (code) { + if (NO_IF_FAILED == stbInfo->continueIfFail) { + warnPrint("The super table parameter " + "continueIfFail: %d, STOP insertion!\n", + stbInfo->continueIfFail); + g_fail = true; + goto free_of_progressive; + } else if (YES_IF_FAILED == stbInfo->continueIfFail) { + infoPrint("The super table parameter " + "continueIfFail: %d, " + "will continue to insert ..\n", + stbInfo->continueIfFail); + } else if (SMART_IF_FAILED == stbInfo->continueIfFail) { + warnPrint("The super table parameter " + "continueIfFail: %d, will create table " + "then insert ..\n", + stbInfo->continueIfFail); + int ret = smartContinueIfFail( + pThreadInfo, + childTbl, i, ttl); + if (0 != ret) { + g_fail = true; + goto free_of_progressive; + } + + code = execInsert(pThreadInfo, generated); + if (code) { + g_fail = true; + goto free_of_progressive; + } + } else { + warnPrint("Unknown super table parameter " + "continueIfFail: %d\n", + stbInfo->continueIfFail); + g_fail = true; + goto free_of_progressive; + } + } + endTs = toolsGetTimestampUs()+1; + + if (stbInfo->insert_interval > 0) { + debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", + __func__, __LINE__, stbInfo->insert_interval); + perfPrint("sleep %" PRIu64 " ms\n", + stbInfo->insert_interval); + toolsMsleep((int32_t)stbInfo->insert_interval); + } + + pThreadInfo->totalInsertRows += generated; + + if (g_arguments->terminate) { + goto free_of_progressive; + } + int protocol = stbInfo->lineProtocol; + switch (stbInfo->iface) { + case REST_IFACE: + case TAOSC_IFACE: + memset(pThreadInfo->buffer, 0, pThreadInfo->max_sql_len); + break; + case SML_REST_IFACE: + memset(pThreadInfo->buffer, 0, + g_arguments->reqPerReq * + (pThreadInfo->max_sql_len + 1)); + case SML_IFACE: + if (TSDB_SML_JSON_PROTOCOL == protocol) { + memset(pThreadInfo->lines[0], 0, + pThreadInfo->line_buf_len); + } else if (SML_JSON_TAOS_FORMAT == protocol) { + if (pThreadInfo->lines && pThreadInfo->lines[0]) { + tmfree(pThreadInfo->lines[0]); + pThreadInfo->lines[0] = NULL; + } + if (pThreadInfo->json_array) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + pThreadInfo->json_array = tools_cJSON_CreateArray(); + } else { + for (int j = 0; j < generated; j++) { + debugPrint("pThreadInfo->lines[%d]: %s\n", + j, pThreadInfo->lines[j]); + memset(pThreadInfo->lines[j], 0, + pThreadInfo->max_sql_len); + } + } + break; + case STMT_IFACE: + break; + } + + int64_t delay = endTs - startTs; + if (delay <= 0) { + debugPrint("thread[%d]: startTs: %"PRId64", endTs: %"PRId64"\n", + pThreadInfo->threadID, startTs, endTs); + } else { + perfPrint("insert execution time is %.6f s\n", + delay / 1E6); + + int64_t * pDelay = benchCalloc(1, sizeof(int64_t), false); + *pDelay = delay; + if (benchArrayPush(pThreadInfo->delayList, pDelay) == NULL) { + tmfree(pDelay); + } + pThreadInfo->totalDelay += delay; + } + + int64_t currentPrintTime = toolsGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30 * 1000) { + infoPrint( + "thread[%d] has currently inserted rows: " + "%" PRId64 ", peroid insert rate: %.3f rows/s \n", + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); + lastPrintTime = currentPrintTime; + lastTotalInsertRows = pThreadInfo->totalInsertRows; + } + if (i >= stbInfo->insertRows) { + break; + } + } // insertRows + } // tableSeq +free_of_progressive: + cleanupAndPrint(pThreadInfo, "progressive"); + return NULL; +} + +static int initStmtDataValue(SSuperTable *stbInfo, SChildTable *childTbl) { + int32_t columnCount = stbInfo->cols->size; + + char *sampleDataBuf; + if (childTbl) { + sampleDataBuf = childTbl->sampleDataBuf; + } else { + sampleDataBuf = stbInfo->sampleDataBuf; + } + int64_t lenOfOneRow = stbInfo->lenOfCols; + + if (stbInfo->useSampleTs) { + columnCount += 1; // for skipping first column + } + for (int i=0; i < g_arguments->prepared_rand; i++) { + int cursor = 0; + + for (int c = 0; c < columnCount; c++) { + char *restStr = sampleDataBuf + + lenOfOneRow * i + cursor; + int lengthOfRest = strlen(restStr); + + int index = 0; + for (index = 0; index < lengthOfRest; index++) { + if (restStr[index] == ',') { + break; + } + } + + cursor += index + 1; // skip ',' too + if ((0 == c) && stbInfo->useSampleTs) { + continue; + } + + char *tmpStr = calloc(1, index + 1); + if (NULL == tmpStr) { + errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + __func__, __LINE__, index + 1); + return -1; + } + Field *col = benchArrayGet(stbInfo->cols, + (stbInfo->useSampleTs?c-1:c)); + char dataType = col->type; + + StmtData *stmtData; + if (childTbl) { + ChildField *childCol = + benchArrayGet(childTbl->childCols, + (stbInfo->useSampleTs?c-1:c)); + stmtData = &childCol->stmtData; + } else { + stmtData = &col->stmtData; + } + + strncpy(tmpStr, restStr, index); + + if (0 == strcmp(tmpStr, "NULL")) { + *(stmtData->is_null + i) = true; + } else { + switch (dataType) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *((int32_t*)stmtData->data + i) = atoi(tmpStr); + break; + case TSDB_DATA_TYPE_FLOAT: + *((float*)stmtData->data +i) = (float)atof(tmpStr); + break; + case TSDB_DATA_TYPE_DOUBLE: + *((double*)stmtData->data + i) = atof(tmpStr); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *((int16_t*)stmtData->data + i) = (int16_t)atoi(tmpStr); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); + break; + case TSDB_DATA_TYPE_BOOL: + *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + { + size_t tmpLen = strlen(tmpStr); + debugPrint("%s() LN%d, index: %d, " + "tmpStr len: %"PRIu64", col->length: %d\n", + __func__, __LINE__, + i, (uint64_t)tmpLen, col->length); + if (tmpLen-2 > col->length) { + errorPrint("data length %"PRIu64" " + "is larger than column length %d\n", + (uint64_t)tmpLen, col->length); + } + if (tmpLen > 2) { + strncpy((char *)stmtData->data + + i * col->length, + tmpStr+1, + min(col->length, tmpLen - 2)); + } else { + strncpy((char *)stmtData->data + + i*col->length, + "", 1); + } + } + break; + default: + break; + } + } + free(tmpStr); + } + } + return 0; +} + +static void initStmtData(char dataType, void **data, uint32_t length) { + char *tmpP = NULL; + + switch (dataType) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + tmpP = calloc(1, sizeof(int) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + tmpP = calloc(1, sizeof(int16_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_BOOL: + tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_FLOAT: + tmpP = calloc(1, sizeof(float) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_DOUBLE: + tmpP = calloc(1, sizeof(double) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + tmpP = calloc(1, g_arguments->prepared_rand * length); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_TIMESTAMP: + tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + default: + errorPrint("Unknown data type: %s\n", + convertDatatypeToString(dataType)); + exit(EXIT_FAILURE); + } +} + +static int parseBufferToStmtBatchChildTbl(SSuperTable *stbInfo, + SChildTable* childTbl) { + int32_t columnCount = stbInfo->cols->size; + + for (int c = 0; c < columnCount; c++) { + Field *col = benchArrayGet(stbInfo->cols, c); + ChildField *childCol = benchArrayGet(childTbl->childCols, c); + char dataType = col->type; + + char *is_null = benchCalloc( + 1, sizeof(char) *g_arguments->prepared_rand, false); + + tmfree(childCol->stmtData.is_null); + childCol->stmtData.is_null = is_null; + + initStmtData(dataType, &(childCol->stmtData.data), col->length); + } + + return initStmtDataValue(stbInfo, childTbl); +} + +static int parseBufferToStmtBatch(SSuperTable* stbInfo) { + int32_t columnCount = stbInfo->cols->size; + + for (int c = 0; c < columnCount; c++) { + Field *col = benchArrayGet(stbInfo->cols, c); + char dataType = col->type; + + char *is_null = benchCalloc( + 1, sizeof(char) *g_arguments->prepared_rand, false); + tmfree(col->stmtData.is_null); + col->stmtData.is_null = is_null; + + initStmtData(dataType, &(col->stmtData.data), col->length); + } + + return initStmtDataValue(stbInfo, NULL); +} + +static int64_t fillChildTblNameByCount(SSuperTable *stbInfo) { + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + snprintf(stbInfo->childTblArray[i]->name, + TSDB_TABLE_NAME_LEN, + "%s%" PRIu64 "", + stbInfo->childTblPrefix, i); + debugPrint("%s(): %s\n", __func__, + stbInfo->childTblArray[i]->name); + } + + return stbInfo->childTblCount; +} + +static int64_t fillChildTblNameByFromTo(SDataBase *database, + SSuperTable* stbInfo) { + for (int64_t i = stbInfo->childTblFrom; i < stbInfo->childTblTo; i++) { + snprintf(stbInfo->childTblArray[i-stbInfo->childTblFrom]->name, + TSDB_TABLE_NAME_LEN, + "%s%" PRIu64 "", + stbInfo->childTblPrefix, i); + } + + return (stbInfo->childTblTo-stbInfo->childTblFrom); +} + +static int64_t fillChildTblNameByLimitOffset(SDataBase *database, + SSuperTable* stbInfo) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + if (g_arguments->taosc_version == 3) { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT DISTINCT(TBNAME) FROM %s.`%s` LIMIT %" PRId64 + " OFFSET %" PRIu64 "", + database->dbName, stbInfo->stbName, stbInfo->childTblLimit, + stbInfo->childTblOffset); + } else { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT TBNAME FROM %s.`%s` LIMIT %" PRId64 + " OFFSET %" PRIu64 "", + database->dbName, stbInfo->stbName, stbInfo->childTblLimit, + stbInfo->childTblOffset); + } + debugPrint("cmd: %s\n", cmd); + TAOS_RES *res = taos_query(conn->taos, cmd); + int32_t code = taos_errno(res); + int64_t count = 0; + if (code) { + printErrCmdCodeStr(cmd, code, res); + closeBenchConn(conn); + return -1; + } + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res)) != NULL) { + int *lengths = taos_fetch_lengths(res); + strncpy(stbInfo->childTblArray[count]->name, row[0], lengths[0]); + stbInfo->childTblArray[count]->name[lengths[0] + 1] = '\0'; + debugPrint("stbInfo->childTblArray[%" PRId64 "]->name: %s\n", + count, stbInfo->childTblArray[count]->name); + count++; + } + taos_free_result(res); + closeBenchConn(conn); + return count; +} + +static void preProcessArgument(SSuperTable *stbInfo) { + if (stbInfo->interlaceRows > g_arguments->reqPerReq) { + infoPrint( + "interlaceRows(%d) is larger than record per request(%u), which " + "will be set to %u\n", + stbInfo->interlaceRows, g_arguments->reqPerReq, + g_arguments->reqPerReq); + stbInfo->interlaceRows = g_arguments->reqPerReq; + } + + if (stbInfo->interlaceRows > stbInfo->insertRows) { + infoPrint( + "interlaceRows larger than insertRows %d > %" PRId64 "\n", + stbInfo->interlaceRows, stbInfo->insertRows); + infoPrint("%s", "interlaceRows will be set to 0\n"); + stbInfo->interlaceRows = 0; + } + + if (stbInfo->interlaceRows == 0 + && g_arguments->reqPerReq > stbInfo->insertRows) { + infoPrint("record per request (%u) is larger than " + "insert rows (%"PRIu64")" + " in progressive mode, which will be set to %"PRIu64"\n", + g_arguments->reqPerReq, stbInfo->insertRows, + stbInfo->insertRows); + g_arguments->reqPerReq = stbInfo->insertRows; + } + + if (stbInfo->interlaceRows > 0 && stbInfo->iface == STMT_IFACE + && stbInfo->autoTblCreating) { + infoPrint("%s", + "not support autocreate table with interlace row in stmt " + "insertion, will change to progressive mode\n"); + stbInfo->interlaceRows = 0; + } +} + +static int printTotalDelay(SDataBase *database, + int64_t totalDelay, + BArray *total_delay_list, + int threads, + int64_t totalInsertRows, + int64_t start, int64_t end) { + succPrint("Spent %.6f seconds to insert rows: %" PRIu64 + " with %d thread(s) into %s %.2f records/second\n", + (end - start)/1E6, totalInsertRows, threads, + database->dbName, + (double)(totalInsertRows / ((end - start)/1E6))); + if (!total_delay_list->size) { + return -1; + } + + succPrint("insert delay, " + "min: %.4fms, " + "avg: %.4fms, " + "p90: %.4fms, " + "p95: %.4fms, " + "p99: %.4fms, " + "max: %.4fms\n", + *(int64_t *)(benchArrayGet(total_delay_list, 0))/1E3, + (double)totalDelay/total_delay_list->size/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + * 0.9)))/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + * 0.95)))/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + * 0.99)))/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + - 1)))/1E3); + return 0; +} + +static int64_t fillChildTblNameImp(SDataBase *database, SSuperTable *stbInfo) { + int64_t ntables; + if (stbInfo->childTblLimit) { + ntables = fillChildTblNameByLimitOffset(database, stbInfo); + } else if (stbInfo->childTblFrom || stbInfo->childTblTo) { + ntables = fillChildTblNameByFromTo(database, stbInfo); + } else { + ntables = fillChildTblNameByCount(stbInfo); + } + return ntables; +} + +static int64_t fillChildTblName(SDataBase *database, SSuperTable *stbInfo) { + int64_t ntables = stbInfo->childTblCount; + stbInfo->childTblArray = benchCalloc(stbInfo->childTblCount, + sizeof(SChildTable*), true); + for (int64_t child = 0; child < stbInfo->childTblCount; child++) { + stbInfo->childTblArray[child] = + benchCalloc(1, sizeof(SChildTable), true); + } + + if (stbInfo->childTblCount == 1 && stbInfo->tags->size == 0) { + // Normal table + snprintf(stbInfo->childTblArray[0]->name, TSDB_TABLE_NAME_LEN, + "%s", stbInfo->stbName); + } else if ((stbInfo->iface != SML_IFACE + && stbInfo->iface != SML_REST_IFACE) + && stbInfo->childTblExists) { + ntables = fillChildTblNameImp(database, stbInfo); + } else { + ntables = fillChildTblNameByCount(stbInfo); + } + + return ntables; +} + +static int startMultiThreadInsertData(SDataBase* database, + SSuperTable* stbInfo) { + if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) + && !stbInfo->use_metric) { + errorPrint("%s", "schemaless cannot work without stable\n"); + return -1; + } + + preProcessArgument(stbInfo); + + int64_t ntables; + if (stbInfo->childTblTo > 0) { + ntables = stbInfo->childTblTo - stbInfo->childTblFrom; + } else if (stbInfo->childTblLimit) { + ntables = stbInfo->childTblLimit; + } else { + ntables = stbInfo->childTblCount; + } + if (ntables == 0) { + return 0; + } + + uint64_t tableFrom = 0; + int32_t threads = g_arguments->nthreads; + int64_t a = 0, b = 0; + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if ((0 == stbInfo->interlaceRows) + && (g_arguments->nthreads_auto)) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); + closeBenchConn(conn); + return -1; + } + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", + database->dbName, + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + if (vgId == vg->vgId) { + vg->tbCountPerVgId++; + } + } + } + + threads = 0; + for (int v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + infoPrint("Total %"PRId64" tables on bb %s's vgroup %d (id: %d)\n", + vg->tbCountPerVgId, database->dbName, v, vg->vgId); + if (vg->tbCountPerVgId) { + threads++; + } else { + continue; + } + vg->childTblArray = benchCalloc( + vg->tbCountPerVgId, sizeof(SChildTable*), true); + vg->tbOffset = 0; + } + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); + + closeBenchConn(conn); + return -1; + } + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", + database->dbName, + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + if (vgId == vg->vgId) { + vg->childTblArray[vg->tbOffset] = + stbInfo->childTblArray[i]; + vg->tbOffset++; + } + } + } + closeBenchConn(conn); + } else { + a = ntables / threads; + if (a < 1) { + threads = (int32_t)ntables; + a = 1; + } + b = 0; + if (threads != 0) { + b = ntables % threads; + } + } + + int32_t vgFrom = 0; +#else + a = ntables / threads; + if (a < 1) { + threads = (int32_t)ntables; + a = 1; + } + b = 0; + if (threads != 0) { + b = ntables % threads; + } +#endif // TD_VER_COMPATIBLE_3_0_0_0 + pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), true); + threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), true); + + for (int32_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + pThreadInfo->dbInfo = database; + pThreadInfo->stbInfo = stbInfo; + pThreadInfo->start_time = stbInfo->startTimestamp; + pThreadInfo->totalInsertRows = 0; + pThreadInfo->samplePos = 0; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if ((0 == stbInfo->interlaceRows) + && (g_arguments->nthreads_auto)) { + int32_t j; + for (j = vgFrom; i < database->vgroups; j++) { + SVGroup *vg = benchArrayGet(database->vgArray, j); + if (0 == vg->tbCountPerVgId) { + continue; + } + pThreadInfo->vg = vg; + pThreadInfo->start_table_from = 0; + pThreadInfo->ntables = vg->tbCountPerVgId; + pThreadInfo->end_table_to = vg->tbCountPerVgId-1; + break; + } + vgFrom = j + 1; + } else { + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = i < b ? a + 1 : a; + pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); + tableFrom = pThreadInfo->end_table_to + 1; + } +#else + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = i < b ? a + 1 : a; + pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); + tableFrom = pThreadInfo->end_table_to + 1; +#endif // TD_VER_COMPATIBLE_3_0_0_0 + pThreadInfo->delayList = benchArrayInit(1, sizeof(int64_t)); + switch (stbInfo->iface) { + case REST_IFACE: { + if (stbInfo->interlaceRows > 0) { + pThreadInfo->buffer = new_ds(0); + } else { + pThreadInfo->buffer = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + } + int sockfd = createSockFd(); + if (sockfd < 0) { + FREE_PIDS_INFOS_RETURN_MINUS_1(); + } + pThreadInfo->sockfd = sockfd; + break; + } + case STMT_IFACE: { + pThreadInfo->conn = initBenchConn(); + if (NULL == pThreadInfo->conn) { + FREE_PIDS_INFOS_RETURN_MINUS_1(); + } + pThreadInfo->conn->stmt = + taos_stmt_init(pThreadInfo->conn->taos); + if (NULL == pThreadInfo->conn->stmt) { + errorPrint("taos_stmt_init() failed, reason: %s\n", + taos_errstr(NULL)); + FREE_RESOURCE(); + return -1; + } + if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { + errorPrint("taos select database(%s) failed\n", + database->dbName); + FREE_RESOURCE(); + return -1; + } + if (!stbInfo->autoTblCreating) { + if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, 0)) { + FREE_RESOURCE(); + return -1; + } + } + + pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); + pThreadInfo->bind_ts_array = + benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, + true); + pThreadInfo->bindParams = benchCalloc( + 1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), + true); + pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, + true); + parseBufferToStmtBatch(stbInfo); + for (int64_t child = 0; + child < stbInfo->childTblCount; child++) { + SChildTable *childTbl = stbInfo->childTblArray[child]; + if (childTbl->useOwnSample) { + parseBufferToStmtBatchChildTbl(stbInfo, childTbl); + } + } + + break; + } + case SML_REST_IFACE: { + int sockfd = createSockFd(); + if (sockfd < 0) { + free(pids); + free(infos); + return -1; + } + pThreadInfo->sockfd = sockfd; + } + case SML_IFACE: { + pThreadInfo->conn = initBenchConn(); + if (pThreadInfo->conn == NULL) { + errorPrint("%s() init connection failed\n", __func__); + FREE_RESOURCE(); + return -1; + } + if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { + errorPrint("taos select database(%s) failed\n", + database->dbName); + FREE_RESOURCE(); + return -1; + } + pThreadInfo->max_sql_len = + stbInfo->lenOfCols + stbInfo->lenOfTags; + if (stbInfo->iface == SML_REST_IFACE) { + pThreadInfo->buffer = + benchCalloc(1, g_arguments->reqPerReq * + (1 + pThreadInfo->max_sql_len), true); + } + int protocol = stbInfo->lineProtocol; + if (TSDB_SML_JSON_PROTOCOL != protocol + && SML_JSON_TAOS_FORMAT != protocol) { + pThreadInfo->sml_tags = + (char **)benchCalloc(pThreadInfo->ntables, + sizeof(char *), true); + for (int t = 0; t < pThreadInfo->ntables; t++) { + pThreadInfo->sml_tags[t] = + benchCalloc(1, stbInfo->lenOfTags, true); + } + + for (int t = 0; t < pThreadInfo->ntables; t++) { + if (generateRandData( + stbInfo, pThreadInfo->sml_tags[t], + stbInfo->lenOfTags, + stbInfo->lenOfCols + stbInfo->lenOfTags, + stbInfo->tags, 1, true, NULL)) { + return -1; + } + debugPrint("pThreadInfo->sml_tags[%d]: %s\n", t, + pThreadInfo->sml_tags[t]); + } + pThreadInfo->lines = + benchCalloc(g_arguments->reqPerReq, + sizeof(char *), true); + + for (int j = 0; (j < g_arguments->reqPerReq + && !g_arguments->terminate); j++) { + pThreadInfo->lines[j] = + benchCalloc(1, pThreadInfo->max_sql_len, true); + } + } else { + pThreadInfo->json_array = tools_cJSON_CreateArray(); + pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); + pThreadInfo->sml_tags_json_array = (char **)benchCalloc( + pThreadInfo->ntables, sizeof(char *), true); + for (int t = 0; t < pThreadInfo->ntables; t++) { + if (stbInfo->lineProtocol == TSDB_SML_JSON_PROTOCOL) { + generateSmlJsonTags( + pThreadInfo->sml_json_tags, + pThreadInfo->sml_tags_json_array, + stbInfo, + pThreadInfo->start_table_from, t); + } else { + generateSmlTaosJsonTags( + pThreadInfo->sml_json_tags, stbInfo, + pThreadInfo->start_table_from, t); + } + } + pThreadInfo->lines = (char **)benchCalloc( + 1, sizeof(char *), true); + if ((0 == stbInfo->interlaceRows) + && (TSDB_SML_JSON_PROTOCOL == protocol)) { + pThreadInfo->line_buf_len = + g_arguments->reqPerReq * + accumulateRowLen(pThreadInfo->stbInfo->tags, + pThreadInfo->stbInfo->iface); + debugPrint("%s() LN%d, line_buf_len=%d\n", + __func__, __LINE__, pThreadInfo->line_buf_len); + pThreadInfo->lines[0] = benchCalloc( + 1, pThreadInfo->line_buf_len, true); + pThreadInfo->sml_json_value_array = + (char **)benchCalloc( + pThreadInfo->ntables, sizeof(char *), true); + for (int t = 0; t < pThreadInfo->ntables; t++) { + generateSmlJsonValues( + pThreadInfo->sml_json_value_array, stbInfo, t); + } + } + } + break; + } + case TAOSC_IFACE: { + pThreadInfo->conn = initBenchConn(); + if (pThreadInfo->conn == NULL) { + errorPrint("%s() failed to connect\n", __func__); + FREE_RESOURCE(); + return -1; + } + char* command = benchCalloc(1, SHORT_1K_SQL_BUFF_LEN, false); + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "USE `%s`" + : "USE %s", + database->dbName); + if (queryDbExecCall(pThreadInfo->conn, command)) { + errorPrint("taos select database(%s) failed\n", + database->dbName); + FREE_RESOURCE(); + tmfree(command); + return -1; + } + tmfree(command); + command = NULL; + + if (stbInfo->interlaceRows > 0) { + pThreadInfo->buffer = new_ds(0); + } else { + pThreadInfo->buffer = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + if (g_arguments->check_sql) { + pThreadInfo->csql = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + memset(pThreadInfo->csql, 0, TSDB_MAX_ALLOWED_SQL_LEN); + } + } + + break; + } + default: + break; + } + } + + infoPrint("Estimate memory usage: %.2fMB\n", + (double)g_memoryUsage / 1048576); + prompt(0); + + // create threads + for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + threadInfo *pThreadInfo = infos + i; + if (stbInfo->interlaceRows > 0) { + pthread_create(pids + i, NULL, + syncWriteInterlace, pThreadInfo); + } else { + pthread_create(pids + i, NULL, + syncWriteProgressive, pThreadInfo); + } + } + + int64_t start = toolsGetTimestampUs(); + + // wait threads + for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + pthread_join(pids[i], NULL); + } + + int64_t end = toolsGetTimestampUs()+1; + + if (g_arguments->terminate) toolsMsleep(100); + + BArray * total_delay_list = benchArrayInit(1, sizeof(int64_t)); + int64_t totalDelay = 0; + uint64_t totalInsertRows = 0; + + // free threads resource + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + // free check sql + if (pThreadInfo->csql) { + tmfree(pThreadInfo->csql); + pThreadInfo->csql = NULL; + } + + int protocol = stbInfo->lineProtocol; + switch (stbInfo->iface) { + case REST_IFACE: + if (g_arguments->terminate) + toolsMsleep(100); + destroySockFd(pThreadInfo->sockfd); + if (stbInfo->interlaceRows > 0) { + free_ds(&pThreadInfo->buffer); + } else { + tmfree(pThreadInfo->buffer); + pThreadInfo->buffer = NULL; + } + break; + case SML_REST_IFACE: + if (g_arguments->terminate) + toolsMsleep(100); + tmfree(pThreadInfo->buffer); + // on-purpose no break here + case SML_IFACE: + if (TSDB_SML_JSON_PROTOCOL != protocol + && SML_JSON_TAOS_FORMAT != protocol) { + for (int t = 0; t < pThreadInfo->ntables; t++) { + tmfree(pThreadInfo->sml_tags[t]); + } + for (int j = 0; j < g_arguments->reqPerReq; j++) { + tmfree(pThreadInfo->lines[j]); + } + tmfree(pThreadInfo->sml_tags); + pThreadInfo->sml_tags = NULL; + } else { + for (int t = 0; t < pThreadInfo->ntables; t++) { + tmfree(pThreadInfo->sml_tags_json_array[t]); + } + tmfree(pThreadInfo->sml_tags_json_array); + pThreadInfo->sml_tags_json_array = NULL; + if (pThreadInfo->sml_json_tags) { + tools_cJSON_Delete(pThreadInfo->sml_json_tags); + pThreadInfo->sml_json_tags = NULL; + } + if (pThreadInfo->json_array) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + } + if (pThreadInfo->lines) { + if ((0 == stbInfo->interlaceRows) + && (TSDB_SML_JSON_PROTOCOL == protocol)) { + tmfree(pThreadInfo->lines[0]); + for (int t = 0; t < pThreadInfo->ntables; t++) { + tmfree(pThreadInfo->sml_json_value_array[t]); + } + tmfree(pThreadInfo->sml_json_value_array); + } + tmfree(pThreadInfo->lines); + pThreadInfo->lines = NULL; + } + break; + + case STMT_IFACE: + taos_stmt_close(pThreadInfo->conn->stmt); + tmfree(pThreadInfo->bind_ts); + tmfree(pThreadInfo->bind_ts_array); + tmfree(pThreadInfo->bindParams); + tmfree(pThreadInfo->is_null); + break; + + case TAOSC_IFACE: + if (stbInfo->interlaceRows > 0) { + free_ds(&pThreadInfo->buffer); + } else { + tmfree(pThreadInfo->buffer); + pThreadInfo->buffer = NULL; + } + break; + + default: + break; + } + totalInsertRows += pThreadInfo->totalInsertRows; + totalDelay += pThreadInfo->totalDelay; + benchArrayAddBatch(total_delay_list, pThreadInfo->delayList->pData, + pThreadInfo->delayList->size); + tmfree(pThreadInfo->delayList); + pThreadInfo->delayList = NULL; + // free conn + if (pThreadInfo->conn) { + closeBenchConn(pThreadInfo->conn); + pThreadInfo->conn = NULL; + } + } + + // calculate result + qsort(total_delay_list->pData, total_delay_list->size, + total_delay_list->elemSize, compare); + + if (g_arguments->terminate) toolsMsleep(100); + + free(pids); + free(infos); + + int ret = printTotalDelay(database, totalDelay, + total_delay_list, threads, + totalInsertRows, start, end); + benchArrayDestroy(total_delay_list); + if (g_fail || ret) { + return -1; + } + return 0; +} + +static int getStbInsertedRows(char* dbName, char* stbName, TAOS* taos) { + int rows = 0; + char command[SHORT_1K_SQL_BUFF_LEN]; + snprintf(command, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(*) FROM %s.%s", + dbName, stbName); + TAOS_RES* res = taos_query(taos, command); + int code = taos_errno(res); + if (code != 0) { + printErrCmdCodeStr(command, code, res); + return -1; + } + TAOS_ROW row = taos_fetch_row(res); + if (row == NULL) { + rows = 0; + } else { + rows = (int)*(int64_t*)row[0]; + } + taos_free_result(res); + return rows; +} + +static void create_tsma(TSMA* tsma, SBenchConn* conn, char* stbName) { + char command[SHORT_1K_SQL_BUFF_LEN]; + int len = snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "CREATE sma INDEX %s ON %s function(%s) " + "INTERVAL (%s) SLIDING (%s)", + tsma->name, stbName, tsma->func, + tsma->interval, tsma->sliding); + if (tsma->custom) { + snprintf(command + len, SHORT_1K_SQL_BUFF_LEN - len, + " %s", tsma->custom); + } + int code = queryDbExecCall(conn, command); + if (code == 0) { + infoPrint("successfully create tsma with command <%s>\n", command); + } +} + +static void* create_tsmas(void* args) { + tsmaThreadInfo* pThreadInfo = (tsmaThreadInfo*) args; + int inserted_rows = 0; + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return NULL; + } + int finished = 0; + if (taos_select_db(conn->taos, pThreadInfo->dbName)) { + errorPrint("failed to use database (%s)\n", pThreadInfo->dbName); + closeBenchConn(conn); + return NULL; + } + while (finished < pThreadInfo->tsmas->size && inserted_rows >= 0) { + inserted_rows = (int)getStbInsertedRows( + pThreadInfo->dbName, pThreadInfo->stbName, conn->taos); + for (int i = 0; i < pThreadInfo->tsmas->size; i++) { + TSMA* tsma = benchArrayGet(pThreadInfo->tsmas, i); + if (!tsma->done && inserted_rows >= tsma->start_when_inserted) { + create_tsma(tsma, conn, pThreadInfo->stbName); + tsma->done = true; + finished++; + break; + } + } + toolsMsleep(10); + } + benchArrayDestroy(pThreadInfo->tsmas); + closeBenchConn(conn); + return NULL; +} + +static int32_t createStream(SSTREAM* stream) { + int32_t code = -1; + char * command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, "DROP STREAM IF EXISTS %s", + stream->stream_name); + infoPrint("%s\n", command); + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + goto END_STREAM; + } + + code = queryDbExecCall(conn, command); + int32_t trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then re-drop stream %s\n", + g_arguments->trying_interval, stream->stream_name); + toolsMsleep(g_arguments->trying_interval); + code = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + + if (code) { + closeBenchConn(conn); + goto END_STREAM; + } + + memset(command, 0, TSDB_MAX_ALLOWED_SQL_LEN); + int pos = snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, + "CREATE STREAM IF NOT EXISTS %s ", stream->stream_name); + if (stream->trigger_mode[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "TRIGGER %s ", stream->trigger_mode); + } + if (stream->watermark[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "WATERMARK %s ", stream->watermark); + } + if (stream->ignore_update[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "IGNORE UPDATE %s ", stream->ignore_update); + } + if (stream->ignore_expired[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "IGNORE EXPIRED %s ", stream->ignore_expired); + } + if (stream->fill_history[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "FILL_HISTORY %s ", stream->fill_history); + } + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "INTO %s ", stream->stream_stb); + if (stream->stream_stb_field[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "%s ", stream->stream_stb_field); + } + if (stream->stream_tag_field[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "TAGS%s ", stream->stream_tag_field); + } + if (stream->subtable[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "SUBTABLE%s ", stream->subtable); + } + snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "as %s", stream->source_sql); + infoPrint("%s\n", command); + + code = queryDbExecCall(conn, command); + trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds " + "then re-create stream %s\n", + g_arguments->trying_interval, stream->stream_name); + toolsMsleep(g_arguments->trying_interval); + code = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + + closeBenchConn(conn); +END_STREAM: + tmfree(command); + return code; +} + +int insertTestProcess() { + prompt(0); + + encodeAuthBase64(); + for (int i = 0; i < g_arguments->databases->size; i++) { + if (REST_IFACE == g_arguments->iface) { + if (0 != convertServAddr(g_arguments->iface, + false, + 1)) { + return -1; + } + } + SDataBase * database = benchArrayGet(g_arguments->databases, i); + + if (database->drop && !(g_arguments->supplementInsert)) { + if (database->superTbls) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, 0); + if (stbInfo && (REST_IFACE == stbInfo->iface)) { + if (0 != convertServAddr(stbInfo->iface, + stbInfo->tcpTransfer, + stbInfo->lineProtocol)) { + return -1; + } + } + } + if (createDatabase(database)) { + errorPrint("failed to create database (%s)\n", + database->dbName); + return -1; + } + succPrint("created database (%s)\n", database->dbName); + } + } + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (int j = 0; j < database->superTbls->size; j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + if (stbInfo->iface != SML_IFACE + && stbInfo->iface != SML_REST_IFACE + && !stbInfo->childTblExists) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + dropSuperTable(database, stbInfo); + } +#endif + if (getSuperTableFromServer(database, stbInfo) != 0) { + if (createSuperTable(database, stbInfo)) { + return -1; + } + } + } + fillChildTblName(database, stbInfo); + if (0 != prepareSampleData(database, stbInfo)) { + return -1; + } + } + } + } + + if (g_arguments->taosc_version == 3) { + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase* database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (int j = 0; (j < database->superTbls->size + && !g_arguments->terminate); j++) { + SSuperTable* stbInfo = + benchArrayGet(database->superTbls, j); + if (stbInfo->tsmas == NULL) { + continue; + } + if (stbInfo->tsmas->size > 0) { + tsmaThreadInfo* pThreadInfo = + benchCalloc(1, sizeof(tsmaThreadInfo), true); + pthread_t tsmas_pid = {0}; + pThreadInfo->dbName = database->dbName; + pThreadInfo->stbName = stbInfo->stbName; + pThreadInfo->tsmas = stbInfo->tsmas; + pthread_create(&tsmas_pid, NULL, + create_tsmas, pThreadInfo); + } + } + } + } + } + + if (createChildTables()) return -1; + + if (g_arguments->taosc_version == 3) { + for (int j = 0; j < g_arguments->streams->size; j++) { + SSTREAM * stream = benchArrayGet(g_arguments->streams, j); + if (stream->drop) { + if (createStream(stream)) { + return -1; + } + } + } + } + + // create sub threads for inserting data + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (uint64_t j = 0; j < database->superTbls->size; j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + if (stbInfo->insertRows == 0) { + continue; + } + prompt(stbInfo->non_stop); + if (startMultiThreadInsertData(database, stbInfo)) { + return -1; + } + } + } + } + return 0; +} From 0fd2167901f4dd5dced27a614eebb71fdce93cce Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 16 May 2023 16:47:16 +0800 Subject: [PATCH 13/25] test: merge develop --- src/benchInsert.c | 7168 ++++++++++++++++++++++---------------------- src/benchJsonOpt.c | 6 + src/benchTmq.c | 2 + 3 files changed, 3589 insertions(+), 3587 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index a1660539..aa4f381c 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1,3587 +1,3581 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the MIT license as published by the Free Software - * Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - */ - -#include -#include -#include - -#define FREE_PIDS_INFOS_RETURN_MINUS_1() \ - do { \ - tmfree(pids); \ - tmfree(infos); \ - return -1; \ - } while (0) - -#define FREE_RESOURCE() \ - do { \ - if (pThreadInfo->conn) \ - closeBenchConn(pThreadInfo->conn); \ - benchArrayDestroy(pThreadInfo->delayList); \ - tmfree(pids); \ - tmfree(infos); \ - } while (0) \ - -static int getSuperTableFromServerRest( - SDataBase* database, SSuperTable* stbInfo, char *command) { - - return -1; - // TODO(me): finish full implementation -#if 0 - int sockfd = createSockFd(); - if (sockfd < 0) { - return -1; - } - - int code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - - destroySockFd(sockfd); -#endif // 0 -} - -static int getSuperTableFromServerTaosc( - SDataBase* database, SSuperTable* stbInfo, char *command) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - return -1; - } -#endif - TAOS_RES * res; - TAOS_ROW row = NULL; - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - - res = taos_query(conn->taos, command); - int32_t code = taos_errno(res); - if (code != 0) { - printWarnCmdCodeStr(command, code, res); - infoPrint("stable %s does not exist, will create one\n", - stbInfo->stbName); - closeBenchConn(conn); - return -1; - } - infoPrint("find stable<%s>, will get meta data from server\n", - stbInfo->stbName); - benchArrayClear(stbInfo->tags); - benchArrayClear(stbInfo->cols); - int count = 0; - while ((row = taos_fetch_row(res)) != NULL) { - if (count == 0) { - count++; - continue; - } - int32_t *lengths = taos_fetch_lengths(res); - if (lengths == NULL) { - errorPrint("%s", "failed to execute taos_fetch_length\n"); - taos_free_result(res); - closeBenchConn(conn); - return -1; - } - if (strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "tag", - strlen("tag")) == 0) { - Field* tag = benchCalloc(1, sizeof(Field), true); - benchArrayPush(stbInfo->tags, tag); - tag = benchArrayGet(stbInfo->tags, stbInfo->tags->size - 1); - tag->type = convertStringToDatatype( - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); - tag->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tag->min = convertDatatypeToDefaultMin(tag->type); - tag->max = convertDatatypeToDefaultMax(tag->type); - tstrncpy(tag->name, - (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); - } else { - Field * col = benchCalloc(1, sizeof(Field), true); - benchArrayPush(stbInfo->cols, col); - col = benchArrayGet(stbInfo->cols, stbInfo->cols->size - 1); - col->type = convertStringToDatatype( - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); - col->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - col->min = convertDatatypeToDefaultMin(col->type); - col->max = convertDatatypeToDefaultMax(col->type); - tstrncpy(col->name, - (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); - } - } - taos_free_result(res); - closeBenchConn(conn); - return 0; -} - -static int getSuperTableFromServer(SDataBase* database, SSuperTable* stbInfo) { - int ret = 0; - - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "DESCRIBE `%s`.`%s`", database->dbName, - stbInfo->stbName); - - if (REST_IFACE == stbInfo->iface) { - ret = getSuperTableFromServerRest(database, stbInfo, command); - } else { - ret = getSuperTableFromServerTaosc(database, stbInfo, command); - } - - return ret; -} - -static int queryDbExec(SDataBase *database, - SSuperTable *stbInfo, char *command) { - int ret = 0; - if (REST_IFACE == stbInfo->iface) { - if (0 != convertServAddr(stbInfo->iface, false, 1)) { - errorPrint("%s", "Failed to convert server address\n"); - return -1; - } - int sockfd = createSockFd(); - if (sockfd < 0) { - ret = -1; - } else { - ret = queryDbExecRest(command, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - sockfd); - destroySockFd(sockfd); - } - } else { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - ret = -1; - } else { - ret = queryDbExecCall(conn, command); - int32_t trying = g_arguments->keep_trying; - while (ret && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then re-create " - "supertable %s\n", - g_arguments->trying_interval, stbInfo->stbName); - toolsMsleep(g_arguments->trying_interval); - ret = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - if (0 != ret) { - errorPrint("create supertable %s failed!\n\n", - stbInfo->stbName); - ret = -1; - } - closeBenchConn(conn); - } - } - - return ret; -} - -#ifdef WEBSOCKET -static void dropSuperTable(SDataBase* database, SSuperTable* stbInfo) { - if (g_arguments->supplementInsert) { - return; - } - - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - snprintf(command, sizeof(command), - g_arguments->escape_character - ? "DROP TABLE `%s`.`%s`" - : "DROP TABLE %s.%s", - database->dbName, - stbInfo->stbName); - - infoPrint("drop stable: <%s>\n", command); - queryDbExec(database, stbInfo, command); - - return; -} -#endif // WEBSOCKET - -static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { - if (g_arguments->supplementInsert) { - return 0; - } - - uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->cols->size; - char *colsBuf = benchCalloc(1, col_buffer_len, false); - char* command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - int len = 0; - - for (int colIndex = 0; colIndex < stbInfo->cols->size; colIndex++) { - Field * col = benchArrayGet(stbInfo->cols, colIndex); - int n; - if (col->type == TSDB_DATA_TYPE_BINARY || - col->type == TSDB_DATA_TYPE_NCHAR) { - n = snprintf(colsBuf + len, col_buffer_len - len, - ",%s %s(%d)", col->name, - convertDatatypeToString(col->type), col->length); - } else { - n = snprintf(colsBuf + len, col_buffer_len - len, - ",%s %s", col->name, - convertDatatypeToString(col->type)); - } - if (n < 0 || n >= col_buffer_len - len) { - errorPrint("%s() LN%d, snprintf overflow on %d\n", - __func__, __LINE__, colIndex); - break; - } else { - len += n; - } - } - - // save for creating child table - stbInfo->colsOfCreateChildTable = - (char *)benchCalloc(len + TIMESTAMP_BUFF_LEN, 1, true); - - snprintf(stbInfo->colsOfCreateChildTable, len + TIMESTAMP_BUFF_LEN, - "(ts timestamp%s)", colsBuf); - - if (stbInfo->tags->size == 0) { - free(colsBuf); - free(command); - return 0; - } - - uint32_t tag_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->tags->size; - char *tagsBuf = benchCalloc(1, tag_buffer_len, false); - int tagIndex; - len = 0; - - int n; - n = snprintf(tagsBuf + len, tag_buffer_len - len, "("); - if (n < 0 || n >= tag_buffer_len - len) { - errorPrint("%s() LN%d snprintf overflow\n", - __func__, __LINE__); - free(colsBuf); - free(command); - tmfree(tagsBuf); - return -1; - } else { - len += n; - } - for (tagIndex = 0; tagIndex < stbInfo->tags->size; tagIndex++) { - Field *tag = benchArrayGet(stbInfo->tags, tagIndex); - if (tag->type == TSDB_DATA_TYPE_BINARY || - tag->type == TSDB_DATA_TYPE_NCHAR) { - n = snprintf(tagsBuf + len, tag_buffer_len - len, - "%s %s(%d),", tag->name, - convertDatatypeToString(tag->type), tag->length); - } else if (tag->type == TSDB_DATA_TYPE_JSON) { - n = snprintf(tagsBuf + len, tag_buffer_len - len, - "%s json", tag->name); - if (n < 0 || n >= tag_buffer_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, tagIndex); - break; - } else { - len += n; - } - goto skip; - } else { - n = snprintf(tagsBuf + len, tag_buffer_len - len, - "%s %s,", tag->name, - convertDatatypeToString(tag->type)); - } - - if (n < 0 || n >= tag_buffer_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, tagIndex); - break; - } else { - len += n; - } - } - len -= 1; -skip: - snprintf(tagsBuf + len, tag_buffer_len - len, ")"); - - int length = snprintf( - command, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` (ts TIMESTAMP%s) TAGS %s" - : "CREATE TABLE %s.%s (ts TIMESTAMP%s) TAGS %s", - database->dbName, stbInfo->stbName, colsBuf, tagsBuf); - tmfree(colsBuf); - tmfree(tagsBuf); - if (stbInfo->comment != NULL) { - length += snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, - " COMMENT '%s'", stbInfo->comment); - } - if (stbInfo->delay >= 0) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, " DELAY %d", - stbInfo->delay); - } - if (stbInfo->file_factor >= 0) { - length += - snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, " FILE_FACTOR %f", - (float)stbInfo->file_factor / 100); - } - if (stbInfo->rollup != NULL) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " ROLLUP(%s)", stbInfo->rollup); - } - - if (stbInfo->max_delay != NULL) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " MAX_DELAY %s", stbInfo->max_delay); - } - - if (stbInfo->watermark != NULL) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " WATERMARK %s", stbInfo->watermark); - } - - if (stbInfo->ttl != 0) { - length += snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " TTL %d", stbInfo->ttl); - } - - bool first_sma = true; - for (int i = 0; i < stbInfo->cols->size; i++) { - Field * col = benchArrayGet(stbInfo->cols, i); - if (col->sma) { - if (first_sma) { - n = snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - " SMA(%s", col->name); - first_sma = false; - } else { - n = snprintf(command + length, - TSDB_MAX_ALLOWED_SQL_LEN - length, - ",%s", col->name); - } - - if (n < 0 || n > TSDB_MAX_ALLOWED_SQL_LEN - length) { - errorPrint("%s() LN%d snprintf overflow on %d iteral\n", - __func__, __LINE__, i); - break; - } else { - length += n; - } - } - } - if (!first_sma) { - snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, ")"); - } - infoPrint("create stable: <%s>\n", command); - - int ret = queryDbExec(database, stbInfo, command); - free(command); - return ret; -} - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 -int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { - int vgroups = 0; - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "USE `%s`" - : "USE %s", - database->dbName); - - int32_t code; - TAOS_RES *res = NULL; - - res = taos_query(conn->taos, cmd); - code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - taos_free_result(res); - - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SHOW VGROUPS"); - res = taos_query(conn->taos, cmd); - code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - - TAOS_ROW row = NULL; - while ((row = taos_fetch_row(res)) != NULL) { - vgroups++; - } - debugPrint("%s() LN%d, vgroups: %d\n", __func__, __LINE__, vgroups); - taos_free_result(res); - - database->vgroups = vgroups; - database->vgArray = benchArrayInit(vgroups, sizeof(SVGroup)); - for (int32_t v = 0; (v < vgroups - && !g_arguments->terminate); v++) { - SVGroup *vg = benchCalloc(1, sizeof(SVGroup), true); - benchArrayPush(database->vgArray, vg); - } - - res = taos_query(conn->taos, cmd); - code = taos_errno(res); - if (code) { - printErrCmdCodeStr(cmd, code, res); - return -1; - } - - int32_t vgItem = 0; - while (((row = taos_fetch_row(res)) != NULL) - && !g_arguments->terminate) { - SVGroup *vg = benchArrayGet(database->vgArray, vgItem); - vg->vgId = *(int32_t*)row[0]; - vgItem++; - } - taos_free_result(res); - - return vgroups; -} -#endif // TD_VER_COMPATIBLE_3_0_0_0 - -int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { - int dataLen = 0; - int n; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto || (-1 != g_arguments->inputted_vgroups)) { - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s` VGROUPS %d" - : "CREATE DATABASE IF NOT EXISTS %s VGROUPS %d", - database->dbName, - (-1 != g_arguments->inputted_vgroups)? - g_arguments->inputted_vgroups: - min(remainVnodes, toolsGetNumberOfCores())); - } else { - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s`" - : "CREATE DATABASE IF NOT EXISTS %s", - database->dbName); - } -#else - n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, - g_arguments->escape_character - ? "CREATE DATABASE IF NOT EXISTS `%s`" - : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); -#endif // TD_VER_COMPATIBLE_3_0_0_0 - if (n < 0 || n >= SHORT_1K_SQL_BUFF_LEN - dataLen) { - errorPrint("%s() LN%d snprintf overflow\n", - __func__, __LINE__); - return -1; - } else { - dataLen += n; - } - - if (database->cfgs) { - for (int i = 0; i < database->cfgs->size; i++) { - SDbCfg* cfg = benchArrayGet(database->cfgs, i); - if (cfg->valuestring) { - n = snprintf(command + dataLen, - TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " %s %s", cfg->name, cfg->valuestring); - } else { - n = snprintf(command + dataLen, - TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " %s %d", cfg->name, cfg->valueint); - } - if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - dataLen) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, i); - break; - } else { - dataLen += n; - } - } - } - - switch (database->precision) { - case TSDB_TIME_PRECISION_MILLI: - snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " PRECISION \'ms\';"); - break; - case TSDB_TIME_PRECISION_MICRO: - snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " PRECISION \'us\';"); - break; - case TSDB_TIME_PRECISION_NANO: - snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, - " PRECISION \'ns\';"); - break; - } - - return dataLen; -} - -int createDatabaseRest(SDataBase* database) { - int32_t code = 0; - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - - int sockfd = createSockFd(); - if (sockfd < 0) { - return -1; - } - - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "DROP DATABASE IF EXISTS `%s`;" - : "DROP DATABASE IF EXISTS %s;", - database->dbName); - code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - if (code != 0) { - errorPrint("Failed to drop database %s\n", database->dbName); - } else { - int remainVnodes = INT_MAX; - geneDbCreateCmd(database, command, remainVnodes); - code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - int32_t trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then " - "re-create database %s\n", - g_arguments->trying_interval, database->dbName); - toolsMsleep(g_arguments->trying_interval); - code = postProceSql(command, - database->dbName, - database->precision, - REST_IFACE, - 0, - g_arguments->port, - false, - sockfd, - NULL); - if (trying != -1) { - trying--; - } - } - } - destroySockFd(sockfd); - return code; -} - -int32_t getRemainVnodes(SBenchConn *conn) { - int remainVnodes = 0; - char command[SHORT_1K_SQL_BUFF_LEN] = "SHOW DNODES"; - - TAOS_RES *res = taos_query(conn->taos, command); - int32_t code = taos_errno(res); - if (code) { - printErrCmdCodeStr(command, code, res); - closeBenchConn(conn); - return -1; - } - TAOS_ROW row = NULL; - while ((row = taos_fetch_row(res)) != NULL) { - remainVnodes += (*(int16_t*)(row[3]) - *(int16_t*)(row[2])); - } - debugPrint("%s() LN%d, remainVnodes: %d\n", - __func__, __LINE__, remainVnodes); - taos_free_result(res); - return remainVnodes; -} - -int createDatabaseTaosc(SDataBase* database) { - char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - if (g_arguments->taosc_version == 3) { - for (int i = 0; i < g_arguments->streams->size; i++) { - SSTREAM* stream = benchArrayGet(g_arguments->streams, i); - if (stream->drop) { - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "DROP STREAM IF EXISTS %s;", - stream->stream_name); - if (queryDbExecCall(conn, command)) { - closeBenchConn(conn); - return -1; - } - infoPrint("%s\n", command); - memset(command, 0, SHORT_1K_SQL_BUFF_LEN); - } - } - } - - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "DROP DATABASE IF EXISTS `%s`;": - "DROP DATABASE IF EXISTS %s;", - database->dbName); - if (0 != queryDbExecCall(conn, command)) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - warnPrint("%s", "TDengine cloud normal users have no privilege " - "to drop database! DROP DATABASE failure is ignored!\n"); - } else { -#endif - closeBenchConn(conn); - return -1; -#ifdef WEBSOCKET - } -#endif - } - - int remainVnodes = INT_MAX; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - remainVnodes = getRemainVnodes(conn); - if (0 >= remainVnodes) { - errorPrint("Remain vnodes %d, failed to create database\n", - remainVnodes); - return -1; - } - } -#endif - geneDbCreateCmd(database, command, remainVnodes); - - int32_t code = queryDbExecCall(conn, command); - int32_t trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then " - "re-create database %s\n", - g_arguments->trying_interval, database->dbName); - toolsMsleep(g_arguments->trying_interval); - code = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - - if (code) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - warnPrint("%s", "TDengine cloud normal users have no privilege " - "to create database! CREATE DATABASE " - "failure is ignored!\n"); - } else { -#endif - - closeBenchConn(conn); - errorPrint("\ncreate database %s failed!\n\n", - database->dbName); - return -1; -#ifdef WEBSOCKET - } -#endif - } - infoPrint("command to create database: <%s>\n", command); - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->superTbls) { - if (g_arguments->nthreads_auto) { - int32_t vgroups = getVgroupsOfDb(conn, database); - if (vgroups <=0) { - closeBenchConn(conn); - errorPrint("Database %s's vgroups is %d\n", - database->dbName, vgroups); - return -1; - } - } - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 - - closeBenchConn(conn); - return 0; -} - -int createDatabase(SDataBase* database) { - int ret = 0; - if (REST_IFACE == g_arguments->iface) { - ret = createDatabaseRest(database); - } else { - ret = createDatabaseTaosc(database); - } -#if 0 -#ifdef LINUX - infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); - sleep(10); - infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); -#elif defined(DARWIN) - sleep(2); -#else - Sleep(2); -#endif -#endif - - return ret; -} - -static int generateChildTblName(int len, char *buffer, SDataBase *database, - SSuperTable *stbInfo, uint64_t i, - char *ttl) { - if (0 == len) { - memset(buffer, 0, TSDB_MAX_ALLOWED_SQL_LEN); - len += snprintf(buffer + len, - TSDB_MAX_ALLOWED_SQL_LEN - len, "CREATE TABLE "); - } - - len += snprintf( - buffer + len, TSDB_MAX_ALLOWED_SQL_LEN - len, - g_arguments->escape_character - ? "`%s`.`%s%" PRIu64 "` USING `%s`.`%s` TAGS (%s) %s " - : "%s.%s%" PRIu64 " USING %s.%s TAGS (%s) %s ", - database->dbName, stbInfo->childTblPrefix, i, database->dbName, - stbInfo->stbName, - stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); - - return len; -} - -static int getBatchOfTblCreating(threadInfo *pThreadInfo, - SSuperTable *stbInfo) { - BArray *batchArray = stbInfo->batchTblCreatingNumbersArray; - if (batchArray) { - int *batch = benchArrayGet( - batchArray, pThreadInfo->posOfTblCreatingBatch); - pThreadInfo->posOfTblCreatingBatch++; - if (pThreadInfo->posOfTblCreatingBatch == batchArray->size) { - pThreadInfo->posOfTblCreatingBatch = 0; - } - return *batch; - } - return 0; -} - -static int getIntervalOfTblCreating(threadInfo *pThreadInfo, - SSuperTable *stbInfo) { - BArray *intervalArray = stbInfo->batchTblCreatingIntervalsArray; - if (intervalArray) { - int *interval = benchArrayGet( - intervalArray, pThreadInfo->posOfTblCreatingInterval); - pThreadInfo->posOfTblCreatingInterval++; - if (pThreadInfo->posOfTblCreatingInterval == intervalArray->size) { - pThreadInfo->posOfTblCreatingInterval = 0; - } - return *interval; - } - return 0; -} - -static void *createTable(void *sarg) { - if (g_arguments->supplementInsert) { - return NULL; - } - - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; -#ifdef LINUX - prctl(PR_SET_NAME, "createTable"); -#endif - uint64_t lastPrintTime = toolsGetTimestampMs(); - pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - int len = 0; - int batchNum = 0; - infoPrint( - "thread[%d] start creating table from %" PRIu64 " to %" PRIu64 - "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to); - - char ttl[SMALL_BUFF_LEN] = ""; - if (stbInfo->ttl != 0) { - snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); - } - - int smallBatchCount = 0; - for (uint64_t i = pThreadInfo->start_table_from + stbInfo->childTblFrom; - (i <= (pThreadInfo->end_table_to + stbInfo->childTblFrom) - && !g_arguments->terminate); i++) { - if (g_arguments->terminate) { - goto create_table_end; - } - if (!stbInfo->use_metric || stbInfo->tags->size == 0) { - if (stbInfo->childTblCount == 1) { - snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` %s;" - : "CREATE TABLE %s.%s %s;", - database->dbName, stbInfo->stbName, - stbInfo->colsOfCreateChildTable); - } else { - snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "CREATE TABLE `%s`.`%s` %s;" - : "CREATE TABLE %s.%s %s;", - database->dbName, - stbInfo->childTblArray[i]->name, - stbInfo->colsOfCreateChildTable); - } - batchNum++; - } else { - if (0 == len) { - batchNum = 0; - } - len = generateChildTblName(len, pThreadInfo->buffer, - database, stbInfo, i, ttl); - - batchNum++; - smallBatchCount++; - - int smallBatch = getBatchOfTblCreating(pThreadInfo, stbInfo); - if ((!smallBatch || (smallBatchCount == smallBatch)) - && (batchNum < stbInfo->batchTblCreatingNum) - && ((TSDB_MAX_ALLOWED_SQL_LEN - len) >= - (stbInfo->lenOfTags + EXTRA_SQL_LEN))) { - continue; - } else { - smallBatchCount = 0; - } - } - - len = 0; - - int ret = 0; - debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, - pThreadInfo->buffer); - if (REST_IFACE == stbInfo->iface) { - ret = queryDbExecRest(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - pThreadInfo->sockfd); - } else { - ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - int32_t trying = g_arguments->keep_trying; - while (ret && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then re-create " - "table %s\n", - g_arguments->trying_interval, pThreadInfo->buffer); - toolsMsleep(g_arguments->trying_interval); - ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - if (trying != -1) { - trying--; - } - } - } - - if (0 != ret) { - g_fail = true; - goto create_table_end; - } - uint64_t intervalOfTblCreating = getIntervalOfTblCreating(pThreadInfo, - stbInfo); - if (intervalOfTblCreating) { - debugPrint("will sleep %"PRIu64" milliseconds " - "for table creating interval\n", intervalOfTblCreating); - toolsMsleep(intervalOfTblCreating); - } - - pThreadInfo->tables_created += batchNum; - batchNum = 0; - uint64_t currentPrintTime = toolsGetTimestampMs(); - if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { - infoPrint( - "thread[%d] already created %" PRId64 " tables\n", - pThreadInfo->threadID, pThreadInfo->tables_created); - lastPrintTime = currentPrintTime; - } - } - - if (0 != len) { - int ret = 0; - debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, - pThreadInfo->buffer); - if (REST_IFACE == stbInfo->iface) { - ret = queryDbExecRest(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - pThreadInfo->sockfd); - } else { - ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - } - if (0 != ret) { - g_fail = true; - goto create_table_end; - } - pThreadInfo->tables_created += batchNum; - debugPrint("thread[%d] already created %" PRId64 " tables\n", - pThreadInfo->threadID, pThreadInfo->tables_created); - } -create_table_end: - tmfree(pThreadInfo->buffer); - pThreadInfo->buffer = NULL; - return NULL; -} - -static int startMultiThreadCreateChildTable( - SDataBase* database, SSuperTable* stbInfo) { - int code = -1; - int threads = g_arguments->table_threads; - int64_t ntables; - if (stbInfo->childTblTo > 0) { - ntables = stbInfo->childTblTo - stbInfo->childTblFrom; - } else { - ntables = stbInfo->childTblCount; - } - pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), false); - threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), false); - uint64_t tableFrom = 0; - if (threads < 1) { - threads = 1; - } - - int64_t a = ntables / threads; - if (a < 1) { - threads = (int)ntables; - a = 1; - } - - if (ntables == 0) { - errorPrint("failed to create child table, childTblCount: %"PRId64"\n", - ntables); - goto over; - } - int64_t b = ntables % threads; - - for (uint32_t i = 0; (i < threads && !g_arguments->terminate); i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - pThreadInfo->stbInfo = stbInfo; - pThreadInfo->dbInfo = database; - if (REST_IFACE == stbInfo->iface) { - int sockfd = createSockFd(); - if (sockfd < 0) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); - } - pThreadInfo->sockfd = sockfd; - } else { - pThreadInfo->conn = initBenchConn(); - if (NULL == pThreadInfo->conn) { - goto over; - } - } - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pThreadInfo->end_table_to + 1; - pThreadInfo->tables_created = 0; - pthread_create(pids + i, NULL, createTable, pThreadInfo); - } - - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - pthread_join(pids[i], NULL); - } - - if (g_arguments->terminate) toolsMsleep(100); - - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - g_arguments->actualChildTables += pThreadInfo->tables_created; - - if ((REST_IFACE != stbInfo->iface) && pThreadInfo->conn) { - closeBenchConn(pThreadInfo->conn); - } - } - - if (g_fail) { - goto over; - } - code = 0; -over: - free(pids); - free(infos); - return code; -} - -static int createChildTables() { - int32_t code; - infoPrint("start creating %" PRId64 " table(s) with %d thread(s)\n", - g_arguments->totalChildTables, g_arguments->table_threads); - if (g_arguments->fpOfInsertResult) { - infoPrintToFile(g_arguments->fpOfInsertResult, - "start creating %" PRId64 " table(s) with %d thread(s)\n", - g_arguments->totalChildTables, g_arguments->table_threads); - } - double start = (double)toolsGetTimestampMs(); - - for (int i = 0; (i < g_arguments->databases->size - && !g_arguments->terminate); i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (int j = 0; (j < database->superTbls->size - && !g_arguments->terminate); j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - if (stbInfo->autoTblCreating || stbInfo->iface == SML_IFACE - || stbInfo->iface == SML_REST_IFACE) { - g_arguments->autoCreatedChildTables += - stbInfo->childTblCount; - continue; - } - if (stbInfo->childTblExists) { - g_arguments->existedChildTables += - stbInfo->childTblCount; - continue; - } - debugPrint("colsOfCreateChildTable: %s\n", - stbInfo->colsOfCreateChildTable); - - code = startMultiThreadCreateChildTable(database, stbInfo); - if (code && !g_arguments->terminate) { - return code; - } - } - } - } - - double end = (double)toolsGetTimestampMs(); - succPrint( - "Spent %.4f seconds to create %" PRId64 - " table(s) with %d thread(s), already exist %" PRId64 - " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 - " table(s) will be auto created\n", - (end - start) / 1000.0, g_arguments->totalChildTables, - g_arguments->table_threads, g_arguments->existedChildTables, - g_arguments->actualChildTables, - g_arguments->autoCreatedChildTables); - return 0; -} - -static void freeChildTable(SChildTable *childTbl, int colsSize) { - if (childTbl->useOwnSample) { - if (childTbl->childCols) { - for (int col = 0; col < colsSize; col++) { - ChildField *childCol = - benchArrayGet(childTbl->childCols, col); - if (childCol) { - tmfree(childCol->stmtData.data); - tmfree(childCol->stmtData.is_null); - } - } - benchArrayDestroy(childTbl->childCols); - } - tmfree(childTbl->sampleDataBuf); - } - tmfree(childTbl); -} - -void postFreeResource() { - if (!g_arguments->terminate) { - tmfclose(g_arguments->fpOfInsertResult); - } - - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->cfgs) { - for (int c = 0; c < database->cfgs->size; c++) { - SDbCfg *cfg = benchArrayGet(database->cfgs, c); - if ((NULL == root) && (0 == strcmp(cfg->name, "replica"))) { - tmfree(cfg->name); - cfg->name = NULL; - } - } - benchArrayDestroy(database->cfgs); - } - if (database->superTbls) { - for (uint64_t j = 0; j < database->superTbls->size; j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - tmfree(stbInfo->colsOfCreateChildTable); - stbInfo->colsOfCreateChildTable = NULL; - tmfree(stbInfo->sampleDataBuf); - stbInfo->sampleDataBuf = NULL; - tmfree(stbInfo->tagDataBuf); - stbInfo->tagDataBuf = NULL; - tmfree(stbInfo->partialColNameBuf); - stbInfo->partialColNameBuf = NULL; - benchArrayDestroy(stbInfo->batchTblCreatingNumbersArray); - benchArrayDestroy(stbInfo->batchTblCreatingIntervalsArray); - for (int k = 0; k < stbInfo->tags->size; k++) { - Field * tag = benchArrayGet(stbInfo->tags, k); - tmfree(tag->stmtData.data); - tag->stmtData.data = NULL; - } - benchArrayDestroy(stbInfo->tags); - - for (int k = 0; k < stbInfo->cols->size; k++) { - Field * col = benchArrayGet(stbInfo->cols, k); - tmfree(col->stmtData.data); - col->stmtData.data = NULL; - tmfree(col->stmtData.is_null); - col->stmtData.is_null = NULL; - } - if (g_arguments->test_mode == INSERT_TEST) { - if (stbInfo->childTblArray) { - for (int64_t child = 0; child < stbInfo->childTblCount; - child++) { - SChildTable *childTbl = - stbInfo->childTblArray[child]; - if (childTbl) { - freeChildTable(childTbl, stbInfo->cols->size); - } - } - } - } - benchArrayDestroy(stbInfo->cols); - tmfree(stbInfo->childTblArray); - stbInfo->childTblArray = NULL; - benchArrayDestroy(stbInfo->tsmas); -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - tmfree(vg->childTblArray); - vg->childTblArray = NULL; - } - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 - } -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (database->vgArray) - benchArrayDestroy(database->vgArray); -#endif // TD_VER_COMPATIBLE_3_0_0_0 - benchArrayDestroy(database->superTbls); - } - } - benchArrayDestroy(g_arguments->databases); - benchArrayDestroy(g_arguments->streams); - tools_cJSON_Delete(root); -} - -int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - TAOS_RES * res = NULL; - int32_t code = 0; - uint16_t iface = stbInfo->iface; - - int32_t trying = (stbInfo->keep_trying)? - stbInfo->keep_trying:g_arguments->keep_trying; - int32_t trying_interval = stbInfo->trying_interval? - stbInfo->trying_interval:g_arguments->trying_interval; - int protocol = stbInfo->lineProtocol; - - switch (iface) { - case TAOSC_IFACE: - debugPrint("buffer: %s\n", pThreadInfo->buffer); - code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - while (code && trying && !g_arguments->terminate) { - infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", - trying_interval); - toolsMsleep(trying_interval); - code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); - if (trying != -1) { - trying--; - } - } - break; - - case REST_IFACE: - debugPrint("buffer: %s\n", pThreadInfo->buffer); - code = postProceSql(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, - pThreadInfo->filePath); - while (code && trying && !g_arguments->terminate) { - infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", - trying_interval); - toolsMsleep(trying_interval); - code = postProceSql(pThreadInfo->buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, - pThreadInfo->filePath); - if (trying != -1) { - trying--; - } - } - break; - - case STMT_IFACE: - code = taos_stmt_execute(pThreadInfo->conn->stmt); - if (code) { - errorPrint( - "failed to execute insert statement. reason: %s\n", - taos_stmt_errstr(pThreadInfo->conn->stmt)); - code = -1; - } - break; - - case SML_IFACE: - res = taos_schemaless_insert( - pThreadInfo->conn->taos, pThreadInfo->lines, - (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) - ? 0 : k, - (SML_JSON_TAOS_FORMAT == protocol) - ? TSDB_SML_JSON_PROTOCOL : protocol, - (TSDB_SML_LINE_PROTOCOL == protocol) - ? database->sml_precision - : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); - code = taos_errno(res); - trying = stbInfo->keep_trying; - while (code && trying && !g_arguments->terminate) { - taos_free_result(res); - infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", - trying_interval); - toolsMsleep(trying_interval); - res = taos_schemaless_insert( - pThreadInfo->conn->taos, pThreadInfo->lines, - (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) - ? 0 : k, - (SML_JSON_TAOS_FORMAT == protocol) - ? TSDB_SML_JSON_PROTOCOL : protocol, - (TSDB_SML_LINE_PROTOCOL == protocol) - ? database->sml_precision - : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); - code = taos_errno(res); - if (trying != -1) { - trying--; - } - } - - if (code != TSDB_CODE_SUCCESS && !g_arguments->terminate) { - debugPrint("Failed to execute " - "schemaless insert content: %s\n\n", - pThreadInfo->lines?(pThreadInfo->lines[0]? - pThreadInfo->lines[0]:""):""); - errorPrint( - "failed to execute schemaless insert. " - "code: 0x%08x reason: %s\n\n", - code, taos_errstr(res)); - } - taos_free_result(res); - break; - - case SML_REST_IFACE: { - if (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) { - code = postProceSql(pThreadInfo->lines[0], database->dbName, - database->precision, stbInfo->iface, - protocol, g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, pThreadInfo->filePath); - } else { - int len = 0; - for (int i = 0; i < k; i++) { - if (strlen(pThreadInfo->lines[i]) != 0) { - int n; - if (TSDB_SML_TELNET_PROTOCOL == protocol - && stbInfo->tcpTransfer) { - n = snprintf(pThreadInfo->buffer + len, - TSDB_MAX_ALLOWED_SQL_LEN - len, - "put %s\n", pThreadInfo->lines[i]); - } else { - n = snprintf(pThreadInfo->buffer + len, - TSDB_MAX_ALLOWED_SQL_LEN - len, - "%s\n", - pThreadInfo->lines[i]); - } - if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, i); - break; - } else { - len += n; - } - } else { - break; - } - } - if (g_arguments->terminate) { - break; - } - code = postProceSql(pThreadInfo->buffer, database->dbName, - database->precision, - stbInfo->iface, protocol, - g_arguments->port, - stbInfo->tcpTransfer, - pThreadInfo->sockfd, pThreadInfo->filePath); - } - break; - } - } - return code; -} - -static int smartContinueIfFail(threadInfo *pThreadInfo, - SChildTable *childTbl, - int64_t i, - char *ttl) { - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - char *buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - snprintf( - buffer, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character ? - "CREATE TABLE `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s " - : "CREATE TABLE %s.%s USING %s.%s TAGS (%s) %s ", - database->dbName, childTbl->name, database->dbName, - stbInfo->stbName, - stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); - debugPrint("creating table: %s\n", buffer); - int ret; - if (REST_IFACE == stbInfo->iface) { - ret = queryDbExecRest(buffer, - database->dbName, - database->precision, - stbInfo->iface, - stbInfo->lineProtocol, - stbInfo->tcpTransfer, - pThreadInfo->sockfd); - } else { - ret = queryDbExecCall(pThreadInfo->conn, buffer); - int32_t trying = g_arguments->keep_trying; - while (ret && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then " - "re-create table %s\n", - g_arguments->trying_interval, buffer); - toolsMsleep(g_arguments->trying_interval); - ret = queryDbExecCall(pThreadInfo->conn, buffer); - if (trying != -1) { - trying--; - } - } - } - tmfree(buffer); - - return ret; -} - -static void cleanupAndPrint(threadInfo *pThreadInfo, char *mode) { - if (pThreadInfo) { - if (pThreadInfo->json_array) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - if (0 == pThreadInfo->totalDelay) { - pThreadInfo->totalDelay = 1; - } - succPrint( - "thread[%d] %s mode, completed total inserted rows: %" PRIu64 - ", %.2f records/second\n", - pThreadInfo->threadID, - mode, - pThreadInfo->totalInsertRows, - (double)(pThreadInfo->totalInsertRows / - ((double)pThreadInfo->totalDelay / 1E6))); - } -} - -static int64_t getDisorderTs(SSuperTable *stbInfo, int *disorderRange) { - int64_t disorderTs = 0; - int64_t startTimestamp = stbInfo->startTimestamp; - if (stbInfo->disorderRatio > 0) { - int rand_num = taosRandom() % 100; - if (rand_num < stbInfo->disorderRatio) { - (*disorderRange)--; - if (0 == *disorderRange) { - *disorderRange = stbInfo->disorderRange; - } - disorderTs = startTimestamp - *disorderRange; - debugPrint("rand_num: %d, < disorderRatio: %d, " - "disorderTs: %"PRId64"\n", - rand_num, stbInfo->disorderRatio, - disorderTs); - } - } - return disorderTs; -} - -static void *syncWriteInterlace(void *sarg) { - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - infoPrint( - "thread[%d] start interlace inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to); - - int64_t insertRows = stbInfo->insertRows; - int32_t interlaceRows = stbInfo->interlaceRows; - int64_t pos = 0; - uint32_t batchPerTblTimes = g_arguments->reqPerReq / interlaceRows; - uint64_t lastPrintTime = toolsGetTimestampMs(); - uint64_t lastTotalInsertRows = 0; - int64_t startTs = toolsGetTimestampUs(); - int64_t endTs; - uint64_t tableSeq = pThreadInfo->start_table_from; - int disorderRange = stbInfo->disorderRange; - - while (insertRows > 0) { - int64_t tmp_total_insert_rows = 0; - uint32_t generated = 0; - if (insertRows <= interlaceRows) { - interlaceRows = insertRows; - } - for (int i = 0; i < batchPerTblTimes; i++) { - if (g_arguments->terminate) { - goto free_of_interlace; - } - int64_t timestamp = pThreadInfo->start_time; - SChildTable *childTbl = stbInfo->childTblArray[tableSeq]; - char * tableName = - stbInfo->childTblArray[tableSeq]->name; - char *sampleDataBuf = childTbl->useOwnSample? - childTbl->sampleDataBuf: - stbInfo->sampleDataBuf; - char ttl[SMALL_BUFF_LEN] = ""; - if (stbInfo->ttl != 0) { - snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); - } - switch (stbInfo->iface) { - case REST_IFACE: - case TAOSC_IFACE: { - char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; - if (g_arguments->escape_character) { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "`%s`", - tableName); - } else { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "%s", - tableName); - } - if (i == 0) { - ds_add_str(&pThreadInfo->buffer, STR_INSERT_INTO); - } - if (stbInfo->partialColNum == stbInfo->cols->size) { - if (stbInfo->autoTblCreating) { - ds_add_strs(&pThreadInfo->buffer, 8, - escapedTbName, - " USING `", - stbInfo->stbName, - "` TAGS (", - stbInfo->tagDataBuf - + stbInfo->lenOfTags * tableSeq, - ") ", ttl, " VALUES "); - } else { - ds_add_strs(&pThreadInfo->buffer, 2, - escapedTbName, " VALUES "); - } - } else { - if (stbInfo->autoTblCreating) { - ds_add_strs(&pThreadInfo->buffer, 10, - escapedTbName, - " (", - stbInfo->partialColNameBuf, - ") USING `", - stbInfo->stbName, - "` TAGS (", - stbInfo->tagDataBuf - + stbInfo->lenOfTags * tableSeq, - ") ", ttl, " VALUES "); - } else { - ds_add_strs(&pThreadInfo->buffer, 4, - escapedTbName, - "(", - stbInfo->partialColNameBuf, - ") VALUES "); - } - } - - for (int64_t j = 0; j < interlaceRows; j++) { - int64_t disorderTs = getDisorderTs(stbInfo, - &disorderRange); - char time_string[BIGINT_BUFF_LEN]; - snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", - disorderTs?disorderTs:timestamp); - ds_add_strs(&pThreadInfo->buffer, 5, - "(", - time_string, - ",", - sampleDataBuf + pos * stbInfo->lenOfCols, - ") "); - if (ds_len(pThreadInfo->buffer) - > stbInfo->max_sql_len) { - errorPrint("sql buffer length (%"PRIu64") " - "is larger than max sql length " - "(%"PRId64")\n", - ds_len(pThreadInfo->buffer), - stbInfo->max_sql_len); - goto free_of_interlace; - } - generated++; - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - timestamp += stbInfo->timestamp_step; - } - break; - } - case STMT_IFACE: { - char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; - if (g_arguments->escape_character) { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, - "`%s`", tableName); - } else { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", - tableName); - } - if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, - escapedTbName)) { - errorPrint( - "taos_stmt_set_tbname(%s) failed, reason: %s\n", - tableName, - taos_stmt_errstr(pThreadInfo->conn->stmt)); - g_fail = true; - goto free_of_interlace; - } - generated = - bindParamBatch(pThreadInfo, interlaceRows, - timestamp, childTbl); - break; - } - case SML_REST_IFACE: - case SML_IFACE: { - int protocol = stbInfo->lineProtocol; - for (int64_t j = 0; j < interlaceRows; j++) { - int64_t disorderTs = getDisorderTs(stbInfo, - &disorderRange); - if (TSDB_SML_JSON_PROTOCOL == protocol) { - tools_cJSON *tag = tools_cJSON_Duplicate( - tools_cJSON_GetArrayItem( - pThreadInfo->sml_json_tags, - (int)tableSeq - - pThreadInfo->start_table_from), - true); - generateSmlJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, - disorderTs?disorderTs:timestamp); - } else if (SML_JSON_TAOS_FORMAT == protocol) { - tools_cJSON *tag = tools_cJSON_Duplicate( - tools_cJSON_GetArrayItem( - pThreadInfo->sml_json_tags, - (int)tableSeq - - pThreadInfo->start_table_from), - true); - generateSmlTaosJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, - disorderTs?disorderTs:timestamp); - } else if (TSDB_SML_LINE_PROTOCOL == protocol) { - snprintf( - pThreadInfo->lines[generated], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %s %" PRId64 "", - pThreadInfo - ->sml_tags[(int)tableSeq - - pThreadInfo->start_table_from], - sampleDataBuf + pos * stbInfo->lenOfCols, - disorderTs?disorderTs:timestamp); - } else { - snprintf( - pThreadInfo->lines[generated], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %" PRId64 " %s %s", stbInfo->stbName, - disorderTs?disorderTs:timestamp, - sampleDataBuf + pos * stbInfo->lenOfCols, - pThreadInfo - ->sml_tags[(int)tableSeq - - pThreadInfo->start_table_from]); - } - generated++; - timestamp += stbInfo->timestamp_step; - } - if (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) { - pThreadInfo->lines[0] = - tools_cJSON_PrintUnformatted( - pThreadInfo->json_array); - } - break; - } - } - tableSeq++; - tmp_total_insert_rows += interlaceRows; - if (tableSeq > pThreadInfo->end_table_to) { - tableSeq = pThreadInfo->start_table_from; - pThreadInfo->start_time += - interlaceRows * stbInfo->timestamp_step; - if (!stbInfo->non_stop) { - insertRows -= interlaceRows; - } - if (stbInfo->insert_interval > 0) { - debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", - __func__, __LINE__, stbInfo->insert_interval); - perfPrint("sleep %" PRIu64 " ms\n", - stbInfo->insert_interval); - toolsMsleep((int32_t)stbInfo->insert_interval); - } - break; - } - } - - startTs = toolsGetTimestampUs(); - if (execInsert(pThreadInfo, generated)) { - g_fail = true; - goto free_of_interlace; - } - endTs = toolsGetTimestampUs(); - - pThreadInfo->totalInsertRows += tmp_total_insert_rows; - - if (g_arguments->terminate) { - goto free_of_interlace; - } - - int protocol = stbInfo->lineProtocol; - switch (stbInfo->iface) { - case TAOSC_IFACE: - case REST_IFACE: - debugPrint("pThreadInfo->buffer: %s\n", - pThreadInfo->buffer); - free_ds(&pThreadInfo->buffer); - pThreadInfo->buffer = new_ds(0); - break; - case SML_REST_IFACE: - memset(pThreadInfo->buffer, 0, - g_arguments->reqPerReq * (pThreadInfo->max_sql_len + 1)); - case SML_IFACE: - if (TSDB_SML_JSON_PROTOCOL == protocol - || SML_JSON_TAOS_FORMAT == protocol) { - debugPrint("pThreadInfo->lines[0]: %s\n", - pThreadInfo->lines[0]); - if (pThreadInfo->json_array && !g_arguments->terminate) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - pThreadInfo->json_array = tools_cJSON_CreateArray(); - if (pThreadInfo->lines && pThreadInfo->lines[0]) { - tmfree(pThreadInfo->lines[0]); - pThreadInfo->lines[0] = NULL; - } - } else { - for (int j = 0; j < generated; j++) { - if (pThreadInfo && pThreadInfo->lines - && !g_arguments->terminate) { - debugPrint("pThreadInfo->lines[%d]: %s\n", j, - pThreadInfo->lines[j]); - memset(pThreadInfo->lines[j], 0, - pThreadInfo->max_sql_len); - } - } - } - break; - case STMT_IFACE: - break; - } - - int64_t delay = endTs - startTs; - if (delay <=0) { - debugPrint("thread[%d]: startTS: %"PRId64", endTS: %"PRId64"\n", - pThreadInfo->threadID, startTs, endTs); - } else { - perfPrint("insert execution time is %10.2f ms\n", - delay / 1E6); - - int64_t * pdelay = benchCalloc(1, sizeof(int64_t), false); - *pdelay = delay; - if (benchArrayPush(pThreadInfo->delayList, pdelay) == NULL) { - tmfree(pdelay); - } - pThreadInfo->totalDelay += delay; - } - - int64_t currentPrintTime = toolsGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30 * 1000) { - infoPrint( - "thread[%d] has currently inserted rows: %" PRIu64 - ", peroid insert rate: %.3f rows/s \n", - pThreadInfo->threadID, pThreadInfo->totalInsertRows, - (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); - lastPrintTime = currentPrintTime; - lastTotalInsertRows = pThreadInfo->totalInsertRows; - } - } -free_of_interlace: - cleanupAndPrint(pThreadInfo, "interlace"); - return NULL; -} - -static int32_t prepareProgressDataStmt( - threadInfo *pThreadInfo, - SChildTable *childTbl, - int64_t *timestamp, uint64_t i, char *ttl) { - SSuperTable *stbInfo = pThreadInfo->stbInfo; - char escapedTbName[TSDB_TABLE_NAME_LEN + 2] = "\0"; - if (g_arguments->escape_character) { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN + 2, - "`%s`", childTbl->name); - } else { - snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", - childTbl->name); - } - if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, - escapedTbName)) { - errorPrint( - "taos_stmt_set_tbname(%s) failed," - "reason: %s\n", escapedTbName, - taos_stmt_errstr(pThreadInfo->conn->stmt)); - return -1; - } - int32_t generated = bindParamBatch( - pThreadInfo, - (g_arguments->reqPerReq > (stbInfo->insertRows - i)) - ? (stbInfo->insertRows - i) - : g_arguments->reqPerReq, - *timestamp, childTbl); - *timestamp += generated * stbInfo->timestamp_step; - return generated; -} - -static void makeTimestampDisorder( - int64_t *timestamp, SSuperTable *stbInfo) { - int64_t startTimestamp = stbInfo->startTimestamp; - int disorderRange = stbInfo->disorderRange; - int rand_num = taosRandom() % 100; - if (rand_num < stbInfo->disorderRatio) { - disorderRange--; - if (0 == disorderRange) { - disorderRange = stbInfo->disorderRange; - } - *timestamp = startTimestamp - disorderRange; - debugPrint("rand_num: %d, < disorderRatio: %d" - ", ts: %"PRId64"\n", - rand_num, - stbInfo->disorderRatio, - *timestamp); - } -} - -static int32_t prepareProgressDataSmlJsonText( - threadInfo *pThreadInfo, - uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { - // prepareProgressDataSmlJsonText - SSuperTable *stbInfo = pThreadInfo->stbInfo; - int32_t generated = 0; - - int len = 0; - - char *line = pThreadInfo->lines[0]; - uint32_t line_buf_len = pThreadInfo->line_buf_len; - - strncat(line + len, "[", 2); - len += 1; - - int32_t pos = 0; - for (int j = 0; (j < g_arguments->reqPerReq) - && !g_arguments->terminate; j++) { - strncat(line + len, "{", 2); - len += 1; - int n; - n = snprintf(line + len, line_buf_len - len, - "\"timestamp\":%"PRId64",", *timestamp); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - - n = snprintf(line + len, line_buf_len - len, "%s", - pThreadInfo->sml_json_value_array[tableSeq]); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - n = snprintf(line + len, line_buf_len - len, "\"tags\":%s,", - pThreadInfo->sml_tags_json_array[tableSeq]); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - n = snprintf(line + len, line_buf_len - len, - "\"metric\":\"%s\"}", stbInfo->stbName); - if (n < 0 || n >= line_buf_len - len) { - errorPrint("%s() LN%d snprintf overflow on %d\n", - __func__, __LINE__, j); - return -1; - } else { - len += n; - } - - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - *timestamp += stbInfo->timestamp_step; - if (stbInfo->disorderRatio > 0) { - makeTimestampDisorder(timestamp, stbInfo); - } - generated++; - if (i + generated >= stbInfo->insertRows) { - break; - } - if ((j+1) < g_arguments->reqPerReq) { - strncat(line + len, ",", 2); - len += 1; - } - } - strncat(line + len, "]", 2); - - debugPrint("%s() LN%d, lines[0]: %s\n", - __func__, __LINE__, pThreadInfo->lines[0]); - return generated; -} - -static int32_t prepareProgressDataSmlJson( - threadInfo *pThreadInfo, - uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { - // prepareProgressDataSmlJson - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - int32_t generated = 0; - - int32_t pos = 0; - int protocol = stbInfo->lineProtocol; - for (int j = 0; (j < g_arguments->reqPerReq) - && !g_arguments->terminate; j++) { - tools_cJSON *tag = tools_cJSON_Duplicate( - tools_cJSON_GetArrayItem( - pThreadInfo->sml_json_tags, - (int)tableSeq - - pThreadInfo->start_table_from), - true); - debugPrintJsonNoTime(tag); - if (TSDB_SML_JSON_PROTOCOL == protocol) { - generateSmlJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, *timestamp); - } else { - generateSmlTaosJsonCols( - pThreadInfo->json_array, tag, stbInfo, - database->sml_precision, *timestamp); - } - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - *timestamp += stbInfo->timestamp_step; - if (stbInfo->disorderRatio > 0) { - makeTimestampDisorder(timestamp, stbInfo); - } - generated++; - if (i + generated >= stbInfo->insertRows) { - break; - } - } - - tmfree(pThreadInfo->lines[0]); - pThreadInfo->lines[0] = NULL; - pThreadInfo->lines[0] = - tools_cJSON_PrintUnformatted( - pThreadInfo->json_array); - debugPrint("pThreadInfo->lines[0]: %s\n", - pThreadInfo->lines[0]); - - return generated; -} - -static int32_t prepareProgressDataSmlLineOrTelnet( - threadInfo *pThreadInfo, uint64_t tableSeq, char *sampleDataBuf, - int64_t *timestamp, uint64_t i, char *ttl, int protocol) { - // prepareProgressDataSmlLine - SSuperTable *stbInfo = pThreadInfo->stbInfo; - int32_t generated = 0; - - int32_t pos = 0; - for (int j = 0; (j < g_arguments->reqPerReq) - && !g_arguments->terminate; j++) { - if (TSDB_SML_LINE_PROTOCOL == protocol) { - snprintf( - pThreadInfo->lines[j], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %s %" PRId64 "", - pThreadInfo->sml_tags[tableSeq - - pThreadInfo->start_table_from], - sampleDataBuf + pos * stbInfo->lenOfCols, - *timestamp); - } else { - snprintf( - pThreadInfo->lines[j], - stbInfo->lenOfCols + stbInfo->lenOfTags, - "%s %" PRId64 " %s %s", stbInfo->stbName, - *timestamp, - sampleDataBuf - + pos * stbInfo->lenOfCols, - pThreadInfo->sml_tags[tableSeq - -pThreadInfo->start_table_from]); - } - pos++; - if (pos >= g_arguments->prepared_rand) { - pos = 0; - } - *timestamp += stbInfo->timestamp_step; - if (stbInfo->disorderRatio > 0) { - makeTimestampDisorder(timestamp, stbInfo); - } - generated++; - if (i + generated >= stbInfo->insertRows) { - break; - } - } - return generated; -} - -static int32_t prepareProgressDataSml( - threadInfo *pThreadInfo, - SChildTable *childTbl, - uint64_t tableSeq, - int64_t *timestamp, uint64_t i, char *ttl) { - // prepareProgressDataSml - SSuperTable *stbInfo = pThreadInfo->stbInfo; - - char *sampleDataBuf; - if (childTbl->useOwnSample) { - sampleDataBuf = childTbl->sampleDataBuf; - } else { - sampleDataBuf = stbInfo->sampleDataBuf; - } - int protocol = stbInfo->lineProtocol; - int32_t generated = -1; - switch (protocol) { - case TSDB_SML_LINE_PROTOCOL: - case TSDB_SML_TELNET_PROTOCOL: - generated = prepareProgressDataSmlLineOrTelnet( - pThreadInfo, - tableSeq, - sampleDataBuf, - timestamp, i, ttl, protocol); - break; - case TSDB_SML_JSON_PROTOCOL: - generated = prepareProgressDataSmlJsonText( - pThreadInfo, - tableSeq - pThreadInfo->start_table_from, - timestamp, i, ttl); - break; - case SML_JSON_TAOS_FORMAT: - generated = prepareProgressDataSmlJson( - pThreadInfo, - tableSeq, - timestamp, i, ttl); - break; - default: - errorPrint("%s() LN%d: unknown protcolor: %d\n", - __func__, __LINE__, protocol); - break; - } - - return generated; -} - -static int32_t prepareProgressDataSql( - threadInfo *pThreadInfo, - SChildTable *childTbl, uint64_t tableSeq, - char *sampleDataBuf, - int64_t *timestamp, uint64_t i, char *ttl, - int32_t *pos, uint64_t *len) { - // prepareProgressDataSql - int32_t generated = 0; - SDataBase *database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - char * pstr = pThreadInfo->buffer; - int disorderRange = stbInfo->disorderRange; - - if (stbInfo->partialColNum == stbInfo->cols->size) { - if (stbInfo->autoTblCreating) { - *len = - snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s VALUES " - : "%s %s.%s USING %s.%s TAGS (%s) %s VALUES ", - STR_INSERT_INTO, database->dbName, - childTbl->name, database->dbName, - stbInfo->stbName, - stbInfo->tagDataBuf + - stbInfo->lenOfTags * tableSeq, ttl); - } else { - *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` VALUES " - : "%s %s.%s VALUES ", - STR_INSERT_INTO, - database->dbName, childTbl->name); - } - } else { - if (stbInfo->autoTblCreating) { - *len = snprintf( - pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` (%s) USING `%s`.`%s` TAGS (%s) %s VALUES " - : "%s %s.%s (%s) USING %s.%s TAGS (%s) %s VALUES ", - STR_INSERT_INTO, database->dbName, - childTbl->name, - stbInfo->partialColNameBuf, - database->dbName, stbInfo->stbName, - stbInfo->tagDataBuf + - stbInfo->lenOfTags * tableSeq, ttl); - } else { - *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, - g_arguments->escape_character - ? "%s `%s`.`%s` (%s) VALUES " - : "%s %s.%s (%s) VALUES ", - STR_INSERT_INTO, database->dbName, - childTbl->name, - stbInfo->partialColNameBuf); - } - } - - char *ownSampleDataBuf; - if (childTbl->useOwnSample) { - debugPrint("%s is using own sample data\n", - childTbl->name); - ownSampleDataBuf = childTbl->sampleDataBuf; - } else { - ownSampleDataBuf = stbInfo->sampleDataBuf; - } - for (int j = 0; j < g_arguments->reqPerReq; j++) { - if (stbInfo->useSampleTs - && (!stbInfo->random_data_source)) { - *len += - snprintf(pstr + *len, - TSDB_MAX_ALLOWED_SQL_LEN - *len, "(%s)", - sampleDataBuf + - *pos * stbInfo->lenOfCols); - } else { - int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); - *len += snprintf(pstr + *len, - TSDB_MAX_ALLOWED_SQL_LEN - *len, - "(%" PRId64 ",%s)", - disorderTs?disorderTs:*timestamp, - ownSampleDataBuf + - *pos * stbInfo->lenOfCols); - } - *pos += 1; - if (*pos >= g_arguments->prepared_rand) { - *pos = 0; - } - *timestamp += stbInfo->timestamp_step; - generated++; - if (*len > (TSDB_MAX_ALLOWED_SQL_LEN - - stbInfo->lenOfCols)) { - break; - } - if (i + generated >= stbInfo->insertRows) { - break; - } - } - - return generated; -} - -void *syncWriteProgressive(void *sarg) { - threadInfo * pThreadInfo = (threadInfo *)sarg; - SDataBase * database = pThreadInfo->dbInfo; - SSuperTable *stbInfo = pThreadInfo->stbInfo; - - // special deal flow for TAOSC_IFACE - if (insertDataMix(pThreadInfo, database, stbInfo)) { - // request be dealt by this function , so return - return NULL; - } - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - if (0 == pThreadInfo->vg->tbCountPerVgId) { - return NULL; - } - } else { - infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); - } -#else - infoPrint( - "thread[%d] start progressive inserting into table from " - "%" PRIu64 " to %" PRIu64 "\n", - pThreadInfo->threadID, pThreadInfo->start_table_from, - pThreadInfo->end_table_to + 1); -#endif - uint64_t lastPrintTime = toolsGetTimestampMs(); - uint64_t lastTotalInsertRows = 0; - int64_t startTs = toolsGetTimestampUs(); - int64_t endTs; - - for (uint64_t tableSeq = pThreadInfo->start_table_from; - tableSeq <= pThreadInfo->end_table_to; tableSeq++) { - char *sampleDataBuf; - SChildTable *childTbl; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if (g_arguments->nthreads_auto) { - childTbl = pThreadInfo->vg->childTblArray[tableSeq]; - } else { - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; - } -#else - childTbl = stbInfo->childTblArray[ - stbInfo->childTblExists? - tableSeq: - stbInfo->childTblFrom + tableSeq]; -#endif - if (childTbl->useOwnSample) { - sampleDataBuf = childTbl->sampleDataBuf; - } else { - sampleDataBuf = stbInfo->sampleDataBuf; - } - - int64_t timestamp = pThreadInfo->start_time; - uint64_t len = 0; - int32_t pos = 0; - if (stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating) { - taos_stmt_close(pThreadInfo->conn->stmt); - pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); - if (NULL == pThreadInfo->conn->stmt) { - errorPrint("taos_stmt_init() failed, reason: %s\n", - taos_errstr(NULL)); - g_fail = true; - goto free_of_progressive; - } - - if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tableSeq)) { - g_fail = true; - goto free_of_progressive; - } - } - - char ttl[SMALL_BUFF_LEN] = ""; - if (stbInfo->ttl != 0) { - snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); - } - for (uint64_t i = 0; i < stbInfo->insertRows;) { - if (g_arguments->terminate) { - goto free_of_progressive; - } - int32_t generated = 0; - switch (stbInfo->iface) { - case TAOSC_IFACE: - case REST_IFACE: - generated = prepareProgressDataSql( - pThreadInfo, - childTbl, - tableSeq, - sampleDataBuf, - ×tamp, i, ttl, &pos, &len); - break; - case STMT_IFACE: { - generated = prepareProgressDataStmt( - pThreadInfo, - childTbl, ×tamp, i, ttl); - break; - } - case SML_REST_IFACE: - case SML_IFACE: - generated = prepareProgressDataSml( - pThreadInfo, - childTbl, - tableSeq, ×tamp, i, ttl); - break; - default: - break; - } - if (generated < 0) { - g_fail = true; - goto free_of_progressive; - } - if (!stbInfo->non_stop) { - i += generated; - } - // only measure insert - startTs = toolsGetTimestampUs(); - int code = execInsert(pThreadInfo, generated); - if (code) { - if (NO_IF_FAILED == stbInfo->continueIfFail) { - warnPrint("The super table parameter " - "continueIfFail: %d, STOP insertion!\n", - stbInfo->continueIfFail); - g_fail = true; - goto free_of_progressive; - } else if (YES_IF_FAILED == stbInfo->continueIfFail) { - infoPrint("The super table parameter " - "continueIfFail: %d, " - "will continue to insert ..\n", - stbInfo->continueIfFail); - } else if (SMART_IF_FAILED == stbInfo->continueIfFail) { - warnPrint("The super table parameter " - "continueIfFail: %d, will create table " - "then insert ..\n", - stbInfo->continueIfFail); - int ret = smartContinueIfFail( - pThreadInfo, - childTbl, i, ttl); - if (0 != ret) { - g_fail = true; - goto free_of_progressive; - } - - code = execInsert(pThreadInfo, generated); - if (code) { - g_fail = true; - goto free_of_progressive; - } - } else { - warnPrint("Unknown super table parameter " - "continueIfFail: %d\n", - stbInfo->continueIfFail); - g_fail = true; - goto free_of_progressive; - } - } - endTs = toolsGetTimestampUs()+1; - - if (stbInfo->insert_interval > 0) { - debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", - __func__, __LINE__, stbInfo->insert_interval); - perfPrint("sleep %" PRIu64 " ms\n", - stbInfo->insert_interval); - toolsMsleep((int32_t)stbInfo->insert_interval); - } - - pThreadInfo->totalInsertRows += generated; - - if (g_arguments->terminate) { - goto free_of_progressive; - } - int protocol = stbInfo->lineProtocol; - switch (stbInfo->iface) { - case REST_IFACE: - case TAOSC_IFACE: - memset(pThreadInfo->buffer, 0, pThreadInfo->max_sql_len); - break; - case SML_REST_IFACE: - memset(pThreadInfo->buffer, 0, - g_arguments->reqPerReq * - (pThreadInfo->max_sql_len + 1)); - case SML_IFACE: - if (TSDB_SML_JSON_PROTOCOL == protocol) { - memset(pThreadInfo->lines[0], 0, - pThreadInfo->line_buf_len); - } else if (SML_JSON_TAOS_FORMAT == protocol) { - if (pThreadInfo->lines && pThreadInfo->lines[0]) { - tmfree(pThreadInfo->lines[0]); - pThreadInfo->lines[0] = NULL; - } - if (pThreadInfo->json_array) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - pThreadInfo->json_array = tools_cJSON_CreateArray(); - } else { - for (int j = 0; j < generated; j++) { - debugPrint("pThreadInfo->lines[%d]: %s\n", - j, pThreadInfo->lines[j]); - memset(pThreadInfo->lines[j], 0, - pThreadInfo->max_sql_len); - } - } - break; - case STMT_IFACE: - break; - } - - int64_t delay = endTs - startTs; - if (delay <= 0) { - debugPrint("thread[%d]: startTs: %"PRId64", endTs: %"PRId64"\n", - pThreadInfo->threadID, startTs, endTs); - } else { - perfPrint("insert execution time is %.6f s\n", - delay / 1E6); - - int64_t * pDelay = benchCalloc(1, sizeof(int64_t), false); - *pDelay = delay; - if (benchArrayPush(pThreadInfo->delayList, pDelay) == NULL) { - tmfree(pDelay); - } - pThreadInfo->totalDelay += delay; - } - - int64_t currentPrintTime = toolsGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30 * 1000) { - infoPrint( - "thread[%d] has currently inserted rows: " - "%" PRId64 ", peroid insert rate: %.3f rows/s \n", - pThreadInfo->threadID, pThreadInfo->totalInsertRows, - (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); - lastPrintTime = currentPrintTime; - lastTotalInsertRows = pThreadInfo->totalInsertRows; - } - if (i >= stbInfo->insertRows) { - break; - } - } // insertRows - } // tableSeq -free_of_progressive: - cleanupAndPrint(pThreadInfo, "progressive"); - return NULL; -} - -static int initStmtDataValue(SSuperTable *stbInfo, SChildTable *childTbl) { - int32_t columnCount = stbInfo->cols->size; - - char *sampleDataBuf; - if (childTbl) { - sampleDataBuf = childTbl->sampleDataBuf; - } else { - sampleDataBuf = stbInfo->sampleDataBuf; - } - int64_t lenOfOneRow = stbInfo->lenOfCols; - - if (stbInfo->useSampleTs) { - columnCount += 1; // for skipping first column - } - for (int i=0; i < g_arguments->prepared_rand; i++) { - int cursor = 0; - - for (int c = 0; c < columnCount; c++) { - char *restStr = sampleDataBuf - + lenOfOneRow * i + cursor; - int lengthOfRest = strlen(restStr); - - int index = 0; - for (index = 0; index < lengthOfRest; index++) { - if (restStr[index] == ',') { - break; - } - } - - cursor += index + 1; // skip ',' too - if ((0 == c) && stbInfo->useSampleTs) { - continue; - } - - char *tmpStr = calloc(1, index + 1); - if (NULL == tmpStr) { - errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", - __func__, __LINE__, index + 1); - return -1; - } - Field *col = benchArrayGet(stbInfo->cols, - (stbInfo->useSampleTs?c-1:c)); - char dataType = col->type; - - StmtData *stmtData; - if (childTbl) { - ChildField *childCol = - benchArrayGet(childTbl->childCols, - (stbInfo->useSampleTs?c-1:c)); - stmtData = &childCol->stmtData; - } else { - stmtData = &col->stmtData; - } - - strncpy(tmpStr, restStr, index); - - if (0 == strcmp(tmpStr, "NULL")) { - *(stmtData->is_null + i) = true; - } else { - switch (dataType) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *((int32_t*)stmtData->data + i) = atoi(tmpStr); - break; - case TSDB_DATA_TYPE_FLOAT: - *((float*)stmtData->data +i) = (float)atof(tmpStr); - break; - case TSDB_DATA_TYPE_DOUBLE: - *((double*)stmtData->data + i) = atof(tmpStr); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *((int16_t*)stmtData->data + i) = (int16_t)atoi(tmpStr); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); - break; - case TSDB_DATA_TYPE_BOOL: - *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); - break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - { - size_t tmpLen = strlen(tmpStr); - debugPrint("%s() LN%d, index: %d, " - "tmpStr len: %"PRIu64", col->length: %d\n", - __func__, __LINE__, - i, (uint64_t)tmpLen, col->length); - if (tmpLen-2 > col->length) { - errorPrint("data length %"PRIu64" " - "is larger than column length %d\n", - (uint64_t)tmpLen, col->length); - } - if (tmpLen > 2) { - strncpy((char *)stmtData->data - + i * col->length, - tmpStr+1, - min(col->length, tmpLen - 2)); - } else { - strncpy((char *)stmtData->data - + i*col->length, - "", 1); - } - } - break; - default: - break; - } - } - free(tmpStr); - } - } - return 0; -} - -static void initStmtData(char dataType, void **data, uint32_t length) { - char *tmpP = NULL; - - switch (dataType) { - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - tmpP = calloc(1, sizeof(int) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - tmpP = calloc(1, sizeof(int16_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_BOOL: - tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_FLOAT: - tmpP = calloc(1, sizeof(float) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_DOUBLE: - tmpP = calloc(1, sizeof(double) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - tmpP = calloc(1, g_arguments->prepared_rand * length); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - case TSDB_DATA_TYPE_TIMESTAMP: - tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); - assert(tmpP); - tmfree(*data); - *data = (void*)tmpP; - break; - - default: - errorPrint("Unknown data type: %s\n", - convertDatatypeToString(dataType)); - exit(EXIT_FAILURE); - } -} - -static int parseBufferToStmtBatchChildTbl(SSuperTable *stbInfo, - SChildTable* childTbl) { - int32_t columnCount = stbInfo->cols->size; - - for (int c = 0; c < columnCount; c++) { - Field *col = benchArrayGet(stbInfo->cols, c); - ChildField *childCol = benchArrayGet(childTbl->childCols, c); - char dataType = col->type; - - char *is_null = benchCalloc( - 1, sizeof(char) *g_arguments->prepared_rand, false); - - tmfree(childCol->stmtData.is_null); - childCol->stmtData.is_null = is_null; - - initStmtData(dataType, &(childCol->stmtData.data), col->length); - } - - return initStmtDataValue(stbInfo, childTbl); -} - -static int parseBufferToStmtBatch(SSuperTable* stbInfo) { - int32_t columnCount = stbInfo->cols->size; - - for (int c = 0; c < columnCount; c++) { - Field *col = benchArrayGet(stbInfo->cols, c); - char dataType = col->type; - - char *is_null = benchCalloc( - 1, sizeof(char) *g_arguments->prepared_rand, false); - tmfree(col->stmtData.is_null); - col->stmtData.is_null = is_null; - - initStmtData(dataType, &(col->stmtData.data), col->length); - } - - return initStmtDataValue(stbInfo, NULL); -} - -static int64_t fillChildTblNameByCount(SSuperTable *stbInfo) { - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - snprintf(stbInfo->childTblArray[i]->name, - TSDB_TABLE_NAME_LEN, - "%s%" PRIu64 "", - stbInfo->childTblPrefix, i); - debugPrint("%s(): %s\n", __func__, - stbInfo->childTblArray[i]->name); - } - - return stbInfo->childTblCount; -} - -static int64_t fillChildTblNameByFromTo(SDataBase *database, - SSuperTable* stbInfo) { - for (int64_t i = stbInfo->childTblFrom; i < stbInfo->childTblTo; i++) { - snprintf(stbInfo->childTblArray[i-stbInfo->childTblFrom]->name, - TSDB_TABLE_NAME_LEN, - "%s%" PRIu64 "", - stbInfo->childTblPrefix, i); - } - - return (stbInfo->childTblTo-stbInfo->childTblFrom); -} - -static int64_t fillChildTblNameByLimitOffset(SDataBase *database, - SSuperTable* stbInfo) { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; - if (g_arguments->taosc_version == 3) { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT DISTINCT(TBNAME) FROM %s.`%s` LIMIT %" PRId64 - " OFFSET %" PRIu64 "", - database->dbName, stbInfo->stbName, stbInfo->childTblLimit, - stbInfo->childTblOffset); - } else { - snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, - "SELECT TBNAME FROM %s.`%s` LIMIT %" PRId64 - " OFFSET %" PRIu64 "", - database->dbName, stbInfo->stbName, stbInfo->childTblLimit, - stbInfo->childTblOffset); - } - debugPrint("cmd: %s\n", cmd); - TAOS_RES *res = taos_query(conn->taos, cmd); - int32_t code = taos_errno(res); - int64_t count = 0; - if (code) { - printErrCmdCodeStr(cmd, code, res); - closeBenchConn(conn); - return -1; - } - TAOS_ROW row = NULL; - while ((row = taos_fetch_row(res)) != NULL) { - int *lengths = taos_fetch_lengths(res); - strncpy(stbInfo->childTblArray[count]->name, row[0], lengths[0]); - stbInfo->childTblArray[count]->name[lengths[0] + 1] = '\0'; - debugPrint("stbInfo->childTblArray[%" PRId64 "]->name: %s\n", - count, stbInfo->childTblArray[count]->name); - count++; - } - taos_free_result(res); - closeBenchConn(conn); - return count; -} - -static void preProcessArgument(SSuperTable *stbInfo) { - if (stbInfo->interlaceRows > g_arguments->reqPerReq) { - infoPrint( - "interlaceRows(%d) is larger than record per request(%u), which " - "will be set to %u\n", - stbInfo->interlaceRows, g_arguments->reqPerReq, - g_arguments->reqPerReq); - stbInfo->interlaceRows = g_arguments->reqPerReq; - } - - if (stbInfo->interlaceRows > stbInfo->insertRows) { - infoPrint( - "interlaceRows larger than insertRows %d > %" PRId64 "\n", - stbInfo->interlaceRows, stbInfo->insertRows); - infoPrint("%s", "interlaceRows will be set to 0\n"); - stbInfo->interlaceRows = 0; - } - - if (stbInfo->interlaceRows == 0 - && g_arguments->reqPerReq > stbInfo->insertRows) { - infoPrint("record per request (%u) is larger than " - "insert rows (%"PRIu64")" - " in progressive mode, which will be set to %"PRIu64"\n", - g_arguments->reqPerReq, stbInfo->insertRows, - stbInfo->insertRows); - g_arguments->reqPerReq = stbInfo->insertRows; - } - - if (stbInfo->interlaceRows > 0 && stbInfo->iface == STMT_IFACE - && stbInfo->autoTblCreating) { - infoPrint("%s", - "not support autocreate table with interlace row in stmt " - "insertion, will change to progressive mode\n"); - stbInfo->interlaceRows = 0; - } -} - -static int printTotalDelay(SDataBase *database, - int64_t totalDelay, - BArray *total_delay_list, - int threads, - int64_t totalInsertRows, - int64_t start, int64_t end) { - succPrint("Spent %.6f seconds to insert rows: %" PRIu64 - " with %d thread(s) into %s %.2f records/second\n", - (end - start)/1E6, totalInsertRows, threads, - database->dbName, - (double)(totalInsertRows / ((end - start)/1E6))); - if (!total_delay_list->size) { - return -1; - } - - succPrint("insert delay, " - "min: %.4fms, " - "avg: %.4fms, " - "p90: %.4fms, " - "p95: %.4fms, " - "p99: %.4fms, " - "max: %.4fms\n", - *(int64_t *)(benchArrayGet(total_delay_list, 0))/1E3, - (double)totalDelay/total_delay_list->size/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - * 0.9)))/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - * 0.95)))/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - * 0.99)))/1E3, - *(int64_t *)(benchArrayGet(total_delay_list, - (int32_t)(total_delay_list->size - - 1)))/1E3); - return 0; -} - -static int64_t fillChildTblNameImp(SDataBase *database, SSuperTable *stbInfo) { - int64_t ntables; - if (stbInfo->childTblLimit) { - ntables = fillChildTblNameByLimitOffset(database, stbInfo); - } else if (stbInfo->childTblFrom || stbInfo->childTblTo) { - ntables = fillChildTblNameByFromTo(database, stbInfo); - } else { - ntables = fillChildTblNameByCount(stbInfo); - } - return ntables; -} - -static int64_t fillChildTblName(SDataBase *database, SSuperTable *stbInfo) { - int64_t ntables = stbInfo->childTblCount; - stbInfo->childTblArray = benchCalloc(stbInfo->childTblCount, - sizeof(SChildTable*), true); - for (int64_t child = 0; child < stbInfo->childTblCount; child++) { - stbInfo->childTblArray[child] = - benchCalloc(1, sizeof(SChildTable), true); - } - - if (stbInfo->childTblCount == 1 && stbInfo->tags->size == 0) { - // Normal table - snprintf(stbInfo->childTblArray[0]->name, TSDB_TABLE_NAME_LEN, - "%s", stbInfo->stbName); - } else if ((stbInfo->iface != SML_IFACE - && stbInfo->iface != SML_REST_IFACE) - && stbInfo->childTblExists) { - ntables = fillChildTblNameImp(database, stbInfo); - } else { - ntables = fillChildTblNameByCount(stbInfo); - } - - return ntables; -} - -static int startMultiThreadInsertData(SDataBase* database, - SSuperTable* stbInfo) { - if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) - && !stbInfo->use_metric) { - errorPrint("%s", "schemaless cannot work without stable\n"); - return -1; - } - - preProcessArgument(stbInfo); - - int64_t ntables; - if (stbInfo->childTblTo > 0) { - ntables = stbInfo->childTblTo - stbInfo->childTblFrom; - } else if (stbInfo->childTblLimit) { - ntables = stbInfo->childTblLimit; - } else { - ntables = stbInfo->childTblCount; - } - if (ntables == 0) { - return 0; - } - - uint64_t tableFrom = 0; - int32_t threads = g_arguments->nthreads; - int64_t a = 0, b = 0; - -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return -1; - } - - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->tbCountPerVgId++; - } - } - } - - threads = 0; - for (int v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - infoPrint("Total %"PRId64" tables on bb %s's vgroup %d (id: %d)\n", - vg->tbCountPerVgId, database->dbName, v, vg->vgId); - if (vg->tbCountPerVgId) { - threads++; - } else { - continue; - } - vg->childTblArray = benchCalloc( - vg->tbCountPerVgId, sizeof(SChildTable*), true); - vg->tbOffset = 0; - } - for (int64_t i = 0; i < stbInfo->childTblCount; i++) { - int vgId; - int ret = taos_get_table_vgId( - conn->taos, database->dbName, - stbInfo->childTblArray[i]->name, &vgId); - if (ret < 0) { - errorPrint("Failed to get %s db's %s table's vgId\n", - database->dbName, - stbInfo->childTblArray[i]->name); - - closeBenchConn(conn); - return -1; - } - debugPrint("Db %s\'s table\'s %s vgId is: %d\n", - database->dbName, - stbInfo->childTblArray[i]->name, vgId); - for (int32_t v = 0; v < database->vgroups; v++) { - SVGroup *vg = benchArrayGet(database->vgArray, v); - if (vgId == vg->vgId) { - vg->childTblArray[vg->tbOffset] = - stbInfo->childTblArray[i]; - vg->tbOffset++; - } - } - } - closeBenchConn(conn); - } else { - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; - } - b = 0; - if (threads != 0) { - b = ntables % threads; - } - } - - int32_t vgFrom = 0; -#else - a = ntables / threads; - if (a < 1) { - threads = (int32_t)ntables; - a = 1; - } - b = 0; - if (threads != 0) { - b = ntables % threads; - } -#endif // TD_VER_COMPATIBLE_3_0_0_0 - pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), true); - threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), true); - - for (int32_t i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - pThreadInfo->threadID = i; - pThreadInfo->dbInfo = database; - pThreadInfo->stbInfo = stbInfo; - pThreadInfo->start_time = stbInfo->startTimestamp; - pThreadInfo->totalInsertRows = 0; - pThreadInfo->samplePos = 0; -#ifdef TD_VER_COMPATIBLE_3_0_0_0 - if ((0 == stbInfo->interlaceRows) - && (g_arguments->nthreads_auto)) { - int32_t j; - for (j = vgFrom; i < database->vgroups; j++) { - SVGroup *vg = benchArrayGet(database->vgArray, j); - if (0 == vg->tbCountPerVgId) { - continue; - } - pThreadInfo->vg = vg; - pThreadInfo->start_table_from = 0; - pThreadInfo->ntables = vg->tbCountPerVgId; - pThreadInfo->end_table_to = vg->tbCountPerVgId-1; - break; - } - vgFrom = j + 1; - } else { - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; - } -#else - pThreadInfo->start_table_from = tableFrom; - pThreadInfo->ntables = i < b ? a + 1 : a; - pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); - tableFrom = pThreadInfo->end_table_to + 1; -#endif // TD_VER_COMPATIBLE_3_0_0_0 - pThreadInfo->delayList = benchArrayInit(1, sizeof(int64_t)); - switch (stbInfo->iface) { - case REST_IFACE: { - if (stbInfo->interlaceRows > 0) { - pThreadInfo->buffer = new_ds(0); - } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); - } - int sockfd = createSockFd(); - if (sockfd < 0) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); - } - pThreadInfo->sockfd = sockfd; - break; - } - case STMT_IFACE: { - pThreadInfo->conn = initBenchConn(); - if (NULL == pThreadInfo->conn) { - FREE_PIDS_INFOS_RETURN_MINUS_1(); - } - pThreadInfo->conn->stmt = - taos_stmt_init(pThreadInfo->conn->taos); - if (NULL == pThreadInfo->conn->stmt) { - errorPrint("taos_stmt_init() failed, reason: %s\n", - taos_errstr(NULL)); - FREE_RESOURCE(); - return -1; - } - if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; - } - if (!stbInfo->autoTblCreating) { - if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, 0)) { - FREE_RESOURCE(); - return -1; - } - } - - pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); - pThreadInfo->bind_ts_array = - benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, - true); - pThreadInfo->bindParams = benchCalloc( - 1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), - true); - pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, - true); - parseBufferToStmtBatch(stbInfo); - for (int64_t child = 0; - child < stbInfo->childTblCount; child++) { - SChildTable *childTbl = stbInfo->childTblArray[child]; - if (childTbl->useOwnSample) { - parseBufferToStmtBatchChildTbl(stbInfo, childTbl); - } - } - - break; - } - case SML_REST_IFACE: { - int sockfd = createSockFd(); - if (sockfd < 0) { - free(pids); - free(infos); - return -1; - } - pThreadInfo->sockfd = sockfd; - } - case SML_IFACE: { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - errorPrint("%s() init connection failed\n", __func__); - FREE_RESOURCE(); - return -1; - } - if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - return -1; - } - pThreadInfo->max_sql_len = - stbInfo->lenOfCols + stbInfo->lenOfTags; - if (stbInfo->iface == SML_REST_IFACE) { - pThreadInfo->buffer = - benchCalloc(1, g_arguments->reqPerReq * - (1 + pThreadInfo->max_sql_len), true); - } - int protocol = stbInfo->lineProtocol; - if (TSDB_SML_JSON_PROTOCOL != protocol - && SML_JSON_TAOS_FORMAT != protocol) { - pThreadInfo->sml_tags = - (char **)benchCalloc(pThreadInfo->ntables, - sizeof(char *), true); - for (int t = 0; t < pThreadInfo->ntables; t++) { - pThreadInfo->sml_tags[t] = - benchCalloc(1, stbInfo->lenOfTags, true); - } - - for (int t = 0; t < pThreadInfo->ntables; t++) { - if (generateRandData( - stbInfo, pThreadInfo->sml_tags[t], - stbInfo->lenOfTags, - stbInfo->lenOfCols + stbInfo->lenOfTags, - stbInfo->tags, 1, true, NULL)) { - return -1; - } - debugPrint("pThreadInfo->sml_tags[%d]: %s\n", t, - pThreadInfo->sml_tags[t]); - } - pThreadInfo->lines = - benchCalloc(g_arguments->reqPerReq, - sizeof(char *), true); - - for (int j = 0; (j < g_arguments->reqPerReq - && !g_arguments->terminate); j++) { - pThreadInfo->lines[j] = - benchCalloc(1, pThreadInfo->max_sql_len, true); - } - } else { - pThreadInfo->json_array = tools_cJSON_CreateArray(); - pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); - pThreadInfo->sml_tags_json_array = (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); - for (int t = 0; t < pThreadInfo->ntables; t++) { - if (stbInfo->lineProtocol == TSDB_SML_JSON_PROTOCOL) { - generateSmlJsonTags( - pThreadInfo->sml_json_tags, - pThreadInfo->sml_tags_json_array, - stbInfo, - pThreadInfo->start_table_from, t); - } else { - generateSmlTaosJsonTags( - pThreadInfo->sml_json_tags, stbInfo, - pThreadInfo->start_table_from, t); - } - } - pThreadInfo->lines = (char **)benchCalloc( - 1, sizeof(char *), true); - if ((0 == stbInfo->interlaceRows) - && (TSDB_SML_JSON_PROTOCOL == protocol)) { - pThreadInfo->line_buf_len = - g_arguments->reqPerReq * - accumulateRowLen(pThreadInfo->stbInfo->tags, - pThreadInfo->stbInfo->iface); - debugPrint("%s() LN%d, line_buf_len=%d\n", - __func__, __LINE__, pThreadInfo->line_buf_len); - pThreadInfo->lines[0] = benchCalloc( - 1, pThreadInfo->line_buf_len, true); - pThreadInfo->sml_json_value_array = - (char **)benchCalloc( - pThreadInfo->ntables, sizeof(char *), true); - for (int t = 0; t < pThreadInfo->ntables; t++) { - generateSmlJsonValues( - pThreadInfo->sml_json_value_array, stbInfo, t); - } - } - } - break; - } - case TAOSC_IFACE: { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - errorPrint("%s() failed to connect\n", __func__); - FREE_RESOURCE(); - return -1; - } - char* command = benchCalloc(1, SHORT_1K_SQL_BUFF_LEN, false); - snprintf(command, SHORT_1K_SQL_BUFF_LEN, - g_arguments->escape_character - ? "USE `%s`" - : "USE %s", - database->dbName); - if (queryDbExecCall(pThreadInfo->conn, command)) { - errorPrint("taos select database(%s) failed\n", - database->dbName); - FREE_RESOURCE(); - tmfree(command); - return -1; - } - tmfree(command); - command = NULL; - - if (stbInfo->interlaceRows > 0) { - pThreadInfo->buffer = new_ds(0); - } else { - pThreadInfo->buffer = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); - if (g_arguments->check_sql) { - pThreadInfo->csql = - benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); - memset(pThreadInfo->csql, 0, TSDB_MAX_ALLOWED_SQL_LEN); - } - } - - break; - } - default: - break; - } - } - - infoPrint("Estimate memory usage: %.2fMB\n", - (double)g_memoryUsage / 1048576); - prompt(0); - - // create threads - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - threadInfo *pThreadInfo = infos + i; - if (stbInfo->interlaceRows > 0) { - pthread_create(pids + i, NULL, - syncWriteInterlace, pThreadInfo); - } else { - pthread_create(pids + i, NULL, - syncWriteProgressive, pThreadInfo); - } - } - - int64_t start = toolsGetTimestampUs(); - - // wait threads - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - pthread_join(pids[i], NULL); - } - - int64_t end = toolsGetTimestampUs()+1; - - if (g_arguments->terminate) toolsMsleep(100); - - BArray * total_delay_list = benchArrayInit(1, sizeof(int64_t)); - int64_t totalDelay = 0; - uint64_t totalInsertRows = 0; - - // free threads resource - for (int i = 0; i < threads; i++) { - threadInfo *pThreadInfo = infos + i; - // free check sql - if (pThreadInfo->csql) { - tmfree(pThreadInfo->csql); - pThreadInfo->csql = NULL; - } - - int protocol = stbInfo->lineProtocol; - switch (stbInfo->iface) { - case REST_IFACE: - if (g_arguments->terminate) - toolsMsleep(100); - destroySockFd(pThreadInfo->sockfd); - if (stbInfo->interlaceRows > 0) { - free_ds(&pThreadInfo->buffer); - } else { - tmfree(pThreadInfo->buffer); - pThreadInfo->buffer = NULL; - } - break; - case SML_REST_IFACE: - if (g_arguments->terminate) - toolsMsleep(100); - tmfree(pThreadInfo->buffer); - // on-purpose no break here - case SML_IFACE: - if (TSDB_SML_JSON_PROTOCOL != protocol - && SML_JSON_TAOS_FORMAT != protocol) { - for (int t = 0; t < pThreadInfo->ntables; t++) { - tmfree(pThreadInfo->sml_tags[t]); - } - for (int j = 0; j < g_arguments->reqPerReq; j++) { - tmfree(pThreadInfo->lines[j]); - } - tmfree(pThreadInfo->sml_tags); - pThreadInfo->sml_tags = NULL; - } else { - for (int t = 0; t < pThreadInfo->ntables; t++) { - tmfree(pThreadInfo->sml_tags_json_array[t]); - } - tmfree(pThreadInfo->sml_tags_json_array); - pThreadInfo->sml_tags_json_array = NULL; - if (pThreadInfo->sml_json_tags) { - tools_cJSON_Delete(pThreadInfo->sml_json_tags); - pThreadInfo->sml_json_tags = NULL; - } - if (pThreadInfo->json_array) { - tools_cJSON_Delete(pThreadInfo->json_array); - pThreadInfo->json_array = NULL; - } - } - if (pThreadInfo->lines) { - if ((0 == stbInfo->interlaceRows) - && (TSDB_SML_JSON_PROTOCOL == protocol)) { - tmfree(pThreadInfo->lines[0]); - for (int t = 0; t < pThreadInfo->ntables; t++) { - tmfree(pThreadInfo->sml_json_value_array[t]); - } - tmfree(pThreadInfo->sml_json_value_array); - } - tmfree(pThreadInfo->lines); - pThreadInfo->lines = NULL; - } - break; - - case STMT_IFACE: - taos_stmt_close(pThreadInfo->conn->stmt); - tmfree(pThreadInfo->bind_ts); - tmfree(pThreadInfo->bind_ts_array); - tmfree(pThreadInfo->bindParams); - tmfree(pThreadInfo->is_null); - break; - - case TAOSC_IFACE: - if (stbInfo->interlaceRows > 0) { - free_ds(&pThreadInfo->buffer); - } else { - tmfree(pThreadInfo->buffer); - pThreadInfo->buffer = NULL; - } - break; - - default: - break; - } - totalInsertRows += pThreadInfo->totalInsertRows; - totalDelay += pThreadInfo->totalDelay; - benchArrayAddBatch(total_delay_list, pThreadInfo->delayList->pData, - pThreadInfo->delayList->size); - tmfree(pThreadInfo->delayList); - pThreadInfo->delayList = NULL; - // free conn - if (pThreadInfo->conn) { - closeBenchConn(pThreadInfo->conn); - pThreadInfo->conn = NULL; - } - } - - // calculate result - qsort(total_delay_list->pData, total_delay_list->size, - total_delay_list->elemSize, compare); - - if (g_arguments->terminate) toolsMsleep(100); - - free(pids); - free(infos); - - int ret = printTotalDelay(database, totalDelay, - total_delay_list, threads, - totalInsertRows, start, end); - benchArrayDestroy(total_delay_list); - if (g_fail || ret) { - return -1; - } - return 0; -} - -static int getStbInsertedRows(char* dbName, char* stbName, TAOS* taos) { - int rows = 0; - char command[SHORT_1K_SQL_BUFF_LEN]; - snprintf(command, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(*) FROM %s.%s", - dbName, stbName); - TAOS_RES* res = taos_query(taos, command); - int code = taos_errno(res); - if (code != 0) { - printErrCmdCodeStr(command, code, res); - return -1; - } - TAOS_ROW row = taos_fetch_row(res); - if (row == NULL) { - rows = 0; - } else { - rows = (int)*(int64_t*)row[0]; - } - taos_free_result(res); - return rows; -} - -static void create_tsma(TSMA* tsma, SBenchConn* conn, char* stbName) { - char command[SHORT_1K_SQL_BUFF_LEN]; - int len = snprintf(command, SHORT_1K_SQL_BUFF_LEN, - "CREATE sma INDEX %s ON %s function(%s) " - "INTERVAL (%s) SLIDING (%s)", - tsma->name, stbName, tsma->func, - tsma->interval, tsma->sliding); - if (tsma->custom) { - snprintf(command + len, SHORT_1K_SQL_BUFF_LEN - len, - " %s", tsma->custom); - } - int code = queryDbExecCall(conn, command); - if (code == 0) { - infoPrint("successfully create tsma with command <%s>\n", command); - } -} - -static void* create_tsmas(void* args) { - tsmaThreadInfo* pThreadInfo = (tsmaThreadInfo*) args; - int inserted_rows = 0; - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - return NULL; - } - int finished = 0; - if (taos_select_db(conn->taos, pThreadInfo->dbName)) { - errorPrint("failed to use database (%s)\n", pThreadInfo->dbName); - closeBenchConn(conn); - return NULL; - } - while (finished < pThreadInfo->tsmas->size && inserted_rows >= 0) { - inserted_rows = (int)getStbInsertedRows( - pThreadInfo->dbName, pThreadInfo->stbName, conn->taos); - for (int i = 0; i < pThreadInfo->tsmas->size; i++) { - TSMA* tsma = benchArrayGet(pThreadInfo->tsmas, i); - if (!tsma->done && inserted_rows >= tsma->start_when_inserted) { - create_tsma(tsma, conn, pThreadInfo->stbName); - tsma->done = true; - finished++; - break; - } - } - toolsMsleep(10); - } - benchArrayDestroy(pThreadInfo->tsmas); - closeBenchConn(conn); - return NULL; -} - -static int32_t createStream(SSTREAM* stream) { - int32_t code = -1; - char * command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); - snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, "DROP STREAM IF EXISTS %s", - stream->stream_name); - infoPrint("%s\n", command); - SBenchConn* conn = initBenchConn(); - if (NULL == conn) { - goto END_STREAM; - } - - code = queryDbExecCall(conn, command); - int32_t trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds then re-drop stream %s\n", - g_arguments->trying_interval, stream->stream_name); - toolsMsleep(g_arguments->trying_interval); - code = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - - if (code) { - closeBenchConn(conn); - goto END_STREAM; - } - - memset(command, 0, TSDB_MAX_ALLOWED_SQL_LEN); - int pos = snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, - "CREATE STREAM IF NOT EXISTS %s ", stream->stream_name); - if (stream->trigger_mode[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "TRIGGER %s ", stream->trigger_mode); - } - if (stream->watermark[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "WATERMARK %s ", stream->watermark); - } - if (stream->ignore_update[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "IGNORE UPDATE %s ", stream->ignore_update); - } - if (stream->ignore_expired[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "IGNORE EXPIRED %s ", stream->ignore_expired); - } - if (stream->fill_history[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "FILL_HISTORY %s ", stream->fill_history); - } - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "INTO %s ", stream->stream_stb); - if (stream->stream_stb_field[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "%s ", stream->stream_stb_field); - } - if (stream->stream_tag_field[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "TAGS%s ", stream->stream_tag_field); - } - if (stream->subtable[0] != '\0') { - pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "SUBTABLE%s ", stream->subtable); - } - snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, - "as %s", stream->source_sql); - infoPrint("%s\n", command); - - code = queryDbExecCall(conn, command); - trying = g_arguments->keep_trying; - while (code && trying) { - infoPrint("will sleep %"PRIu32" milliseconds " - "then re-create stream %s\n", - g_arguments->trying_interval, stream->stream_name); - toolsMsleep(g_arguments->trying_interval); - code = queryDbExecCall(conn, command); - if (trying != -1) { - trying--; - } - } - - closeBenchConn(conn); -END_STREAM: - tmfree(command); - return code; -} - -int insertTestProcess() { - prompt(0); - - encodeAuthBase64(); - for (int i = 0; i < g_arguments->databases->size; i++) { - if (REST_IFACE == g_arguments->iface) { - if (0 != convertServAddr(g_arguments->iface, - false, - 1)) { - return -1; - } - } - SDataBase * database = benchArrayGet(g_arguments->databases, i); - - if (database->drop && !(g_arguments->supplementInsert)) { - if (database->superTbls) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, 0); - if (stbInfo && (REST_IFACE == stbInfo->iface)) { - if (0 != convertServAddr(stbInfo->iface, - stbInfo->tcpTransfer, - stbInfo->lineProtocol)) { - return -1; - } - } - } - if (createDatabase(database)) { - errorPrint("failed to create database (%s)\n", - database->dbName); - return -1; - } - succPrint("created database (%s)\n", database->dbName); - } - } - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (int j = 0; j < database->superTbls->size; j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - if (stbInfo->iface != SML_IFACE - && stbInfo->iface != SML_REST_IFACE - && !stbInfo->childTblExists) { -#ifdef WEBSOCKET - if (g_arguments->websocket) { - dropSuperTable(database, stbInfo); - } -#endif - if (getSuperTableFromServer(database, stbInfo) != 0) { - if (createSuperTable(database, stbInfo)) { - return -1; - } - } - } - fillChildTblName(database, stbInfo); - if (0 != prepareSampleData(database, stbInfo)) { - return -1; - } - } - } - } - - if (g_arguments->taosc_version == 3) { - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase* database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (int j = 0; (j < database->superTbls->size - && !g_arguments->terminate); j++) { - SSuperTable* stbInfo = - benchArrayGet(database->superTbls, j); - if (stbInfo->tsmas == NULL) { - continue; - } - if (stbInfo->tsmas->size > 0) { - tsmaThreadInfo* pThreadInfo = - benchCalloc(1, sizeof(tsmaThreadInfo), true); - pthread_t tsmas_pid = {0}; - pThreadInfo->dbName = database->dbName; - pThreadInfo->stbName = stbInfo->stbName; - pThreadInfo->tsmas = stbInfo->tsmas; - pthread_create(&tsmas_pid, NULL, - create_tsmas, pThreadInfo); - } - } - } - } - } - - if (createChildTables()) return -1; - - if (g_arguments->taosc_version == 3) { - for (int j = 0; j < g_arguments->streams->size; j++) { - SSTREAM * stream = benchArrayGet(g_arguments->streams, j); - if (stream->drop) { - if (createStream(stream)) { - return -1; - } - } - } - } - - // create sub threads for inserting data - for (int i = 0; i < g_arguments->databases->size; i++) { - SDataBase * database = benchArrayGet(g_arguments->databases, i); - if (database->superTbls) { - for (uint64_t j = 0; j < database->superTbls->size; j++) { - SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); - if (stbInfo->insertRows == 0) { - continue; - } - prompt(stbInfo->non_stop); - if (startMultiThreadInsertData(database, stbInfo)) { - return -1; - } - } - } - } - return 0; -} +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the MIT license as published by the Free Software + * Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + */ + +#include +#include +#include + +#define FREE_PIDS_INFOS_RETURN_MINUS_1() \ + do { \ + tmfree(pids); \ + tmfree(infos); \ + return -1; \ + } while (0) + +#define FREE_RESOURCE() \ + do { \ + if (pThreadInfo->conn) \ + closeBenchConn(pThreadInfo->conn); \ + benchArrayDestroy(pThreadInfo->delayList); \ + tmfree(pids); \ + tmfree(infos); \ + } while (0) \ + +static int getSuperTableFromServerRest( + SDataBase* database, SSuperTable* stbInfo, char *command) { + + return -1; + // TODO(me): finish full implementation +#if 0 + int sockfd = createSockFd(); + if (sockfd < 0) { + return -1; + } + + int code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + + destroySockFd(sockfd); +#endif // 0 +} + +static int getSuperTableFromServerTaosc( + SDataBase* database, SSuperTable* stbInfo, char *command) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + return -1; + } +#endif + TAOS_RES * res; + TAOS_ROW row = NULL; + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + + res = taos_query(conn->taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printWarnCmdCodeStr(command, code, res); + infoPrint("stable %s does not exist, will create one\n", + stbInfo->stbName); + closeBenchConn(conn); + return -1; + } + infoPrint("find stable<%s>, will get meta data from server\n", + stbInfo->stbName); + benchArrayClear(stbInfo->tags); + benchArrayClear(stbInfo->cols); + int count = 0; + while ((row = taos_fetch_row(res)) != NULL) { + if (count == 0) { + count++; + continue; + } + int32_t *lengths = taos_fetch_lengths(res); + if (lengths == NULL) { + errorPrint("%s", "failed to execute taos_fetch_length\n"); + taos_free_result(res); + closeBenchConn(conn); + return -1; + } + if (strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "tag", + strlen("tag")) == 0) { + Field* tag = benchCalloc(1, sizeof(Field), true); + benchArrayPush(stbInfo->tags, tag); + tag = benchArrayGet(stbInfo->tags, stbInfo->tags->size - 1); + tag->type = convertStringToDatatype( + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); + tag->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tag->min = convertDatatypeToDefaultMin(tag->type); + tag->max = convertDatatypeToDefaultMax(tag->type); + tstrncpy(tag->name, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); + } else { + Field * col = benchCalloc(1, sizeof(Field), true); + benchArrayPush(stbInfo->cols, col); + col = benchArrayGet(stbInfo->cols, stbInfo->cols->size - 1); + col->type = convertStringToDatatype( + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + lengths[TSDB_DESCRIBE_METRIC_TYPE_INDEX]); + col->length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + col->min = convertDatatypeToDefaultMin(col->type); + col->max = convertDatatypeToDefaultMax(col->type); + tstrncpy(col->name, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + lengths[TSDB_DESCRIBE_METRIC_FIELD_INDEX] + 1); + } + } + taos_free_result(res); + closeBenchConn(conn); + return 0; +} + +static int getSuperTableFromServer(SDataBase* database, SSuperTable* stbInfo) { + int ret = 0; + + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "DESCRIBE `%s`.`%s`", database->dbName, + stbInfo->stbName); + + if (REST_IFACE == stbInfo->iface) { + ret = getSuperTableFromServerRest(database, stbInfo, command); + } else { + ret = getSuperTableFromServerTaosc(database, stbInfo, command); + } + + return ret; +} + +static int queryDbExec(SDataBase *database, + SSuperTable *stbInfo, char *command) { + int ret = 0; + if (REST_IFACE == stbInfo->iface) { + if (0 != convertServAddr(stbInfo->iface, false, 1)) { + errorPrint("%s", "Failed to convert server address\n"); + return -1; + } + int sockfd = createSockFd(); + if (sockfd < 0) { + ret = -1; + } else { + ret = queryDbExecRest(command, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + sockfd); + destroySockFd(sockfd); + } + } else { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + ret = -1; + } else { + ret = queryDbExecCall(conn, command); + int32_t trying = g_arguments->keep_trying; + while (ret && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then re-create " + "supertable %s\n", + g_arguments->trying_interval, stbInfo->stbName); + toolsMsleep(g_arguments->trying_interval); + ret = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + if (0 != ret) { + errorPrint("create supertable %s failed!\n\n", + stbInfo->stbName); + ret = -1; + } + closeBenchConn(conn); + } + } + + return ret; +} + +#ifdef WEBSOCKET +static void dropSuperTable(SDataBase* database, SSuperTable* stbInfo) { + if (g_arguments->supplementInsert) { + return; + } + + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + snprintf(command, sizeof(command), + g_arguments->escape_character + ? "DROP TABLE `%s`.`%s`" + : "DROP TABLE %s.%s", + database->dbName, + stbInfo->stbName); + + infoPrint("drop stable: <%s>\n", command); + queryDbExec(database, stbInfo, command); + + return; +} +#endif // WEBSOCKET + +static int createSuperTable(SDataBase* database, SSuperTable* stbInfo) { + if (g_arguments->supplementInsert) { + return 0; + } + + uint32_t col_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->cols->size; + char *colsBuf = benchCalloc(1, col_buffer_len, false); + char* command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + int len = 0; + + for (int colIndex = 0; colIndex < stbInfo->cols->size; colIndex++) { + Field * col = benchArrayGet(stbInfo->cols, colIndex); + int n; + if (col->type == TSDB_DATA_TYPE_BINARY || + col->type == TSDB_DATA_TYPE_NCHAR) { + n = snprintf(colsBuf + len, col_buffer_len - len, + ",%s %s(%d)", col->name, + convertDatatypeToString(col->type), col->length); + } else { + n = snprintf(colsBuf + len, col_buffer_len - len, + ",%s %s", col->name, + convertDatatypeToString(col->type)); + } + if (n < 0 || n >= col_buffer_len - len) { + errorPrint("%s() LN%d, snprintf overflow on %d\n", + __func__, __LINE__, colIndex); + break; + } else { + len += n; + } + } + + // save for creating child table + stbInfo->colsOfCreateChildTable = + (char *)benchCalloc(len + TIMESTAMP_BUFF_LEN, 1, true); + + snprintf(stbInfo->colsOfCreateChildTable, len + TIMESTAMP_BUFF_LEN, + "(ts timestamp%s)", colsBuf); + + if (stbInfo->tags->size == 0) { + free(colsBuf); + free(command); + return 0; + } + + uint32_t tag_buffer_len = (TSDB_COL_NAME_LEN + 15) * stbInfo->tags->size; + char *tagsBuf = benchCalloc(1, tag_buffer_len, false); + int tagIndex; + len = 0; + + int n; + n = snprintf(tagsBuf + len, tag_buffer_len - len, "("); + if (n < 0 || n >= tag_buffer_len - len) { + errorPrint("%s() LN%d snprintf overflow\n", + __func__, __LINE__); + free(colsBuf); + free(command); + tmfree(tagsBuf); + return -1; + } else { + len += n; + } + for (tagIndex = 0; tagIndex < stbInfo->tags->size; tagIndex++) { + Field *tag = benchArrayGet(stbInfo->tags, tagIndex); + if (tag->type == TSDB_DATA_TYPE_BINARY || + tag->type == TSDB_DATA_TYPE_NCHAR) { + n = snprintf(tagsBuf + len, tag_buffer_len - len, + "%s %s(%d),", tag->name, + convertDatatypeToString(tag->type), tag->length); + } else if (tag->type == TSDB_DATA_TYPE_JSON) { + n = snprintf(tagsBuf + len, tag_buffer_len - len, + "%s json", tag->name); + if (n < 0 || n >= tag_buffer_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, tagIndex); + break; + } else { + len += n; + } + goto skip; + } else { + n = snprintf(tagsBuf + len, tag_buffer_len - len, + "%s %s,", tag->name, + convertDatatypeToString(tag->type)); + } + + if (n < 0 || n >= tag_buffer_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, tagIndex); + break; + } else { + len += n; + } + } + len -= 1; +skip: + snprintf(tagsBuf + len, tag_buffer_len - len, ")"); + + int length = snprintf( + command, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "CREATE TABLE `%s`.`%s` (ts TIMESTAMP%s) TAGS %s" + : "CREATE TABLE %s.%s (ts TIMESTAMP%s) TAGS %s", + database->dbName, stbInfo->stbName, colsBuf, tagsBuf); + tmfree(colsBuf); + tmfree(tagsBuf); + if (stbInfo->comment != NULL) { + length += snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, + " COMMENT '%s'", stbInfo->comment); + } + if (stbInfo->delay >= 0) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, " DELAY %d", + stbInfo->delay); + } + if (stbInfo->file_factor >= 0) { + length += + snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, " FILE_FACTOR %f", + (float)stbInfo->file_factor / 100); + } + if (stbInfo->rollup != NULL) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " ROLLUP(%s)", stbInfo->rollup); + } + + if (stbInfo->max_delay != NULL) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " MAX_DELAY %s", stbInfo->max_delay); + } + + if (stbInfo->watermark != NULL) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " WATERMARK %s", stbInfo->watermark); + } + + if (stbInfo->ttl != 0) { + length += snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " TTL %d", stbInfo->ttl); + } + + bool first_sma = true; + for (int i = 0; i < stbInfo->cols->size; i++) { + Field * col = benchArrayGet(stbInfo->cols, i); + if (col->sma) { + if (first_sma) { + n = snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + " SMA(%s", col->name); + first_sma = false; + } else { + n = snprintf(command + length, + TSDB_MAX_ALLOWED_SQL_LEN - length, + ",%s", col->name); + } + + if (n < 0 || n > TSDB_MAX_ALLOWED_SQL_LEN - length) { + errorPrint("%s() LN%d snprintf overflow on %d iteral\n", + __func__, __LINE__, i); + break; + } else { + length += n; + } + } + } + if (!first_sma) { + snprintf(command + length, TSDB_MAX_ALLOWED_SQL_LEN - length, ")"); + } + infoPrint("create stable: <%s>\n", command); + + int ret = queryDbExec(database, stbInfo, command); + free(command); + return ret; +} + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 +int32_t getVgroupsOfDb(SBenchConn *conn, SDataBase *database) { + int vgroups = 0; + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "USE `%s`" + : "USE %s", + database->dbName); + + int32_t code; + TAOS_RES *res = NULL; + + res = taos_query(conn->taos, cmd); + code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + return -1; + } + taos_free_result(res); + + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, "SHOW VGROUPS"); + res = taos_query(conn->taos, cmd); + code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + return -1; + } + + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res)) != NULL) { + vgroups++; + } + debugPrint("%s() LN%d, vgroups: %d\n", __func__, __LINE__, vgroups); + taos_free_result(res); + + database->vgroups = vgroups; + database->vgArray = benchArrayInit(vgroups, sizeof(SVGroup)); + for (int32_t v = 0; (v < vgroups + && !g_arguments->terminate); v++) { + SVGroup *vg = benchCalloc(1, sizeof(SVGroup), true); + benchArrayPush(database->vgArray, vg); + } + + res = taos_query(conn->taos, cmd); + code = taos_errno(res); + if (code) { + printErrCmdCodeStr(cmd, code, res); + return -1; + } + + int32_t vgItem = 0; + while (((row = taos_fetch_row(res)) != NULL) + && !g_arguments->terminate) { + SVGroup *vg = benchArrayGet(database->vgArray, vgItem); + vg->vgId = *(int32_t*)row[0]; + vgItem++; + } + taos_free_result(res); + + return vgroups; +} +#endif // TD_VER_COMPATIBLE_3_0_0_0 + +int geneDbCreateCmd(SDataBase *database, char *command, int remainVnodes) { + int dataLen = 0; + int n; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto || (-1 != g_arguments->inputted_vgroups)) { + n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, + g_arguments->escape_character + ? "CREATE DATABASE IF NOT EXISTS `%s` VGROUPS %d" + : "CREATE DATABASE IF NOT EXISTS %s VGROUPS %d", + database->dbName, + (-1 != g_arguments->inputted_vgroups)? + g_arguments->inputted_vgroups: + min(remainVnodes, toolsGetNumberOfCores())); + } else { + n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, + g_arguments->escape_character + ? "CREATE DATABASE IF NOT EXISTS `%s`" + : "CREATE DATABASE IF NOT EXISTS %s", + database->dbName); + } +#else + n = snprintf(command + dataLen, SHORT_1K_SQL_BUFF_LEN - dataLen, + g_arguments->escape_character + ? "CREATE DATABASE IF NOT EXISTS `%s`" + : "CREATE DATABASE IF NOT EXISTS %s", database->dbName); +#endif // TD_VER_COMPATIBLE_3_0_0_0 + if (n < 0 || n >= SHORT_1K_SQL_BUFF_LEN - dataLen) { + errorPrint("%s() LN%d snprintf overflow\n", + __func__, __LINE__); + return -1; + } else { + dataLen += n; + } + + if (database->cfgs) { + for (int i = 0; i < database->cfgs->size; i++) { + SDbCfg* cfg = benchArrayGet(database->cfgs, i); + if (cfg->valuestring) { + n = snprintf(command + dataLen, + TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " %s %s", cfg->name, cfg->valuestring); + } else { + n = snprintf(command + dataLen, + TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " %s %d", cfg->name, cfg->valueint); + } + if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - dataLen) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, i); + break; + } else { + dataLen += n; + } + } + } + + switch (database->precision) { + case TSDB_TIME_PRECISION_MILLI: + snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " PRECISION \'ms\';"); + break; + case TSDB_TIME_PRECISION_MICRO: + snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " PRECISION \'us\';"); + break; + case TSDB_TIME_PRECISION_NANO: + snprintf(command + dataLen, TSDB_MAX_ALLOWED_SQL_LEN - dataLen, + " PRECISION \'ns\';"); + break; + } + + return dataLen; +} + +int createDatabaseRest(SDataBase* database) { + int32_t code = 0; + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + + int sockfd = createSockFd(); + if (sockfd < 0) { + return -1; + } + + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "DROP DATABASE IF EXISTS `%s`;" + : "DROP DATABASE IF EXISTS %s;", + database->dbName); + code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + if (code != 0) { + errorPrint("Failed to drop database %s\n", database->dbName); + } else { + int remainVnodes = INT_MAX; + geneDbCreateCmd(database, command, remainVnodes); + code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + int32_t trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then " + "re-create database %s\n", + g_arguments->trying_interval, database->dbName); + toolsMsleep(g_arguments->trying_interval); + code = postProceSql(command, + database->dbName, + database->precision, + REST_IFACE, + 0, + g_arguments->port, + false, + sockfd, + NULL); + if (trying != -1) { + trying--; + } + } + } + destroySockFd(sockfd); + return code; +} + +int32_t getRemainVnodes(SBenchConn *conn) { + int remainVnodes = 0; + char command[SHORT_1K_SQL_BUFF_LEN] = "SHOW DNODES"; + + TAOS_RES *res = taos_query(conn->taos, command); + int32_t code = taos_errno(res); + if (code) { + printErrCmdCodeStr(command, code, res); + closeBenchConn(conn); + return -1; + } + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res)) != NULL) { + remainVnodes += (*(int16_t*)(row[3]) - *(int16_t*)(row[2])); + } + debugPrint("%s() LN%d, remainVnodes: %d\n", + __func__, __LINE__, remainVnodes); + taos_free_result(res); + return remainVnodes; +} + +int createDatabaseTaosc(SDataBase* database) { + char command[SHORT_1K_SQL_BUFF_LEN] = "\0"; + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + if (g_arguments->taosc_version == 3) { + for (int i = 0; i < g_arguments->streams->size; i++) { + SSTREAM* stream = benchArrayGet(g_arguments->streams, i); + if (stream->drop) { + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "DROP STREAM IF EXISTS %s;", + stream->stream_name); + if (queryDbExecCall(conn, command)) { + closeBenchConn(conn); + return -1; + } + infoPrint("%s\n", command); + memset(command, 0, SHORT_1K_SQL_BUFF_LEN); + } + } + } + + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "DROP DATABASE IF EXISTS `%s`;": + "DROP DATABASE IF EXISTS %s;", + database->dbName); + if (0 != queryDbExecCall(conn, command)) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + warnPrint("%s", "TDengine cloud normal users have no privilege " + "to drop database! DROP DATABASE failure is ignored!\n"); + } else { +#endif + closeBenchConn(conn); + return -1; +#ifdef WEBSOCKET + } +#endif + } + + int remainVnodes = INT_MAX; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto) { + remainVnodes = getRemainVnodes(conn); + if (0 >= remainVnodes) { + errorPrint("Remain vnodes %d, failed to create database\n", + remainVnodes); + return -1; + } + } +#endif + geneDbCreateCmd(database, command, remainVnodes); + + int32_t code = queryDbExecCall(conn, command); + int32_t trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then " + "re-create database %s\n", + g_arguments->trying_interval, database->dbName); + toolsMsleep(g_arguments->trying_interval); + code = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + + if (code) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + warnPrint("%s", "TDengine cloud normal users have no privilege " + "to create database! CREATE DATABASE " + "failure is ignored!\n"); + } else { +#endif + + closeBenchConn(conn); + errorPrint("\ncreate database %s failed!\n\n", + database->dbName); + return -1; +#ifdef WEBSOCKET + } +#endif + } + infoPrint("command to create database: <%s>\n", command); + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (database->superTbls) { + if (g_arguments->nthreads_auto) { + int32_t vgroups = getVgroupsOfDb(conn, database); + if (vgroups <=0) { + closeBenchConn(conn); + errorPrint("Database %s's vgroups is %d\n", + database->dbName, vgroups); + return -1; + } + } + } +#endif // TD_VER_COMPATIBLE_3_0_0_0 + + closeBenchConn(conn); + return 0; +} + +int createDatabase(SDataBase* database) { + int ret = 0; + if (REST_IFACE == g_arguments->iface) { + ret = createDatabaseRest(database); + } else { + ret = createDatabaseTaosc(database); + } +#if 0 +#ifdef LINUX + infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); + sleep(10); + infoPrint("%s() LN%d, ret: %d\n", __func__, __LINE__, ret); +#elif defined(DARWIN) + sleep(2); +#else + Sleep(2); +#endif +#endif + + return ret; +} + +static int generateChildTblName(int len, char *buffer, SDataBase *database, + SSuperTable *stbInfo, uint64_t i, + char *ttl) { + if (0 == len) { + memset(buffer, 0, TSDB_MAX_ALLOWED_SQL_LEN); + len += snprintf(buffer + len, + TSDB_MAX_ALLOWED_SQL_LEN - len, "CREATE TABLE "); + } + + len += snprintf( + buffer + len, TSDB_MAX_ALLOWED_SQL_LEN - len, + g_arguments->escape_character + ? "`%s`.`%s%" PRIu64 "` USING `%s`.`%s` TAGS (%s) %s " + : "%s.%s%" PRIu64 " USING %s.%s TAGS (%s) %s ", + database->dbName, stbInfo->childTblPrefix, i, database->dbName, + stbInfo->stbName, + stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); + + return len; +} + +static int getBatchOfTblCreating(threadInfo *pThreadInfo, + SSuperTable *stbInfo) { + BArray *batchArray = stbInfo->batchTblCreatingNumbersArray; + if (batchArray) { + int *batch = benchArrayGet( + batchArray, pThreadInfo->posOfTblCreatingBatch); + pThreadInfo->posOfTblCreatingBatch++; + if (pThreadInfo->posOfTblCreatingBatch == batchArray->size) { + pThreadInfo->posOfTblCreatingBatch = 0; + } + return *batch; + } + return 0; +} + +static int getIntervalOfTblCreating(threadInfo *pThreadInfo, + SSuperTable *stbInfo) { + BArray *intervalArray = stbInfo->batchTblCreatingIntervalsArray; + if (intervalArray) { + int *interval = benchArrayGet( + intervalArray, pThreadInfo->posOfTblCreatingInterval); + pThreadInfo->posOfTblCreatingInterval++; + if (pThreadInfo->posOfTblCreatingInterval == intervalArray->size) { + pThreadInfo->posOfTblCreatingInterval = 0; + } + return *interval; + } + return 0; +} + +static void *createTable(void *sarg) { + if (g_arguments->supplementInsert) { + return NULL; + } + + threadInfo * pThreadInfo = (threadInfo *)sarg; + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; +#ifdef LINUX + prctl(PR_SET_NAME, "createTable"); +#endif + uint64_t lastPrintTime = toolsGetTimestampMs(); + pThreadInfo->buffer = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + int len = 0; + int batchNum = 0; + infoPrint( + "thread[%d] start creating table from %" PRIu64 " to %" PRIu64 + "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to); + + char ttl[SMALL_BUFF_LEN] = ""; + if (stbInfo->ttl != 0) { + snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); + } + + int smallBatchCount = 0; + for (uint64_t i = pThreadInfo->start_table_from + stbInfo->childTblFrom; + (i <= (pThreadInfo->end_table_to + stbInfo->childTblFrom) + && !g_arguments->terminate); i++) { + if (g_arguments->terminate) { + goto create_table_end; + } + if (!stbInfo->use_metric || stbInfo->tags->size == 0) { + if (stbInfo->childTblCount == 1) { + snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "CREATE TABLE `%s`.`%s` %s;" + : "CREATE TABLE %s.%s %s;", + database->dbName, stbInfo->stbName, + stbInfo->colsOfCreateChildTable); + } else { + snprintf(pThreadInfo->buffer, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "CREATE TABLE `%s`.`%s` %s;" + : "CREATE TABLE %s.%s %s;", + database->dbName, + stbInfo->childTblArray[i]->name, + stbInfo->colsOfCreateChildTable); + } + batchNum++; + } else { + if (0 == len) { + batchNum = 0; + } + len = generateChildTblName(len, pThreadInfo->buffer, + database, stbInfo, i, ttl); + + batchNum++; + smallBatchCount++; + + int smallBatch = getBatchOfTblCreating(pThreadInfo, stbInfo); + if ((!smallBatch || (smallBatchCount == smallBatch)) + && (batchNum < stbInfo->batchTblCreatingNum) + && ((TSDB_MAX_ALLOWED_SQL_LEN - len) >= + (stbInfo->lenOfTags + EXTRA_SQL_LEN))) { + continue; + } else { + smallBatchCount = 0; + } + } + + len = 0; + + int ret = 0; + debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, + pThreadInfo->buffer); + if (REST_IFACE == stbInfo->iface) { + ret = queryDbExecRest(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + pThreadInfo->sockfd); + } else { + ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + int32_t trying = g_arguments->keep_trying; + while (ret && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then re-create " + "table %s\n", + g_arguments->trying_interval, pThreadInfo->buffer); + toolsMsleep(g_arguments->trying_interval); + ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + if (trying != -1) { + trying--; + } + } + } + + if (0 != ret) { + g_fail = true; + goto create_table_end; + } + uint64_t intervalOfTblCreating = getIntervalOfTblCreating(pThreadInfo, + stbInfo); + if (intervalOfTblCreating) { + debugPrint("will sleep %"PRIu64" milliseconds " + "for table creating interval\n", intervalOfTblCreating); + toolsMsleep(intervalOfTblCreating); + } + + pThreadInfo->tables_created += batchNum; + batchNum = 0; + uint64_t currentPrintTime = toolsGetTimestampMs(); + if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { + infoPrint( + "thread[%d] already created %" PRId64 " tables\n", + pThreadInfo->threadID, pThreadInfo->tables_created); + lastPrintTime = currentPrintTime; + } + } + + if (0 != len) { + int ret = 0; + debugPrint("thread[%d] creating table: %s\n", pThreadInfo->threadID, + pThreadInfo->buffer); + if (REST_IFACE == stbInfo->iface) { + ret = queryDbExecRest(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + pThreadInfo->sockfd); + } else { + ret = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + } + if (0 != ret) { + g_fail = true; + goto create_table_end; + } + pThreadInfo->tables_created += batchNum; + debugPrint("thread[%d] already created %" PRId64 " tables\n", + pThreadInfo->threadID, pThreadInfo->tables_created); + } +create_table_end: + tmfree(pThreadInfo->buffer); + pThreadInfo->buffer = NULL; + return NULL; +} + +static int startMultiThreadCreateChildTable( + SDataBase* database, SSuperTable* stbInfo) { + int code = -1; + int threads = g_arguments->table_threads; + int64_t ntables; + if (stbInfo->childTblTo > 0) { + ntables = stbInfo->childTblTo - stbInfo->childTblFrom; + } else { + ntables = stbInfo->childTblCount; + } + pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), false); + threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), false); + uint64_t tableFrom = 0; + if (threads < 1) { + threads = 1; + } + + int64_t a = ntables / threads; + if (a < 1) { + threads = (int)ntables; + a = 1; + } + + if (ntables == 0) { + errorPrint("failed to create child table, childTblCount: %"PRId64"\n", + ntables); + goto over; + } + int64_t b = ntables % threads; + + for (uint32_t i = 0; (i < threads && !g_arguments->terminate); i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + pThreadInfo->stbInfo = stbInfo; + pThreadInfo->dbInfo = database; + if (REST_IFACE == stbInfo->iface) { + int sockfd = createSockFd(); + if (sockfd < 0) { + FREE_PIDS_INFOS_RETURN_MINUS_1(); + } + pThreadInfo->sockfd = sockfd; + } else { + pThreadInfo->conn = initBenchConn(); + if (NULL == pThreadInfo->conn) { + goto over; + } + } + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = i < b ? a + 1 : a; + pThreadInfo->end_table_to = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pThreadInfo->end_table_to + 1; + pThreadInfo->tables_created = 0; + pthread_create(pids + i, NULL, createTable, pThreadInfo); + } + + for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + pthread_join(pids[i], NULL); + } + + if (g_arguments->terminate) toolsMsleep(100); + + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + g_arguments->actualChildTables += pThreadInfo->tables_created; + + if ((REST_IFACE != stbInfo->iface) && pThreadInfo->conn) { + closeBenchConn(pThreadInfo->conn); + } + } + + if (g_fail) { + goto over; + } + code = 0; +over: + free(pids); + free(infos); + return code; +} + +static int createChildTables() { + int32_t code; + infoPrint("start creating %" PRId64 " table(s) with %d thread(s)\n", + g_arguments->totalChildTables, g_arguments->table_threads); + if (g_arguments->fpOfInsertResult) { + infoPrintToFile(g_arguments->fpOfInsertResult, + "start creating %" PRId64 " table(s) with %d thread(s)\n", + g_arguments->totalChildTables, g_arguments->table_threads); + } + double start = (double)toolsGetTimestampMs(); + + for (int i = 0; (i < g_arguments->databases->size + && !g_arguments->terminate); i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (int j = 0; (j < database->superTbls->size + && !g_arguments->terminate); j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + if (stbInfo->autoTblCreating || stbInfo->iface == SML_IFACE + || stbInfo->iface == SML_REST_IFACE) { + g_arguments->autoCreatedChildTables += + stbInfo->childTblCount; + continue; + } + if (stbInfo->childTblExists) { + g_arguments->existedChildTables += + stbInfo->childTblCount; + continue; + } + debugPrint("colsOfCreateChildTable: %s\n", + stbInfo->colsOfCreateChildTable); + + code = startMultiThreadCreateChildTable(database, stbInfo); + if (code && !g_arguments->terminate) { + return code; + } + } + } + } + + double end = (double)toolsGetTimestampMs(); + succPrint( + "Spent %.4f seconds to create %" PRId64 + " table(s) with %d thread(s), already exist %" PRId64 + " table(s), actual %" PRId64 " table(s) pre created, %" PRId64 + " table(s) will be auto created\n", + (end - start) / 1000.0, g_arguments->totalChildTables, + g_arguments->table_threads, g_arguments->existedChildTables, + g_arguments->actualChildTables, + g_arguments->autoCreatedChildTables); + return 0; +} + +static void freeChildTable(SChildTable *childTbl, int colsSize) { + if (childTbl->useOwnSample) { + if (childTbl->childCols) { + for (int col = 0; col < colsSize; col++) { + ChildField *childCol = + benchArrayGet(childTbl->childCols, col); + if (childCol) { + tmfree(childCol->stmtData.data); + tmfree(childCol->stmtData.is_null); + } + } + benchArrayDestroy(childTbl->childCols); + } + tmfree(childTbl->sampleDataBuf); + } + tmfree(childTbl); +} + +void postFreeResource() { + if (!g_arguments->terminate) { + tmfclose(g_arguments->fpOfInsertResult); + } + + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->cfgs) { + for (int c = 0; c < database->cfgs->size; c++) { + SDbCfg *cfg = benchArrayGet(database->cfgs, c); + if ((NULL == root) && (0 == strcmp(cfg->name, "replica"))) { + tmfree(cfg->name); + cfg->name = NULL; + } + } + benchArrayDestroy(database->cfgs); + } + if (database->superTbls) { + for (uint64_t j = 0; j < database->superTbls->size; j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + tmfree(stbInfo->colsOfCreateChildTable); + stbInfo->colsOfCreateChildTable = NULL; + tmfree(stbInfo->sampleDataBuf); + stbInfo->sampleDataBuf = NULL; + tmfree(stbInfo->tagDataBuf); + stbInfo->tagDataBuf = NULL; + tmfree(stbInfo->partialColNameBuf); + stbInfo->partialColNameBuf = NULL; + benchArrayDestroy(stbInfo->batchTblCreatingNumbersArray); + benchArrayDestroy(stbInfo->batchTblCreatingIntervalsArray); + for (int k = 0; k < stbInfo->tags->size; k++) { + Field * tag = benchArrayGet(stbInfo->tags, k); + tmfree(tag->stmtData.data); + tag->stmtData.data = NULL; + } + benchArrayDestroy(stbInfo->tags); + + for (int k = 0; k < stbInfo->cols->size; k++) { + Field * col = benchArrayGet(stbInfo->cols, k); + tmfree(col->stmtData.data); + col->stmtData.data = NULL; + tmfree(col->stmtData.is_null); + col->stmtData.is_null = NULL; + } + if (g_arguments->test_mode == INSERT_TEST) { + if (stbInfo->childTblArray) { + for (int64_t child = 0; child < stbInfo->childTblCount; + child++) { + SChildTable *childTbl = + stbInfo->childTblArray[child]; + if (childTbl) { + freeChildTable(childTbl, stbInfo->cols->size); + } + } + } + } + benchArrayDestroy(stbInfo->cols); + tmfree(stbInfo->childTblArray); + stbInfo->childTblArray = NULL; + benchArrayDestroy(stbInfo->tsmas); +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if ((0 == stbInfo->interlaceRows) + && (g_arguments->nthreads_auto)) { + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + tmfree(vg->childTblArray); + vg->childTblArray = NULL; + } + } +#endif // TD_VER_COMPATIBLE_3_0_0_0 + } +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (database->vgArray) + benchArrayDestroy(database->vgArray); +#endif // TD_VER_COMPATIBLE_3_0_0_0 + benchArrayDestroy(database->superTbls); + } + } + benchArrayDestroy(g_arguments->databases); + benchArrayDestroy(g_arguments->streams); + tools_cJSON_Delete(root); +} + +int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + TAOS_RES * res = NULL; + int32_t code = 0; + uint16_t iface = stbInfo->iface; + + int32_t trying = (stbInfo->keep_trying)? + stbInfo->keep_trying:g_arguments->keep_trying; + int32_t trying_interval = stbInfo->trying_interval? + stbInfo->trying_interval:g_arguments->trying_interval; + int protocol = stbInfo->lineProtocol; + + switch (iface) { + case TAOSC_IFACE: + debugPrint("buffer: %s\n", pThreadInfo->buffer); + code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + while (code && trying && !g_arguments->terminate) { + infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", + trying_interval); + toolsMsleep(trying_interval); + code = queryDbExecCall(pThreadInfo->conn, pThreadInfo->buffer); + if (trying != -1) { + trying--; + } + } + break; + + case REST_IFACE: + debugPrint("buffer: %s\n", pThreadInfo->buffer); + code = postProceSql(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, + pThreadInfo->filePath); + while (code && trying && !g_arguments->terminate) { + infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", + trying_interval); + toolsMsleep(trying_interval); + code = postProceSql(pThreadInfo->buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, + pThreadInfo->filePath); + if (trying != -1) { + trying--; + } + } + break; + + case STMT_IFACE: + code = taos_stmt_execute(pThreadInfo->conn->stmt); + if (code) { + errorPrint( + "failed to execute insert statement. reason: %s\n", + taos_stmt_errstr(pThreadInfo->conn->stmt)); + code = -1; + } + break; + + case SML_IFACE: + res = taos_schemaless_insert( + pThreadInfo->conn->taos, pThreadInfo->lines, + (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) + ? 0 : k, + (SML_JSON_TAOS_FORMAT == protocol) + ? TSDB_SML_JSON_PROTOCOL : protocol, + (TSDB_SML_LINE_PROTOCOL == protocol) + ? database->sml_precision + : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(res); + trying = stbInfo->keep_trying; + while (code && trying && !g_arguments->terminate) { + taos_free_result(res); + infoPrint("will sleep %"PRIu32" milliseconds then re-insert\n", + trying_interval); + toolsMsleep(trying_interval); + res = taos_schemaless_insert( + pThreadInfo->conn->taos, pThreadInfo->lines, + (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) + ? 0 : k, + (SML_JSON_TAOS_FORMAT == protocol) + ? TSDB_SML_JSON_PROTOCOL : protocol, + (TSDB_SML_LINE_PROTOCOL == protocol) + ? database->sml_precision + : TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(res); + if (trying != -1) { + trying--; + } + } + + if (code != TSDB_CODE_SUCCESS && !g_arguments->terminate) { + debugPrint("Failed to execute " + "schemaless insert content: %s\n\n", + pThreadInfo->lines?(pThreadInfo->lines[0]? + pThreadInfo->lines[0]:""):""); + errorPrint( + "failed to execute schemaless insert. " + "code: 0x%08x reason: %s\n\n", + code, taos_errstr(res)); + } + taos_free_result(res); + break; + + case SML_REST_IFACE: { + if (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) { + code = postProceSql(pThreadInfo->lines[0], database->dbName, + database->precision, stbInfo->iface, + protocol, g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, pThreadInfo->filePath); + } else { + int len = 0; + for (int i = 0; i < k; i++) { + if (strlen(pThreadInfo->lines[i]) != 0) { + int n; + if (TSDB_SML_TELNET_PROTOCOL == protocol + && stbInfo->tcpTransfer) { + n = snprintf(pThreadInfo->buffer + len, + TSDB_MAX_ALLOWED_SQL_LEN - len, + "put %s\n", pThreadInfo->lines[i]); + } else { + n = snprintf(pThreadInfo->buffer + len, + TSDB_MAX_ALLOWED_SQL_LEN - len, + "%s\n", + pThreadInfo->lines[i]); + } + if (n < 0 || n >= TSDB_MAX_ALLOWED_SQL_LEN - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, i); + break; + } else { + len += n; + } + } else { + break; + } + } + if (g_arguments->terminate) { + break; + } + code = postProceSql(pThreadInfo->buffer, database->dbName, + database->precision, + stbInfo->iface, protocol, + g_arguments->port, + stbInfo->tcpTransfer, + pThreadInfo->sockfd, pThreadInfo->filePath); + } + break; + } + } + return code; +} + +static int smartContinueIfFail(threadInfo *pThreadInfo, + SChildTable *childTbl, + int64_t i, + char *ttl) { + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + char *buffer = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + snprintf( + buffer, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character ? + "CREATE TABLE `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s " + : "CREATE TABLE %s.%s USING %s.%s TAGS (%s) %s ", + database->dbName, childTbl->name, database->dbName, + stbInfo->stbName, + stbInfo->tagDataBuf + i * stbInfo->lenOfTags, ttl); + debugPrint("creating table: %s\n", buffer); + int ret; + if (REST_IFACE == stbInfo->iface) { + ret = queryDbExecRest(buffer, + database->dbName, + database->precision, + stbInfo->iface, + stbInfo->lineProtocol, + stbInfo->tcpTransfer, + pThreadInfo->sockfd); + } else { + ret = queryDbExecCall(pThreadInfo->conn, buffer); + int32_t trying = g_arguments->keep_trying; + while (ret && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then " + "re-create table %s\n", + g_arguments->trying_interval, buffer); + toolsMsleep(g_arguments->trying_interval); + ret = queryDbExecCall(pThreadInfo->conn, buffer); + if (trying != -1) { + trying--; + } + } + } + tmfree(buffer); + + return ret; +} + +static void cleanupAndPrint(threadInfo *pThreadInfo, char *mode) { + if (pThreadInfo) { + if (pThreadInfo->json_array) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + if (0 == pThreadInfo->totalDelay) { + pThreadInfo->totalDelay = 1; + } + succPrint( + "thread[%d] %s mode, completed total inserted rows: %" PRIu64 + ", %.2f records/second\n", + pThreadInfo->threadID, + mode, + pThreadInfo->totalInsertRows, + (double)(pThreadInfo->totalInsertRows / + ((double)pThreadInfo->totalDelay / 1E6))); + } +} + +static int64_t getDisorderTs(SSuperTable *stbInfo, int *disorderRange) { + int64_t disorderTs = 0; + int64_t startTimestamp = stbInfo->startTimestamp; + if (stbInfo->disorderRatio > 0) { + int rand_num = taosRandom() % 100; + if (rand_num < stbInfo->disorderRatio) { + (*disorderRange)--; + if (0 == *disorderRange) { + *disorderRange = stbInfo->disorderRange; + } + disorderTs = startTimestamp - *disorderRange; + debugPrint("rand_num: %d, < disorderRatio: %d, " + "disorderTs: %"PRId64"\n", + rand_num, stbInfo->disorderRatio, + disorderTs); + } + } + return disorderTs; +} + +static void *syncWriteInterlace(void *sarg) { + threadInfo * pThreadInfo = (threadInfo *)sarg; + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + infoPrint( + "thread[%d] start interlace inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to); + + int64_t insertRows = stbInfo->insertRows; + int32_t interlaceRows = stbInfo->interlaceRows; + int64_t pos = 0; + uint32_t batchPerTblTimes = g_arguments->reqPerReq / interlaceRows; + uint64_t lastPrintTime = toolsGetTimestampMs(); + int64_t startTs = toolsGetTimestampUs(); + int64_t endTs; + uint64_t tableSeq = pThreadInfo->start_table_from; + int disorderRange = stbInfo->disorderRange; + + while (insertRows > 0) { + int64_t tmp_total_insert_rows = 0; + uint32_t generated = 0; + if (insertRows <= interlaceRows) { + interlaceRows = insertRows; + } + for (int i = 0; i < batchPerTblTimes; i++) { + if (g_arguments->terminate) { + goto free_of_interlace; + } + int64_t timestamp = pThreadInfo->start_time; + SChildTable *childTbl = stbInfo->childTblArray[tableSeq]; + char * tableName = + stbInfo->childTblArray[tableSeq]->name; + char *sampleDataBuf = childTbl->useOwnSample? + childTbl->sampleDataBuf: + stbInfo->sampleDataBuf; + char ttl[SMALL_BUFF_LEN] = ""; + if (stbInfo->ttl != 0) { + snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); + } + switch (stbInfo->iface) { + case REST_IFACE: + case TAOSC_IFACE: { + char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; + if (g_arguments->escape_character) { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "`%s`", + tableName); + } else { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, "%s", + tableName); + } + if (i == 0) { + ds_add_str(&pThreadInfo->buffer, STR_INSERT_INTO); + } + if (stbInfo->partialColNum == stbInfo->cols->size) { + if (stbInfo->autoTblCreating) { + ds_add_strs(&pThreadInfo->buffer, 8, + escapedTbName, + " USING `", + stbInfo->stbName, + "` TAGS (", + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, + ") ", ttl, " VALUES "); + } else { + ds_add_strs(&pThreadInfo->buffer, 2, + escapedTbName, " VALUES "); + } + } else { + if (stbInfo->autoTblCreating) { + ds_add_strs(&pThreadInfo->buffer, 10, + escapedTbName, + " (", + stbInfo->partialColNameBuf, + ") USING `", + stbInfo->stbName, + "` TAGS (", + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, + ") ", ttl, " VALUES "); + } else { + ds_add_strs(&pThreadInfo->buffer, 4, + escapedTbName, + "(", + stbInfo->partialColNameBuf, + ") VALUES "); + } + } + + for (int64_t j = 0; j < interlaceRows; j++) { + int64_t disorderTs = getDisorderTs(stbInfo, + &disorderRange); + char time_string[BIGINT_BUFF_LEN]; + snprintf(time_string, BIGINT_BUFF_LEN, "%"PRId64"", + disorderTs?disorderTs:timestamp); + ds_add_strs(&pThreadInfo->buffer, 5, + "(", + time_string, + ",", + sampleDataBuf + pos * stbInfo->lenOfCols, + ") "); + if (ds_len(pThreadInfo->buffer) + > stbInfo->max_sql_len) { + errorPrint("sql buffer length (%"PRIu64") " + "is larger than max sql length " + "(%"PRId64")\n", + ds_len(pThreadInfo->buffer), + stbInfo->max_sql_len); + goto free_of_interlace; + } + generated++; + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + timestamp += stbInfo->timestamp_step; + } + break; + } + case STMT_IFACE: { + char escapedTbName[TSDB_TABLE_NAME_LEN+2] = "\0"; + if (g_arguments->escape_character) { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN+2, + "`%s`", tableName); + } else { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", + tableName); + } + if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, + escapedTbName)) { + errorPrint( + "taos_stmt_set_tbname(%s) failed, reason: %s\n", + tableName, + taos_stmt_errstr(pThreadInfo->conn->stmt)); + g_fail = true; + goto free_of_interlace; + } + generated = + bindParamBatch(pThreadInfo, interlaceRows, + timestamp, childTbl); + break; + } + case SML_REST_IFACE: + case SML_IFACE: { + int protocol = stbInfo->lineProtocol; + for (int64_t j = 0; j < interlaceRows; j++) { + int64_t disorderTs = getDisorderTs(stbInfo, + &disorderRange); + if (TSDB_SML_JSON_PROTOCOL == protocol) { + tools_cJSON *tag = tools_cJSON_Duplicate( + tools_cJSON_GetArrayItem( + pThreadInfo->sml_json_tags, + (int)tableSeq - + pThreadInfo->start_table_from), + true); + generateSmlJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, + disorderTs?disorderTs:timestamp); + } else if (SML_JSON_TAOS_FORMAT == protocol) { + tools_cJSON *tag = tools_cJSON_Duplicate( + tools_cJSON_GetArrayItem( + pThreadInfo->sml_json_tags, + (int)tableSeq - + pThreadInfo->start_table_from), + true); + generateSmlTaosJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, + disorderTs?disorderTs:timestamp); + } else if (TSDB_SML_LINE_PROTOCOL == protocol) { + snprintf( + pThreadInfo->lines[generated], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %s %" PRId64 "", + pThreadInfo + ->sml_tags[(int)tableSeq - + pThreadInfo->start_table_from], + sampleDataBuf + pos * stbInfo->lenOfCols, + disorderTs?disorderTs:timestamp); + } else { + snprintf( + pThreadInfo->lines[generated], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %" PRId64 " %s %s", stbInfo->stbName, + disorderTs?disorderTs:timestamp, + sampleDataBuf + pos * stbInfo->lenOfCols, + pThreadInfo + ->sml_tags[(int)tableSeq - + pThreadInfo->start_table_from]); + } + generated++; + timestamp += stbInfo->timestamp_step; + } + if (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) { + pThreadInfo->lines[0] = + tools_cJSON_PrintUnformatted( + pThreadInfo->json_array); + } + break; + } + } + tableSeq++; + tmp_total_insert_rows += interlaceRows; + if (tableSeq > pThreadInfo->end_table_to) { + tableSeq = pThreadInfo->start_table_from; + pThreadInfo->start_time += + interlaceRows * stbInfo->timestamp_step; + if (!stbInfo->non_stop) { + insertRows -= interlaceRows; + } + if (stbInfo->insert_interval > 0) { + debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", + __func__, __LINE__, stbInfo->insert_interval); + perfPrint("sleep %" PRIu64 " ms\n", + stbInfo->insert_interval); + toolsMsleep((int32_t)stbInfo->insert_interval); + } + break; + } + } + + startTs = toolsGetTimestampUs(); + if (execInsert(pThreadInfo, generated)) { + g_fail = true; + goto free_of_interlace; + } + endTs = toolsGetTimestampUs(); + + pThreadInfo->totalInsertRows += tmp_total_insert_rows; + + if (g_arguments->terminate) { + goto free_of_interlace; + } + + int protocol = stbInfo->lineProtocol; + switch (stbInfo->iface) { + case TAOSC_IFACE: + case REST_IFACE: + debugPrint("pThreadInfo->buffer: %s\n", + pThreadInfo->buffer); + free_ds(&pThreadInfo->buffer); + pThreadInfo->buffer = new_ds(0); + break; + case SML_REST_IFACE: + memset(pThreadInfo->buffer, 0, + g_arguments->reqPerReq * (pThreadInfo->max_sql_len + 1)); + case SML_IFACE: + if (TSDB_SML_JSON_PROTOCOL == protocol + || SML_JSON_TAOS_FORMAT == protocol) { + debugPrint("pThreadInfo->lines[0]: %s\n", + pThreadInfo->lines[0]); + if (pThreadInfo->json_array && !g_arguments->terminate) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + pThreadInfo->json_array = tools_cJSON_CreateArray(); + if (pThreadInfo->lines && pThreadInfo->lines[0]) { + tmfree(pThreadInfo->lines[0]); + pThreadInfo->lines[0] = NULL; + } + } else { + for (int j = 0; j < generated; j++) { + if (pThreadInfo && pThreadInfo->lines + && !g_arguments->terminate) { + debugPrint("pThreadInfo->lines[%d]: %s\n", j, + pThreadInfo->lines[j]); + memset(pThreadInfo->lines[j], 0, + pThreadInfo->max_sql_len); + } + } + } + break; + case STMT_IFACE: + break; + } + + int64_t delay = endTs - startTs; + if (delay <=0) { + debugPrint("thread[%d]: startTS: %"PRId64", endTS: %"PRId64"\n", + pThreadInfo->threadID, startTs, endTs); + } else { + perfPrint("insert execution time is %10.2f ms\n", + delay / 1E6); + + int64_t * pdelay = benchCalloc(1, sizeof(int64_t), false); + *pdelay = delay; + if (benchArrayPush(pThreadInfo->delayList, pdelay) == NULL) { + tmfree(pdelay); + } + pThreadInfo->totalDelay += delay; + } + + int64_t currentPrintTime = toolsGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30 * 1000) { + infoPrint( + "thread[%d] has currently inserted rows: %" PRIu64 + "\n", + pThreadInfo->threadID, pThreadInfo->totalInsertRows); + lastPrintTime = currentPrintTime; + } + } +free_of_interlace: + cleanupAndPrint(pThreadInfo, "interlace"); + return NULL; +} + +static int32_t prepareProgressDataStmt( + threadInfo *pThreadInfo, + SChildTable *childTbl, + int64_t *timestamp, uint64_t i, char *ttl) { + SSuperTable *stbInfo = pThreadInfo->stbInfo; + char escapedTbName[TSDB_TABLE_NAME_LEN + 2] = "\0"; + if (g_arguments->escape_character) { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN + 2, + "`%s`", childTbl->name); + } else { + snprintf(escapedTbName, TSDB_TABLE_NAME_LEN, "%s", + childTbl->name); + } + if (taos_stmt_set_tbname(pThreadInfo->conn->stmt, + escapedTbName)) { + errorPrint( + "taos_stmt_set_tbname(%s) failed," + "reason: %s\n", escapedTbName, + taos_stmt_errstr(pThreadInfo->conn->stmt)); + return -1; + } + int32_t generated = bindParamBatch( + pThreadInfo, + (g_arguments->reqPerReq > (stbInfo->insertRows - i)) + ? (stbInfo->insertRows - i) + : g_arguments->reqPerReq, + *timestamp, childTbl); + *timestamp += generated * stbInfo->timestamp_step; + return generated; +} + +static void makeTimestampDisorder( + int64_t *timestamp, SSuperTable *stbInfo) { + int64_t startTimestamp = stbInfo->startTimestamp; + int disorderRange = stbInfo->disorderRange; + int rand_num = taosRandom() % 100; + if (rand_num < stbInfo->disorderRatio) { + disorderRange--; + if (0 == disorderRange) { + disorderRange = stbInfo->disorderRange; + } + *timestamp = startTimestamp - disorderRange; + debugPrint("rand_num: %d, < disorderRatio: %d" + ", ts: %"PRId64"\n", + rand_num, + stbInfo->disorderRatio, + *timestamp); + } +} + +static int32_t prepareProgressDataSmlJsonText( + threadInfo *pThreadInfo, + uint64_t tableSeq, + int64_t *timestamp, uint64_t i, char *ttl) { + // prepareProgressDataSmlJsonText + SSuperTable *stbInfo = pThreadInfo->stbInfo; + int32_t generated = 0; + + int len = 0; + + char *line = pThreadInfo->lines[0]; + uint32_t line_buf_len = pThreadInfo->line_buf_len; + + strncat(line + len, "[", 2); + len += 1; + + int32_t pos = 0; + for (int j = 0; (j < g_arguments->reqPerReq) + && !g_arguments->terminate; j++) { + strncat(line + len, "{", 2); + len += 1; + int n; + n = snprintf(line + len, line_buf_len - len, + "\"timestamp\":%"PRId64",", *timestamp); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + + n = snprintf(line + len, line_buf_len - len, "%s", + pThreadInfo->sml_json_value_array[tableSeq]); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + n = snprintf(line + len, line_buf_len - len, "\"tags\":%s,", + pThreadInfo->sml_tags_json_array[tableSeq]); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + n = snprintf(line + len, line_buf_len - len, + "\"metric\":\"%s\"}", stbInfo->stbName); + if (n < 0 || n >= line_buf_len - len) { + errorPrint("%s() LN%d snprintf overflow on %d\n", + __func__, __LINE__, j); + return -1; + } else { + len += n; + } + + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + *timestamp += stbInfo->timestamp_step; + if (stbInfo->disorderRatio > 0) { + makeTimestampDisorder(timestamp, stbInfo); + } + generated++; + if (i + generated >= stbInfo->insertRows) { + break; + } + if ((j+1) < g_arguments->reqPerReq) { + strncat(line + len, ",", 2); + len += 1; + } + } + strncat(line + len, "]", 2); + + debugPrint("%s() LN%d, lines[0]: %s\n", + __func__, __LINE__, pThreadInfo->lines[0]); + return generated; +} + +static int32_t prepareProgressDataSmlJson( + threadInfo *pThreadInfo, + uint64_t tableSeq, + int64_t *timestamp, uint64_t i, char *ttl) { + // prepareProgressDataSmlJson + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + int32_t generated = 0; + + int32_t pos = 0; + int protocol = stbInfo->lineProtocol; + for (int j = 0; (j < g_arguments->reqPerReq) + && !g_arguments->terminate; j++) { + tools_cJSON *tag = tools_cJSON_Duplicate( + tools_cJSON_GetArrayItem( + pThreadInfo->sml_json_tags, + (int)tableSeq - + pThreadInfo->start_table_from), + true); + debugPrintJsonNoTime(tag); + if (TSDB_SML_JSON_PROTOCOL == protocol) { + generateSmlJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, *timestamp); + } else { + generateSmlTaosJsonCols( + pThreadInfo->json_array, tag, stbInfo, + database->sml_precision, *timestamp); + } + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + *timestamp += stbInfo->timestamp_step; + if (stbInfo->disorderRatio > 0) { + makeTimestampDisorder(timestamp, stbInfo); + } + generated++; + if (i + generated >= stbInfo->insertRows) { + break; + } + } + + tmfree(pThreadInfo->lines[0]); + pThreadInfo->lines[0] = NULL; + pThreadInfo->lines[0] = + tools_cJSON_PrintUnformatted( + pThreadInfo->json_array); + debugPrint("pThreadInfo->lines[0]: %s\n", + pThreadInfo->lines[0]); + + return generated; +} + +static int32_t prepareProgressDataSmlLineOrTelnet( + threadInfo *pThreadInfo, uint64_t tableSeq, char *sampleDataBuf, + int64_t *timestamp, uint64_t i, char *ttl, int protocol) { + // prepareProgressDataSmlLine + SSuperTable *stbInfo = pThreadInfo->stbInfo; + int32_t generated = 0; + + int32_t pos = 0; + for (int j = 0; (j < g_arguments->reqPerReq) + && !g_arguments->terminate; j++) { + if (TSDB_SML_LINE_PROTOCOL == protocol) { + snprintf( + pThreadInfo->lines[j], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %s %" PRId64 "", + pThreadInfo->sml_tags[tableSeq + - pThreadInfo->start_table_from], + sampleDataBuf + pos * stbInfo->lenOfCols, + *timestamp); + } else { + snprintf( + pThreadInfo->lines[j], + stbInfo->lenOfCols + stbInfo->lenOfTags, + "%s %" PRId64 " %s %s", stbInfo->stbName, + *timestamp, + sampleDataBuf + + pos * stbInfo->lenOfCols, + pThreadInfo->sml_tags[tableSeq + -pThreadInfo->start_table_from]); + } + pos++; + if (pos >= g_arguments->prepared_rand) { + pos = 0; + } + *timestamp += stbInfo->timestamp_step; + if (stbInfo->disorderRatio > 0) { + makeTimestampDisorder(timestamp, stbInfo); + } + generated++; + if (i + generated >= stbInfo->insertRows) { + break; + } + } + return generated; +} + +static int32_t prepareProgressDataSml( + threadInfo *pThreadInfo, + SChildTable *childTbl, + uint64_t tableSeq, + int64_t *timestamp, uint64_t i, char *ttl) { + // prepareProgressDataSml + SSuperTable *stbInfo = pThreadInfo->stbInfo; + + char *sampleDataBuf; + if (childTbl->useOwnSample) { + sampleDataBuf = childTbl->sampleDataBuf; + } else { + sampleDataBuf = stbInfo->sampleDataBuf; + } + int protocol = stbInfo->lineProtocol; + int32_t generated = -1; + switch (protocol) { + case TSDB_SML_LINE_PROTOCOL: + case TSDB_SML_TELNET_PROTOCOL: + generated = prepareProgressDataSmlLineOrTelnet( + pThreadInfo, + tableSeq, + sampleDataBuf, + timestamp, i, ttl, protocol); + break; + case TSDB_SML_JSON_PROTOCOL: + generated = prepareProgressDataSmlJsonText( + pThreadInfo, + tableSeq - pThreadInfo->start_table_from, + timestamp, i, ttl); + break; + case SML_JSON_TAOS_FORMAT: + generated = prepareProgressDataSmlJson( + pThreadInfo, + tableSeq, + timestamp, i, ttl); + break; + default: + errorPrint("%s() LN%d: unknown protcolor: %d\n", + __func__, __LINE__, protocol); + break; + } + + return generated; +} + +static int32_t prepareProgressDataSql( + threadInfo *pThreadInfo, + SChildTable *childTbl, uint64_t tableSeq, + char *sampleDataBuf, + int64_t *timestamp, uint64_t i, char *ttl, + int32_t *pos, uint64_t *len) { + // prepareProgressDataSql + int32_t generated = 0; + SDataBase *database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + char * pstr = pThreadInfo->buffer; + int disorderRange = stbInfo->disorderRange; + + if (stbInfo->partialColNum == stbInfo->cols->size) { + if (stbInfo->autoTblCreating) { + *len = + snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` USING `%s`.`%s` TAGS (%s) %s VALUES " + : "%s %s.%s USING %s.%s TAGS (%s) %s VALUES ", + STR_INSERT_INTO, database->dbName, + childTbl->name, database->dbName, + stbInfo->stbName, + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, ttl); + } else { + *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` VALUES " + : "%s %s.%s VALUES ", + STR_INSERT_INTO, + database->dbName, childTbl->name); + } + } else { + if (stbInfo->autoTblCreating) { + *len = snprintf( + pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` (%s) USING `%s`.`%s` TAGS (%s) %s VALUES " + : "%s %s.%s (%s) USING %s.%s TAGS (%s) %s VALUES ", + STR_INSERT_INTO, database->dbName, + childTbl->name, + stbInfo->partialColNameBuf, + database->dbName, stbInfo->stbName, + stbInfo->tagDataBuf + + stbInfo->lenOfTags * tableSeq, ttl); + } else { + *len = snprintf(pstr, TSDB_MAX_ALLOWED_SQL_LEN, + g_arguments->escape_character + ? "%s `%s`.`%s` (%s) VALUES " + : "%s %s.%s (%s) VALUES ", + STR_INSERT_INTO, database->dbName, + childTbl->name, + stbInfo->partialColNameBuf); + } + } + + char *ownSampleDataBuf; + if (childTbl->useOwnSample) { + debugPrint("%s is using own sample data\n", + childTbl->name); + ownSampleDataBuf = childTbl->sampleDataBuf; + } else { + ownSampleDataBuf = stbInfo->sampleDataBuf; + } + for (int j = 0; j < g_arguments->reqPerReq; j++) { + if (stbInfo->useSampleTs + && (!stbInfo->random_data_source)) { + *len += + snprintf(pstr + *len, + TSDB_MAX_ALLOWED_SQL_LEN - *len, "(%s)", + sampleDataBuf + + *pos * stbInfo->lenOfCols); + } else { + int64_t disorderTs = getDisorderTs(stbInfo, &disorderRange); + *len += snprintf(pstr + *len, + TSDB_MAX_ALLOWED_SQL_LEN - *len, + "(%" PRId64 ",%s)", + disorderTs?disorderTs:*timestamp, + ownSampleDataBuf + + *pos * stbInfo->lenOfCols); + } + *pos += 1; + if (*pos >= g_arguments->prepared_rand) { + *pos = 0; + } + *timestamp += stbInfo->timestamp_step; + generated++; + if (*len > (TSDB_MAX_ALLOWED_SQL_LEN + - stbInfo->lenOfCols)) { + break; + } + if (i + generated >= stbInfo->insertRows) { + break; + } + } + + return generated; +} + +void *syncWriteProgressive(void *sarg) { + threadInfo * pThreadInfo = (threadInfo *)sarg; + SDataBase * database = pThreadInfo->dbInfo; + SSuperTable *stbInfo = pThreadInfo->stbInfo; + + // special deal flow for TAOSC_IFACE + if (insertDataMix(pThreadInfo, database, stbInfo)) { + // request be dealt by this function , so return + return NULL; + } + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto) { + if (0 == pThreadInfo->vg->tbCountPerVgId) { + return NULL; + } + } else { + infoPrint( + "thread[%d] start progressive inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to + 1); + } +#else + infoPrint( + "thread[%d] start progressive inserting into table from " + "%" PRIu64 " to %" PRIu64 "\n", + pThreadInfo->threadID, pThreadInfo->start_table_from, + pThreadInfo->end_table_to + 1); +#endif + uint64_t lastPrintTime = toolsGetTimestampMs(); + int64_t startTs = toolsGetTimestampUs(); + int64_t endTs; + + for (uint64_t tableSeq = pThreadInfo->start_table_from; + tableSeq <= pThreadInfo->end_table_to; tableSeq++) { + char *sampleDataBuf; + SChildTable *childTbl; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if (g_arguments->nthreads_auto) { + childTbl = pThreadInfo->vg->childTblArray[tableSeq]; + } else { + childTbl = stbInfo->childTblArray[ + stbInfo->childTblExists? + tableSeq: + stbInfo->childTblFrom + tableSeq]; + } +#else + childTbl = stbInfo->childTblArray[ + stbInfo->childTblExists? + tableSeq: + stbInfo->childTblFrom + tableSeq]; +#endif + if (childTbl->useOwnSample) { + sampleDataBuf = childTbl->sampleDataBuf; + } else { + sampleDataBuf = stbInfo->sampleDataBuf; + } + + int64_t timestamp = pThreadInfo->start_time; + uint64_t len = 0; + int32_t pos = 0; + if (stbInfo->iface == STMT_IFACE && stbInfo->autoTblCreating) { + taos_stmt_close(pThreadInfo->conn->stmt); + pThreadInfo->conn->stmt = taos_stmt_init(pThreadInfo->conn->taos); + if (NULL == pThreadInfo->conn->stmt) { + errorPrint("taos_stmt_init() failed, reason: %s\n", + taos_errstr(NULL)); + g_fail = true; + goto free_of_progressive; + } + + if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, tableSeq)) { + g_fail = true; + goto free_of_progressive; + } + } + + char ttl[SMALL_BUFF_LEN] = ""; + if (stbInfo->ttl != 0) { + snprintf(ttl, SMALL_BUFF_LEN, "TTL %d", stbInfo->ttl); + } + for (uint64_t i = 0; i < stbInfo->insertRows;) { + if (g_arguments->terminate) { + goto free_of_progressive; + } + int32_t generated = 0; + switch (stbInfo->iface) { + case TAOSC_IFACE: + case REST_IFACE: + generated = prepareProgressDataSql( + pThreadInfo, + childTbl, + tableSeq, + sampleDataBuf, + ×tamp, i, ttl, &pos, &len); + break; + case STMT_IFACE: { + generated = prepareProgressDataStmt( + pThreadInfo, + childTbl, ×tamp, i, ttl); + break; + } + case SML_REST_IFACE: + case SML_IFACE: + generated = prepareProgressDataSml( + pThreadInfo, + childTbl, + tableSeq, ×tamp, i, ttl); + break; + default: + break; + } + if (generated < 0) { + g_fail = true; + goto free_of_progressive; + } + if (!stbInfo->non_stop) { + i += generated; + } + // only measure insert + startTs = toolsGetTimestampUs(); + int code = execInsert(pThreadInfo, generated); + if (code) { + if (NO_IF_FAILED == stbInfo->continueIfFail) { + warnPrint("The super table parameter " + "continueIfFail: %d, STOP insertion!\n", + stbInfo->continueIfFail); + g_fail = true; + goto free_of_progressive; + } else if (YES_IF_FAILED == stbInfo->continueIfFail) { + infoPrint("The super table parameter " + "continueIfFail: %d, " + "will continue to insert ..\n", + stbInfo->continueIfFail); + } else if (SMART_IF_FAILED == stbInfo->continueIfFail) { + warnPrint("The super table parameter " + "continueIfFail: %d, will create table " + "then insert ..\n", + stbInfo->continueIfFail); + int ret = smartContinueIfFail( + pThreadInfo, + childTbl, i, ttl); + if (0 != ret) { + g_fail = true; + goto free_of_progressive; + } + + code = execInsert(pThreadInfo, generated); + if (code) { + g_fail = true; + goto free_of_progressive; + } + } else { + warnPrint("Unknown super table parameter " + "continueIfFail: %d\n", + stbInfo->continueIfFail); + g_fail = true; + goto free_of_progressive; + } + } + endTs = toolsGetTimestampUs()+1; + + if (stbInfo->insert_interval > 0) { + debugPrint("%s() LN%d, insert_interval: %"PRIu64"\n", + __func__, __LINE__, stbInfo->insert_interval); + perfPrint("sleep %" PRIu64 " ms\n", + stbInfo->insert_interval); + toolsMsleep((int32_t)stbInfo->insert_interval); + } + + pThreadInfo->totalInsertRows += generated; + + if (g_arguments->terminate) { + goto free_of_progressive; + } + int protocol = stbInfo->lineProtocol; + switch (stbInfo->iface) { + case REST_IFACE: + case TAOSC_IFACE: + memset(pThreadInfo->buffer, 0, pThreadInfo->max_sql_len); + break; + case SML_REST_IFACE: + memset(pThreadInfo->buffer, 0, + g_arguments->reqPerReq * + (pThreadInfo->max_sql_len + 1)); + case SML_IFACE: + if (TSDB_SML_JSON_PROTOCOL == protocol) { + memset(pThreadInfo->lines[0], 0, + pThreadInfo->line_buf_len); + } else if (SML_JSON_TAOS_FORMAT == protocol) { + if (pThreadInfo->lines && pThreadInfo->lines[0]) { + tmfree(pThreadInfo->lines[0]); + pThreadInfo->lines[0] = NULL; + } + if (pThreadInfo->json_array) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + pThreadInfo->json_array = tools_cJSON_CreateArray(); + } else { + for (int j = 0; j < generated; j++) { + debugPrint("pThreadInfo->lines[%d]: %s\n", + j, pThreadInfo->lines[j]); + memset(pThreadInfo->lines[j], 0, + pThreadInfo->max_sql_len); + } + } + break; + case STMT_IFACE: + break; + } + + int64_t delay = endTs - startTs; + if (delay <= 0) { + debugPrint("thread[%d]: startTs: %"PRId64", endTs: %"PRId64"\n", + pThreadInfo->threadID, startTs, endTs); + } else { + perfPrint("insert execution time is %.6f s\n", + delay / 1E6); + + int64_t * pDelay = benchCalloc(1, sizeof(int64_t), false); + *pDelay = delay; + if (benchArrayPush(pThreadInfo->delayList, pDelay) == NULL) { + tmfree(pDelay); + } + pThreadInfo->totalDelay += delay; + } + + int64_t currentPrintTime = toolsGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30 * 1000) { + infoPrint( + "thread[%d] has currently inserted rows: " + "%" PRId64 "\n", + pThreadInfo->threadID, pThreadInfo->totalInsertRows); + lastPrintTime = currentPrintTime; + } + if (i >= stbInfo->insertRows) { + break; + } + } // insertRows + } // tableSeq +free_of_progressive: + cleanupAndPrint(pThreadInfo, "progressive"); + return NULL; +} + +static int initStmtDataValue(SSuperTable *stbInfo, SChildTable *childTbl) { + int32_t columnCount = stbInfo->cols->size; + + char *sampleDataBuf; + if (childTbl) { + sampleDataBuf = childTbl->sampleDataBuf; + } else { + sampleDataBuf = stbInfo->sampleDataBuf; + } + int64_t lenOfOneRow = stbInfo->lenOfCols; + + if (stbInfo->useSampleTs) { + columnCount += 1; // for skipping first column + } + for (int i=0; i < g_arguments->prepared_rand; i++) { + int cursor = 0; + + for (int c = 0; c < columnCount; c++) { + char *restStr = sampleDataBuf + + lenOfOneRow * i + cursor; + int lengthOfRest = strlen(restStr); + + int index = 0; + for (index = 0; index < lengthOfRest; index++) { + if (restStr[index] == ',') { + break; + } + } + + cursor += index + 1; // skip ',' too + if ((0 == c) && stbInfo->useSampleTs) { + continue; + } + + char *tmpStr = calloc(1, index + 1); + if (NULL == tmpStr) { + errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + __func__, __LINE__, index + 1); + return -1; + } + Field *col = benchArrayGet(stbInfo->cols, + (stbInfo->useSampleTs?c-1:c)); + char dataType = col->type; + + StmtData *stmtData; + if (childTbl) { + ChildField *childCol = + benchArrayGet(childTbl->childCols, + (stbInfo->useSampleTs?c-1:c)); + stmtData = &childCol->stmtData; + } else { + stmtData = &col->stmtData; + } + + strncpy(tmpStr, restStr, index); + + if (0 == strcmp(tmpStr, "NULL")) { + *(stmtData->is_null + i) = true; + } else { + switch (dataType) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *((int32_t*)stmtData->data + i) = atoi(tmpStr); + break; + case TSDB_DATA_TYPE_FLOAT: + *((float*)stmtData->data +i) = (float)atof(tmpStr); + break; + case TSDB_DATA_TYPE_DOUBLE: + *((double*)stmtData->data + i) = atof(tmpStr); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *((int16_t*)stmtData->data + i) = (int16_t)atoi(tmpStr); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); + break; + case TSDB_DATA_TYPE_BOOL: + *((int8_t*)stmtData->data + i) = (int8_t)atoi(tmpStr); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + *((int64_t*)stmtData->data + i) = (int64_t)atol(tmpStr); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + { + size_t tmpLen = strlen(tmpStr); + debugPrint("%s() LN%d, index: %d, " + "tmpStr len: %"PRIu64", col->length: %d\n", + __func__, __LINE__, + i, (uint64_t)tmpLen, col->length); + if (tmpLen-2 > col->length) { + errorPrint("data length %"PRIu64" " + "is larger than column length %d\n", + (uint64_t)tmpLen, col->length); + } + if (tmpLen > 2) { + strncpy((char *)stmtData->data + + i * col->length, + tmpStr+1, + min(col->length, tmpLen - 2)); + } else { + strncpy((char *)stmtData->data + + i*col->length, + "", 1); + } + } + break; + default: + break; + } + } + free(tmpStr); + } + } + return 0; +} + +static void initStmtData(char dataType, void **data, uint32_t length) { + char *tmpP = NULL; + + switch (dataType) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + tmpP = calloc(1, sizeof(int) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + tmpP = calloc(1, sizeof(int16_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_BOOL: + tmpP = calloc(1, sizeof(int8_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_FLOAT: + tmpP = calloc(1, sizeof(float) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_DOUBLE: + tmpP = calloc(1, sizeof(double) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + tmpP = calloc(1, g_arguments->prepared_rand * length); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + case TSDB_DATA_TYPE_TIMESTAMP: + tmpP = calloc(1, sizeof(int64_t) * g_arguments->prepared_rand); + assert(tmpP); + tmfree(*data); + *data = (void*)tmpP; + break; + + default: + errorPrint("Unknown data type: %s\n", + convertDatatypeToString(dataType)); + exit(EXIT_FAILURE); + } +} + +static int parseBufferToStmtBatchChildTbl(SSuperTable *stbInfo, + SChildTable* childTbl) { + int32_t columnCount = stbInfo->cols->size; + + for (int c = 0; c < columnCount; c++) { + Field *col = benchArrayGet(stbInfo->cols, c); + ChildField *childCol = benchArrayGet(childTbl->childCols, c); + char dataType = col->type; + + char *is_null = benchCalloc( + 1, sizeof(char) *g_arguments->prepared_rand, false); + + tmfree(childCol->stmtData.is_null); + childCol->stmtData.is_null = is_null; + + initStmtData(dataType, &(childCol->stmtData.data), col->length); + } + + return initStmtDataValue(stbInfo, childTbl); +} + +static int parseBufferToStmtBatch(SSuperTable* stbInfo) { + int32_t columnCount = stbInfo->cols->size; + + for (int c = 0; c < columnCount; c++) { + Field *col = benchArrayGet(stbInfo->cols, c); + char dataType = col->type; + + char *is_null = benchCalloc( + 1, sizeof(char) *g_arguments->prepared_rand, false); + tmfree(col->stmtData.is_null); + col->stmtData.is_null = is_null; + + initStmtData(dataType, &(col->stmtData.data), col->length); + } + + return initStmtDataValue(stbInfo, NULL); +} + +static int64_t fillChildTblNameByCount(SSuperTable *stbInfo) { + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + snprintf(stbInfo->childTblArray[i]->name, + TSDB_TABLE_NAME_LEN, + "%s%" PRIu64 "", + stbInfo->childTblPrefix, i); + debugPrint("%s(): %s\n", __func__, + stbInfo->childTblArray[i]->name); + } + + return stbInfo->childTblCount; +} + +static int64_t fillChildTblNameByFromTo(SDataBase *database, + SSuperTable* stbInfo) { + for (int64_t i = stbInfo->childTblFrom; i < stbInfo->childTblTo; i++) { + snprintf(stbInfo->childTblArray[i-stbInfo->childTblFrom]->name, + TSDB_TABLE_NAME_LEN, + "%s%" PRIu64 "", + stbInfo->childTblPrefix, i); + } + + return (stbInfo->childTblTo-stbInfo->childTblFrom); +} + +static int64_t fillChildTblNameByLimitOffset(SDataBase *database, + SSuperTable* stbInfo) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + char cmd[SHORT_1K_SQL_BUFF_LEN] = "\0"; + if (g_arguments->taosc_version == 3) { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT DISTINCT(TBNAME) FROM %s.`%s` LIMIT %" PRId64 + " OFFSET %" PRIu64 "", + database->dbName, stbInfo->stbName, stbInfo->childTblLimit, + stbInfo->childTblOffset); + } else { + snprintf(cmd, SHORT_1K_SQL_BUFF_LEN, + "SELECT TBNAME FROM %s.`%s` LIMIT %" PRId64 + " OFFSET %" PRIu64 "", + database->dbName, stbInfo->stbName, stbInfo->childTblLimit, + stbInfo->childTblOffset); + } + debugPrint("cmd: %s\n", cmd); + TAOS_RES *res = taos_query(conn->taos, cmd); + int32_t code = taos_errno(res); + int64_t count = 0; + if (code) { + printErrCmdCodeStr(cmd, code, res); + closeBenchConn(conn); + return -1; + } + TAOS_ROW row = NULL; + while ((row = taos_fetch_row(res)) != NULL) { + int *lengths = taos_fetch_lengths(res); + strncpy(stbInfo->childTblArray[count]->name, row[0], lengths[0]); + stbInfo->childTblArray[count]->name[lengths[0] + 1] = '\0'; + debugPrint("stbInfo->childTblArray[%" PRId64 "]->name: %s\n", + count, stbInfo->childTblArray[count]->name); + count++; + } + taos_free_result(res); + closeBenchConn(conn); + return count; +} + +static void preProcessArgument(SSuperTable *stbInfo) { + if (stbInfo->interlaceRows > g_arguments->reqPerReq) { + infoPrint( + "interlaceRows(%d) is larger than record per request(%u), which " + "will be set to %u\n", + stbInfo->interlaceRows, g_arguments->reqPerReq, + g_arguments->reqPerReq); + stbInfo->interlaceRows = g_arguments->reqPerReq; + } + + if (stbInfo->interlaceRows > stbInfo->insertRows) { + infoPrint( + "interlaceRows larger than insertRows %d > %" PRId64 "\n", + stbInfo->interlaceRows, stbInfo->insertRows); + infoPrint("%s", "interlaceRows will be set to 0\n"); + stbInfo->interlaceRows = 0; + } + + if (stbInfo->interlaceRows == 0 + && g_arguments->reqPerReq > stbInfo->insertRows) { + infoPrint("record per request (%u) is larger than " + "insert rows (%"PRIu64")" + " in progressive mode, which will be set to %"PRIu64"\n", + g_arguments->reqPerReq, stbInfo->insertRows, + stbInfo->insertRows); + g_arguments->reqPerReq = stbInfo->insertRows; + } + + if (stbInfo->interlaceRows > 0 && stbInfo->iface == STMT_IFACE + && stbInfo->autoTblCreating) { + infoPrint("%s", + "not support autocreate table with interlace row in stmt " + "insertion, will change to progressive mode\n"); + stbInfo->interlaceRows = 0; + } +} + +static int printTotalDelay(SDataBase *database, + int64_t totalDelay, + BArray *total_delay_list, + int threads, + int64_t totalInsertRows, + int64_t start, int64_t end) { + succPrint("Spent %.6f seconds to insert rows: %" PRIu64 + " with %d thread(s) into %s %.2f records/second\n", + (end - start)/1E6, totalInsertRows, threads, + database->dbName, + (double)(totalInsertRows / ((end - start)/1E6))); + if (!total_delay_list->size) { + return -1; + } + + succPrint("insert delay, " + "min: %.4fms, " + "avg: %.4fms, " + "p90: %.4fms, " + "p95: %.4fms, " + "p99: %.4fms, " + "max: %.4fms\n", + *(int64_t *)(benchArrayGet(total_delay_list, 0))/1E3, + (double)totalDelay/total_delay_list->size/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + * 0.9)))/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + * 0.95)))/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + * 0.99)))/1E3, + *(int64_t *)(benchArrayGet(total_delay_list, + (int32_t)(total_delay_list->size + - 1)))/1E3); + return 0; +} + +static int64_t fillChildTblNameImp(SDataBase *database, SSuperTable *stbInfo) { + int64_t ntables; + if (stbInfo->childTblLimit) { + ntables = fillChildTblNameByLimitOffset(database, stbInfo); + } else if (stbInfo->childTblFrom || stbInfo->childTblTo) { + ntables = fillChildTblNameByFromTo(database, stbInfo); + } else { + ntables = fillChildTblNameByCount(stbInfo); + } + return ntables; +} + +static int64_t fillChildTblName(SDataBase *database, SSuperTable *stbInfo) { + int64_t ntables = stbInfo->childTblCount; + stbInfo->childTblArray = benchCalloc(stbInfo->childTblCount, + sizeof(SChildTable*), true); + for (int64_t child = 0; child < stbInfo->childTblCount; child++) { + stbInfo->childTblArray[child] = + benchCalloc(1, sizeof(SChildTable), true); + } + + if (stbInfo->childTblCount == 1 && stbInfo->tags->size == 0) { + // Normal table + snprintf(stbInfo->childTblArray[0]->name, TSDB_TABLE_NAME_LEN, + "%s", stbInfo->stbName); + } else if ((stbInfo->iface != SML_IFACE + && stbInfo->iface != SML_REST_IFACE) + && stbInfo->childTblExists) { + ntables = fillChildTblNameImp(database, stbInfo); + } else { + ntables = fillChildTblNameByCount(stbInfo); + } + + return ntables; +} + +static int startMultiThreadInsertData(SDataBase* database, + SSuperTable* stbInfo) { + if ((stbInfo->iface == SML_IFACE || stbInfo->iface == SML_REST_IFACE) + && !stbInfo->use_metric) { + errorPrint("%s", "schemaless cannot work without stable\n"); + return -1; + } + + preProcessArgument(stbInfo); + + int64_t ntables; + if (stbInfo->childTblTo > 0) { + ntables = stbInfo->childTblTo - stbInfo->childTblFrom; + } else if (stbInfo->childTblLimit) { + ntables = stbInfo->childTblLimit; + } else { + ntables = stbInfo->childTblCount; + } + if (ntables == 0) { + return 0; + } + + uint64_t tableFrom = 0; + int32_t threads = g_arguments->nthreads; + int64_t a = 0, b = 0; + +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if ((0 == stbInfo->interlaceRows) + && (g_arguments->nthreads_auto)) { + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return -1; + } + + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); + closeBenchConn(conn); + return -1; + } + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", + database->dbName, + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + if (vgId == vg->vgId) { + vg->tbCountPerVgId++; + } + } + } + + threads = 0; + for (int v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + infoPrint("Total %"PRId64" tables on bb %s's vgroup %d (id: %d)\n", + vg->tbCountPerVgId, database->dbName, v, vg->vgId); + if (vg->tbCountPerVgId) { + threads++; + } else { + continue; + } + vg->childTblArray = benchCalloc( + vg->tbCountPerVgId, sizeof(SChildTable*), true); + vg->tbOffset = 0; + } + for (int64_t i = 0; i < stbInfo->childTblCount; i++) { + int vgId; + int ret = taos_get_table_vgId( + conn->taos, database->dbName, + stbInfo->childTblArray[i]->name, &vgId); + if (ret < 0) { + errorPrint("Failed to get %s db's %s table's vgId\n", + database->dbName, + stbInfo->childTblArray[i]->name); + + closeBenchConn(conn); + return -1; + } + debugPrint("Db %s\'s table\'s %s vgId is: %d\n", + database->dbName, + stbInfo->childTblArray[i]->name, vgId); + for (int32_t v = 0; v < database->vgroups; v++) { + SVGroup *vg = benchArrayGet(database->vgArray, v); + if (vgId == vg->vgId) { + vg->childTblArray[vg->tbOffset] = + stbInfo->childTblArray[i]; + vg->tbOffset++; + } + } + } + closeBenchConn(conn); + } else { + a = ntables / threads; + if (a < 1) { + threads = (int32_t)ntables; + a = 1; + } + b = 0; + if (threads != 0) { + b = ntables % threads; + } + } + + int32_t vgFrom = 0; +#else + a = ntables / threads; + if (a < 1) { + threads = (int32_t)ntables; + a = 1; + } + b = 0; + if (threads != 0) { + b = ntables % threads; + } +#endif // TD_VER_COMPATIBLE_3_0_0_0 + pthread_t *pids = benchCalloc(1, threads * sizeof(pthread_t), true); + threadInfo *infos = benchCalloc(1, threads * sizeof(threadInfo), true); + + for (int32_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->threadID = i; + pThreadInfo->dbInfo = database; + pThreadInfo->stbInfo = stbInfo; + pThreadInfo->start_time = stbInfo->startTimestamp; + pThreadInfo->totalInsertRows = 0; + pThreadInfo->samplePos = 0; +#ifdef TD_VER_COMPATIBLE_3_0_0_0 + if ((0 == stbInfo->interlaceRows) + && (g_arguments->nthreads_auto)) { + int32_t j; + for (j = vgFrom; i < database->vgroups; j++) { + SVGroup *vg = benchArrayGet(database->vgArray, j); + if (0 == vg->tbCountPerVgId) { + continue; + } + pThreadInfo->vg = vg; + pThreadInfo->start_table_from = 0; + pThreadInfo->ntables = vg->tbCountPerVgId; + pThreadInfo->end_table_to = vg->tbCountPerVgId-1; + break; + } + vgFrom = j + 1; + } else { + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = i < b ? a + 1 : a; + pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); + tableFrom = pThreadInfo->end_table_to + 1; + } +#else + pThreadInfo->start_table_from = tableFrom; + pThreadInfo->ntables = i < b ? a + 1 : a; + pThreadInfo->end_table_to = (i < b)?(tableFrom+a):(tableFrom+a-1); + tableFrom = pThreadInfo->end_table_to + 1; +#endif // TD_VER_COMPATIBLE_3_0_0_0 + pThreadInfo->delayList = benchArrayInit(1, sizeof(int64_t)); + switch (stbInfo->iface) { + case REST_IFACE: { + if (stbInfo->interlaceRows > 0) { + pThreadInfo->buffer = new_ds(0); + } else { + pThreadInfo->buffer = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + } + int sockfd = createSockFd(); + if (sockfd < 0) { + FREE_PIDS_INFOS_RETURN_MINUS_1(); + } + pThreadInfo->sockfd = sockfd; + break; + } + case STMT_IFACE: { + pThreadInfo->conn = initBenchConn(); + if (NULL == pThreadInfo->conn) { + FREE_PIDS_INFOS_RETURN_MINUS_1(); + } + pThreadInfo->conn->stmt = + taos_stmt_init(pThreadInfo->conn->taos); + if (NULL == pThreadInfo->conn->stmt) { + errorPrint("taos_stmt_init() failed, reason: %s\n", + taos_errstr(NULL)); + FREE_RESOURCE(); + return -1; + } + if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { + errorPrint("taos select database(%s) failed\n", + database->dbName); + FREE_RESOURCE(); + return -1; + } + if (!stbInfo->autoTblCreating) { + if (prepareStmt(stbInfo, pThreadInfo->conn->stmt, 0)) { + FREE_RESOURCE(); + return -1; + } + } + + pThreadInfo->bind_ts = benchCalloc(1, sizeof(int64_t), true); + pThreadInfo->bind_ts_array = + benchCalloc(1, sizeof(int64_t)*g_arguments->reqPerReq, + true); + pThreadInfo->bindParams = benchCalloc( + 1, sizeof(TAOS_MULTI_BIND)*(stbInfo->cols->size + 1), + true); + pThreadInfo->is_null = benchCalloc(1, g_arguments->reqPerReq, + true); + parseBufferToStmtBatch(stbInfo); + for (int64_t child = 0; + child < stbInfo->childTblCount; child++) { + SChildTable *childTbl = stbInfo->childTblArray[child]; + if (childTbl->useOwnSample) { + parseBufferToStmtBatchChildTbl(stbInfo, childTbl); + } + } + + break; + } + case SML_REST_IFACE: { + int sockfd = createSockFd(); + if (sockfd < 0) { + free(pids); + free(infos); + return -1; + } + pThreadInfo->sockfd = sockfd; + } + case SML_IFACE: { + pThreadInfo->conn = initBenchConn(); + if (pThreadInfo->conn == NULL) { + errorPrint("%s() init connection failed\n", __func__); + FREE_RESOURCE(); + return -1; + } + if (taos_select_db(pThreadInfo->conn->taos, database->dbName)) { + errorPrint("taos select database(%s) failed\n", + database->dbName); + FREE_RESOURCE(); + return -1; + } + pThreadInfo->max_sql_len = + stbInfo->lenOfCols + stbInfo->lenOfTags; + if (stbInfo->iface == SML_REST_IFACE) { + pThreadInfo->buffer = + benchCalloc(1, g_arguments->reqPerReq * + (1 + pThreadInfo->max_sql_len), true); + } + int protocol = stbInfo->lineProtocol; + if (TSDB_SML_JSON_PROTOCOL != protocol + && SML_JSON_TAOS_FORMAT != protocol) { + pThreadInfo->sml_tags = + (char **)benchCalloc(pThreadInfo->ntables, + sizeof(char *), true); + for (int t = 0; t < pThreadInfo->ntables; t++) { + pThreadInfo->sml_tags[t] = + benchCalloc(1, stbInfo->lenOfTags, true); + } + + for (int t = 0; t < pThreadInfo->ntables; t++) { + if (generateRandData( + stbInfo, pThreadInfo->sml_tags[t], + stbInfo->lenOfTags, + stbInfo->lenOfCols + stbInfo->lenOfTags, + stbInfo->tags, 1, true, NULL)) { + return -1; + } + debugPrint("pThreadInfo->sml_tags[%d]: %s\n", t, + pThreadInfo->sml_tags[t]); + } + pThreadInfo->lines = + benchCalloc(g_arguments->reqPerReq, + sizeof(char *), true); + + for (int j = 0; (j < g_arguments->reqPerReq + && !g_arguments->terminate); j++) { + pThreadInfo->lines[j] = + benchCalloc(1, pThreadInfo->max_sql_len, true); + } + } else { + pThreadInfo->json_array = tools_cJSON_CreateArray(); + pThreadInfo->sml_json_tags = tools_cJSON_CreateArray(); + pThreadInfo->sml_tags_json_array = (char **)benchCalloc( + pThreadInfo->ntables, sizeof(char *), true); + for (int t = 0; t < pThreadInfo->ntables; t++) { + if (stbInfo->lineProtocol == TSDB_SML_JSON_PROTOCOL) { + generateSmlJsonTags( + pThreadInfo->sml_json_tags, + pThreadInfo->sml_tags_json_array, + stbInfo, + pThreadInfo->start_table_from, t); + } else { + generateSmlTaosJsonTags( + pThreadInfo->sml_json_tags, stbInfo, + pThreadInfo->start_table_from, t); + } + } + pThreadInfo->lines = (char **)benchCalloc( + 1, sizeof(char *), true); + if ((0 == stbInfo->interlaceRows) + && (TSDB_SML_JSON_PROTOCOL == protocol)) { + pThreadInfo->line_buf_len = + g_arguments->reqPerReq * + accumulateRowLen(pThreadInfo->stbInfo->tags, + pThreadInfo->stbInfo->iface); + debugPrint("%s() LN%d, line_buf_len=%d\n", + __func__, __LINE__, pThreadInfo->line_buf_len); + pThreadInfo->lines[0] = benchCalloc( + 1, pThreadInfo->line_buf_len, true); + pThreadInfo->sml_json_value_array = + (char **)benchCalloc( + pThreadInfo->ntables, sizeof(char *), true); + for (int t = 0; t < pThreadInfo->ntables; t++) { + generateSmlJsonValues( + pThreadInfo->sml_json_value_array, stbInfo, t); + } + } + } + break; + } + case TAOSC_IFACE: { + pThreadInfo->conn = initBenchConn(); + if (pThreadInfo->conn == NULL) { + errorPrint("%s() failed to connect\n", __func__); + FREE_RESOURCE(); + return -1; + } + char* command = benchCalloc(1, SHORT_1K_SQL_BUFF_LEN, false); + snprintf(command, SHORT_1K_SQL_BUFF_LEN, + g_arguments->escape_character + ? "USE `%s`" + : "USE %s", + database->dbName); + if (queryDbExecCall(pThreadInfo->conn, command)) { + errorPrint("taos select database(%s) failed\n", + database->dbName); + FREE_RESOURCE(); + tmfree(command); + return -1; + } + tmfree(command); + command = NULL; + + if (stbInfo->interlaceRows > 0) { + pThreadInfo->buffer = new_ds(0); + } else { + pThreadInfo->buffer = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + if (g_arguments->check_sql) { + pThreadInfo->csql = + benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, true); + memset(pThreadInfo->csql, 0, TSDB_MAX_ALLOWED_SQL_LEN); + } + } + + break; + } + default: + break; + } + } + + infoPrint("Estimate memory usage: %.2fMB\n", + (double)g_memoryUsage / 1048576); + prompt(0); + + // create threads + for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + threadInfo *pThreadInfo = infos + i; + if (stbInfo->interlaceRows > 0) { + pthread_create(pids + i, NULL, + syncWriteInterlace, pThreadInfo); + } else { + pthread_create(pids + i, NULL, + syncWriteProgressive, pThreadInfo); + } + } + + int64_t start = toolsGetTimestampUs(); + + // wait threads + for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + pthread_join(pids[i], NULL); + } + + int64_t end = toolsGetTimestampUs()+1; + + if (g_arguments->terminate) toolsMsleep(100); + + BArray * total_delay_list = benchArrayInit(1, sizeof(int64_t)); + int64_t totalDelay = 0; + uint64_t totalInsertRows = 0; + + // free threads resource + for (int i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + // free check sql + if (pThreadInfo->csql) { + tmfree(pThreadInfo->csql); + pThreadInfo->csql = NULL; + } + + int protocol = stbInfo->lineProtocol; + switch (stbInfo->iface) { + case REST_IFACE: + if (g_arguments->terminate) + toolsMsleep(100); + destroySockFd(pThreadInfo->sockfd); + if (stbInfo->interlaceRows > 0) { + free_ds(&pThreadInfo->buffer); + } else { + tmfree(pThreadInfo->buffer); + pThreadInfo->buffer = NULL; + } + break; + case SML_REST_IFACE: + if (g_arguments->terminate) + toolsMsleep(100); + tmfree(pThreadInfo->buffer); + // on-purpose no break here + case SML_IFACE: + if (TSDB_SML_JSON_PROTOCOL != protocol + && SML_JSON_TAOS_FORMAT != protocol) { + for (int t = 0; t < pThreadInfo->ntables; t++) { + tmfree(pThreadInfo->sml_tags[t]); + } + for (int j = 0; j < g_arguments->reqPerReq; j++) { + tmfree(pThreadInfo->lines[j]); + } + tmfree(pThreadInfo->sml_tags); + pThreadInfo->sml_tags = NULL; + } else { + for (int t = 0; t < pThreadInfo->ntables; t++) { + tmfree(pThreadInfo->sml_tags_json_array[t]); + } + tmfree(pThreadInfo->sml_tags_json_array); + pThreadInfo->sml_tags_json_array = NULL; + if (pThreadInfo->sml_json_tags) { + tools_cJSON_Delete(pThreadInfo->sml_json_tags); + pThreadInfo->sml_json_tags = NULL; + } + if (pThreadInfo->json_array) { + tools_cJSON_Delete(pThreadInfo->json_array); + pThreadInfo->json_array = NULL; + } + } + if (pThreadInfo->lines) { + if ((0 == stbInfo->interlaceRows) + && (TSDB_SML_JSON_PROTOCOL == protocol)) { + tmfree(pThreadInfo->lines[0]); + for (int t = 0; t < pThreadInfo->ntables; t++) { + tmfree(pThreadInfo->sml_json_value_array[t]); + } + tmfree(pThreadInfo->sml_json_value_array); + } + tmfree(pThreadInfo->lines); + pThreadInfo->lines = NULL; + } + break; + + case STMT_IFACE: + taos_stmt_close(pThreadInfo->conn->stmt); + tmfree(pThreadInfo->bind_ts); + tmfree(pThreadInfo->bind_ts_array); + tmfree(pThreadInfo->bindParams); + tmfree(pThreadInfo->is_null); + break; + + case TAOSC_IFACE: + if (stbInfo->interlaceRows > 0) { + free_ds(&pThreadInfo->buffer); + } else { + tmfree(pThreadInfo->buffer); + pThreadInfo->buffer = NULL; + } + break; + + default: + break; + } + totalInsertRows += pThreadInfo->totalInsertRows; + totalDelay += pThreadInfo->totalDelay; + benchArrayAddBatch(total_delay_list, pThreadInfo->delayList->pData, + pThreadInfo->delayList->size); + tmfree(pThreadInfo->delayList); + pThreadInfo->delayList = NULL; + // free conn + if (pThreadInfo->conn) { + closeBenchConn(pThreadInfo->conn); + pThreadInfo->conn = NULL; + } + } + + // calculate result + qsort(total_delay_list->pData, total_delay_list->size, + total_delay_list->elemSize, compare); + + if (g_arguments->terminate) toolsMsleep(100); + + free(pids); + free(infos); + + int ret = printTotalDelay(database, totalDelay, + total_delay_list, threads, + totalInsertRows, start, end); + benchArrayDestroy(total_delay_list); + if (g_fail || ret) { + return -1; + } + return 0; +} + +static int getStbInsertedRows(char* dbName, char* stbName, TAOS* taos) { + int rows = 0; + char command[SHORT_1K_SQL_BUFF_LEN]; + snprintf(command, SHORT_1K_SQL_BUFF_LEN, "SELECT COUNT(*) FROM %s.%s", + dbName, stbName); + TAOS_RES* res = taos_query(taos, command); + int code = taos_errno(res); + if (code != 0) { + printErrCmdCodeStr(command, code, res); + return -1; + } + TAOS_ROW row = taos_fetch_row(res); + if (row == NULL) { + rows = 0; + } else { + rows = (int)*(int64_t*)row[0]; + } + taos_free_result(res); + return rows; +} + +static void create_tsma(TSMA* tsma, SBenchConn* conn, char* stbName) { + char command[SHORT_1K_SQL_BUFF_LEN]; + int len = snprintf(command, SHORT_1K_SQL_BUFF_LEN, + "CREATE sma INDEX %s ON %s function(%s) " + "INTERVAL (%s) SLIDING (%s)", + tsma->name, stbName, tsma->func, + tsma->interval, tsma->sliding); + if (tsma->custom) { + snprintf(command + len, SHORT_1K_SQL_BUFF_LEN - len, + " %s", tsma->custom); + } + int code = queryDbExecCall(conn, command); + if (code == 0) { + infoPrint("successfully create tsma with command <%s>\n", command); + } +} + +static void* create_tsmas(void* args) { + tsmaThreadInfo* pThreadInfo = (tsmaThreadInfo*) args; + int inserted_rows = 0; + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + return NULL; + } + int finished = 0; + if (taos_select_db(conn->taos, pThreadInfo->dbName)) { + errorPrint("failed to use database (%s)\n", pThreadInfo->dbName); + closeBenchConn(conn); + return NULL; + } + while (finished < pThreadInfo->tsmas->size && inserted_rows >= 0) { + inserted_rows = (int)getStbInsertedRows( + pThreadInfo->dbName, pThreadInfo->stbName, conn->taos); + for (int i = 0; i < pThreadInfo->tsmas->size; i++) { + TSMA* tsma = benchArrayGet(pThreadInfo->tsmas, i); + if (!tsma->done && inserted_rows >= tsma->start_when_inserted) { + create_tsma(tsma, conn, pThreadInfo->stbName); + tsma->done = true; + finished++; + break; + } + } + toolsMsleep(10); + } + benchArrayDestroy(pThreadInfo->tsmas); + closeBenchConn(conn); + return NULL; +} + +static int32_t createStream(SSTREAM* stream) { + int32_t code = -1; + char * command = benchCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN, false); + snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, "DROP STREAM IF EXISTS %s", + stream->stream_name); + infoPrint("%s\n", command); + SBenchConn* conn = initBenchConn(); + if (NULL == conn) { + goto END_STREAM; + } + + code = queryDbExecCall(conn, command); + int32_t trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds then re-drop stream %s\n", + g_arguments->trying_interval, stream->stream_name); + toolsMsleep(g_arguments->trying_interval); + code = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + + if (code) { + closeBenchConn(conn); + goto END_STREAM; + } + + memset(command, 0, TSDB_MAX_ALLOWED_SQL_LEN); + int pos = snprintf(command, TSDB_MAX_ALLOWED_SQL_LEN, + "CREATE STREAM IF NOT EXISTS %s ", stream->stream_name); + if (stream->trigger_mode[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "TRIGGER %s ", stream->trigger_mode); + } + if (stream->watermark[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "WATERMARK %s ", stream->watermark); + } + if (stream->ignore_update[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "IGNORE UPDATE %s ", stream->ignore_update); + } + if (stream->ignore_expired[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "IGNORE EXPIRED %s ", stream->ignore_expired); + } + if (stream->fill_history[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "FILL_HISTORY %s ", stream->fill_history); + } + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "INTO %s ", stream->stream_stb); + if (stream->stream_stb_field[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "%s ", stream->stream_stb_field); + } + if (stream->stream_tag_field[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "TAGS%s ", stream->stream_tag_field); + } + if (stream->subtable[0] != '\0') { + pos += snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "SUBTABLE%s ", stream->subtable); + } + snprintf(command + pos, TSDB_MAX_ALLOWED_SQL_LEN - pos, + "as %s", stream->source_sql); + infoPrint("%s\n", command); + + code = queryDbExecCall(conn, command); + trying = g_arguments->keep_trying; + while (code && trying) { + infoPrint("will sleep %"PRIu32" milliseconds " + "then re-create stream %s\n", + g_arguments->trying_interval, stream->stream_name); + toolsMsleep(g_arguments->trying_interval); + code = queryDbExecCall(conn, command); + if (trying != -1) { + trying--; + } + } + + closeBenchConn(conn); +END_STREAM: + tmfree(command); + return code; +} + +int insertTestProcess() { + prompt(0); + + encodeAuthBase64(); + for (int i = 0; i < g_arguments->databases->size; i++) { + if (REST_IFACE == g_arguments->iface) { + if (0 != convertServAddr(g_arguments->iface, + false, + 1)) { + return -1; + } + } + SDataBase * database = benchArrayGet(g_arguments->databases, i); + + if (database->drop && !(g_arguments->supplementInsert)) { + if (database->superTbls) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, 0); + if (stbInfo && (REST_IFACE == stbInfo->iface)) { + if (0 != convertServAddr(stbInfo->iface, + stbInfo->tcpTransfer, + stbInfo->lineProtocol)) { + return -1; + } + } + } + if (createDatabase(database)) { + errorPrint("failed to create database (%s)\n", + database->dbName); + return -1; + } + succPrint("created database (%s)\n", database->dbName); + } + } + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (int j = 0; j < database->superTbls->size; j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + if (stbInfo->iface != SML_IFACE + && stbInfo->iface != SML_REST_IFACE + && !stbInfo->childTblExists) { +#ifdef WEBSOCKET + if (g_arguments->websocket) { + dropSuperTable(database, stbInfo); + } +#endif + if (getSuperTableFromServer(database, stbInfo) != 0) { + if (createSuperTable(database, stbInfo)) { + return -1; + } + } + } + fillChildTblName(database, stbInfo); + if (0 != prepareSampleData(database, stbInfo)) { + return -1; + } + } + } + } + + if (g_arguments->taosc_version == 3) { + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase* database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (int j = 0; (j < database->superTbls->size + && !g_arguments->terminate); j++) { + SSuperTable* stbInfo = + benchArrayGet(database->superTbls, j); + if (stbInfo->tsmas == NULL) { + continue; + } + if (stbInfo->tsmas->size > 0) { + tsmaThreadInfo* pThreadInfo = + benchCalloc(1, sizeof(tsmaThreadInfo), true); + pthread_t tsmas_pid = {0}; + pThreadInfo->dbName = database->dbName; + pThreadInfo->stbName = stbInfo->stbName; + pThreadInfo->tsmas = stbInfo->tsmas; + pthread_create(&tsmas_pid, NULL, + create_tsmas, pThreadInfo); + } + } + } + } + } + + if (createChildTables()) return -1; + + if (g_arguments->taosc_version == 3) { + for (int j = 0; j < g_arguments->streams->size; j++) { + SSTREAM * stream = benchArrayGet(g_arguments->streams, j); + if (stream->drop) { + if (createStream(stream)) { + return -1; + } + } + } + } + + // create sub threads for inserting data + for (int i = 0; i < g_arguments->databases->size; i++) { + SDataBase * database = benchArrayGet(g_arguments->databases, i); + if (database->superTbls) { + for (uint64_t j = 0; j < database->superTbls->size; j++) { + SSuperTable * stbInfo = benchArrayGet(database->superTbls, j); + if (stbInfo->insertRows == 0) { + continue; + } + prompt(stbInfo->non_stop); + if (startMultiThreadInsertData(database, stbInfo)) { + return -1; + } + } + } + } + return 0; +} diff --git a/src/benchJsonOpt.c b/src/benchJsonOpt.c index 084f1cb4..41592b42 100644 --- a/src/benchJsonOpt.c +++ b/src/benchJsonOpt.c @@ -1776,6 +1776,12 @@ static int getMetaFromTmqJsonFile(tools_cJSON *json) { enableHeartbeatBackground->valuestring; } + tools_cJSON *snapshotEnable = tools_cJSON_GetObjectItem( + tmqInfo, "experimental.snapshot.enable"); + if (tools_cJSON_IsString(snapshotEnable)) { + g_tmqInfo.consumerInfo.snapshotEnable = snapshotEnable->valuestring; + } + tools_cJSON *msgWithTableName = tools_cJSON_GetObjectItem( tmqInfo, "msg.with.table.name"); if (tools_cJSON_IsString(msgWithTableName)) { diff --git a/src/benchTmq.c b/src/benchTmq.c index b289bcb5..77af2fc9 100644 --- a/src/benchTmq.c +++ b/src/benchTmq.c @@ -41,6 +41,7 @@ void printfTmqConfigIntoFile() { infoPrintToFile(g_arguments->fpOfInsertResult, "enableAutoCommit: %s\n", pConsumerInfo->enableAutoCommit); infoPrintToFile(g_arguments->fpOfInsertResult, "autoCommitIntervalMs: %d\n", pConsumerInfo->autoCommitIntervalMs); infoPrintToFile(g_arguments->fpOfInsertResult, "enableHeartbeatBackground: %s\n", pConsumerInfo->enableHeartbeatBackground); + infoPrintToFile(g_arguments->fpOfInsertResult, "snapshotEnable: %s\n", pConsumerInfo->snapshotEnable); infoPrintToFile(g_arguments->fpOfInsertResult, "msgWithTableName: %s\n", pConsumerInfo->msgWithTableName); infoPrintToFile(g_arguments->fpOfInsertResult, "rowsFile: %s\n", pConsumerInfo->rowsFile); infoPrintToFile(g_arguments->fpOfInsertResult, "expectRows: %d\n", pConsumerInfo->expectRows); @@ -280,6 +281,7 @@ int subscribeTestProcess() { tmq_conf_set(conf, "auto.commit.interval.ms", tmpBuff); tmq_conf_set(conf, "enable.heartbeat.background", pConsumerInfo->enableHeartbeatBackground); + tmq_conf_set(conf, "experimental.snapshot.enable", pConsumerInfo->snapshotEnable); tmq_conf_set(conf, "msg.with.table.name", pConsumerInfo->msgWithTableName); pThreadInfo->tmq = tmq_consumer_new(conf, NULL, 0); From a3c36e5b544e782b48a013c8ca5f41b01da4f96e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 16 May 2023 16:54:28 +0800 Subject: [PATCH 14/25] test: add printf insert rate --- src/benchInsert.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index aa4f381c..d66c3b1d 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1447,6 +1447,7 @@ static void *syncWriteInterlace(void *sarg) { int64_t pos = 0; uint32_t batchPerTblTimes = g_arguments->reqPerReq / interlaceRows; uint64_t lastPrintTime = toolsGetTimestampMs(); + uint64_t lastTotalInsertRows = 0; int64_t startTs = toolsGetTimestampUs(); int64_t endTs; uint64_t tableSeq = pThreadInfo->start_table_from; @@ -1731,9 +1732,11 @@ static void *syncWriteInterlace(void *sarg) { if (currentPrintTime - lastPrintTime > 30 * 1000) { infoPrint( "thread[%d] has currently inserted rows: %" PRIu64 - "\n", - pThreadInfo->threadID, pThreadInfo->totalInsertRows); + ", peroid insert rate: %.3f rows/s \n", + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); lastPrintTime = currentPrintTime; + lastTotalInsertRows = pThreadInfo->totalInsertRows; } } free_of_interlace: @@ -2152,6 +2155,7 @@ void *syncWriteProgressive(void *sarg) { pThreadInfo->end_table_to + 1); #endif uint64_t lastPrintTime = toolsGetTimestampMs(); + uint64_t lastTotalInsertRows = 0; int64_t startTs = toolsGetTimestampUs(); int64_t endTs; @@ -2354,9 +2358,11 @@ void *syncWriteProgressive(void *sarg) { if (currentPrintTime - lastPrintTime > 30 * 1000) { infoPrint( "thread[%d] has currently inserted rows: " - "%" PRId64 "\n", - pThreadInfo->threadID, pThreadInfo->totalInsertRows); + "%" PRId64 ", peroid insert rate: %.3f rows/s \n", + pThreadInfo->threadID, pThreadInfo->totalInsertRows, + (double)(pThreadInfo->totalInsertRows - lastTotalInsertRows) * 1000.0/(currentPrintTime - lastPrintTime)); lastPrintTime = currentPrintTime; + lastTotalInsertRows = pThreadInfo->totalInsertRows; } if (i >= stbInfo->insertRows) { break; From aa67feb11b29ee815c42e858a039175f1c974f9a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 17 May 2023 22:59:57 +0800 Subject: [PATCH 15/25] test: refine test cases (#655) * test: refine test cases * fix: 3.0-taosbenchmark-* * fix: add schedule event * test: merge with develop --- .github/workflows/2.x-aarch64.yml | 5 +++ .github/workflows/2.x-archlinux-release.yml | 9 +++++ .github/workflows/2.x-armv7.yml | 3 ++ .github/workflows/2.x-centos-release.yml | 9 +++++ .github/workflows/2.x-macos-debug.yml | 4 +-- .github/workflows/2.x-macos-release.yml | 4 +-- .github/workflows/2.x-taosbenchmark-debug.yml | 4 +-- .../workflows/2.x-taosbenchmark-release.yml | 4 +-- .github/workflows/2.x-taosdump-debug.yml | 4 +-- .github/workflows/2.x-taosdump-release.yml | 4 +-- .github/workflows/2.x-windows-build.yml | 6 ++-- .github/workflows/3.0-alpine.yml | 5 ++- .github/workflows/3.0-archlinux-release.yml | 13 +++++-- .github/workflows/3.0-coveralls.yml | 36 +++++++++++++++++-- .github/workflows/3.0-macos-debug.yml | 11 +++++- .github/workflows/3.0-macos-release.yml | 11 +++++- .github/workflows/3.0-non-x64.yml | 6 ++-- .github/workflows/3.0-taosbenchmark-debug.yml | 21 +++++++++-- .../workflows/3.0-taosbenchmark-release.yml | 21 +++++++++-- .github/workflows/3.0-taosdump-debug.yml | 14 ++++++-- .github/workflows/3.0-taosdump-release-ws.yml | 15 ++++++-- .github/workflows/3.0-taosdump-release.yml | 14 ++++++-- .github/workflows/3.0-windows-build.yml | 6 ++++ .github/workflows/codeql.yml | 4 +-- .github/workflows/cppcheck.yml | 2 ++ .../commandline-partial-col-numpy.py | 2 +- .../taosbenchmark/commandline-single-table.py | 1 - .../query_json-with-error-sqlfile.py | 2 -- tests/taosbenchmark/reuse-exist-stb.py | 2 -- .../taosc_insert_alltypes_json-partial-col.py | 6 +--- ...aosdemoTestInsertWithJsonStmt-otherPara.py | 2 -- .../taosdemoTestQueryWithJson.py | 1 - tests/taosbenchmark/v3/commandline-vgroups.py | 1 - tests/taosbenchmark/v3/default_tmq_json.py | 32 +++++++++-------- .../v3/query_json-kill-slow-query.py | 1 - 35 files changed, 218 insertions(+), 67 deletions(-) diff --git a/.github/workflows/2.x-aarch64.yml b/.github/workflows/2.x-aarch64.yml index 4adeb404..eaae6533 100644 --- a/.github/workflows/2.x-aarch64.yml +++ b/.github/workflows/2.x-aarch64.yml @@ -92,6 +92,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: recursive @@ -104,6 +105,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "The uname output was ${{ steps.setup.outputs.uname }}" sudo timedatectl set-timezone Asia/Shanghai @@ -131,6 +133,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: uraimo/run-on-arch-action@v2 with: arch: aarch64 @@ -156,6 +159,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: uraimo/run-on-arch-action@v2 with: arch: aarch64 @@ -182,6 +186,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: uraimo/run-on-arch-action@v2 with: arch: aarch64 diff --git a/.github/workflows/2.x-archlinux-release.yml b/.github/workflows/2.x-archlinux-release.yml index 061ba74a..b982846a 100644 --- a/.github/workflows/2.x-archlinux-release.yml +++ b/.github/workflows/2.x-archlinux-release.yml @@ -83,6 +83,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pacman -Syu --noconfirm > /dev/null pacman -Sy git --noconfirm > /dev/null @@ -92,6 +93,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: recursive @@ -104,6 +106,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "disable timezone changing" #timedatectl set-timezone Asia/Shanghai @@ -123,6 +126,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -132,6 +136,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -141,6 +146,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' id: cache-rust with: path: | @@ -155,6 +161,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pacman -Sy --noconfirm gcc make cmake zlib snappy gcc-libs pkg-config python3 python-pip > /dev/null gcc --version @@ -171,6 +178,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pip3 install decorator numpy fabric2 psutil pandas faker toml > /dev/null pip3 install taospy > /dev/null @@ -187,6 +195,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | if [ -f ~/taosdump-failed.txt ]; then cat ~/taosdump-failed.txt; diff --git a/.github/workflows/2.x-armv7.yml b/.github/workflows/2.x-armv7.yml index a22f30cd..21a71c5d 100644 --- a/.github/workflows/2.x-armv7.yml +++ b/.github/workflows/2.x-armv7.yml @@ -92,6 +92,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: recursive @@ -104,6 +105,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "The uname output was ${{ steps.setup.outputs.uname }}" sudo timedatectl set-timezone Asia/Shanghai @@ -131,6 +133,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: uraimo/run-on-arch-action@v2 with: arch: armv7 diff --git a/.github/workflows/2.x-centos-release.yml b/.github/workflows/2.x-centos-release.yml index 7bdb9b1e..337803a3 100644 --- a/.github/workflows/2.x-centos-release.yml +++ b/.github/workflows/2.x-centos-release.yml @@ -83,6 +83,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "yum update" yum update -y > /dev/null @@ -99,6 +100,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: recursive @@ -111,6 +113,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "disable timezone changing" #timedatectl set-timezone Asia/Shanghai @@ -130,6 +133,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -139,6 +143,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -148,6 +153,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' id: cache-rust with: path: | @@ -162,6 +168,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | yum install -y python3 python3-pip zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel > /dev/null python3 -m pip install --upgrade pip > /dev/null 2>&1 @@ -177,6 +184,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pip3 install decorator numpy fabric2 psutil pandas faker toml > /dev/null pip3 install taospy > /dev/null @@ -193,6 +201,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | if [ -f ~/taosdump-failed.txt ]; then cat ~/taosdump-failed.txt; diff --git a/.github/workflows/2.x-macos-debug.yml b/.github/workflows/2.x-macos-debug.yml index 85110fde..7dde0740 100644 --- a/.github/workflows/2.x-macos-debug.yml +++ b/.github/workflows/2.x-macos-debug.yml @@ -73,8 +73,8 @@ jobs: - name: Checkout TDengine's taos-tools to PR number if: | - github.event_name == 'pull_request' && - steps.changed-files-specific.outputs.any_changed == 'true' + (github.event_name == 'pull_request' + && steps.changed-files-specific.outputs.any_changed == 'true') run: | cd TDengine/src/kit/taos-tools git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge diff --git a/.github/workflows/2.x-macos-release.yml b/.github/workflows/2.x-macos-release.yml index 064d2568..7023fb88 100644 --- a/.github/workflows/2.x-macos-release.yml +++ b/.github/workflows/2.x-macos-release.yml @@ -73,8 +73,8 @@ jobs: - name: Checkout TDengine's taos-tools to PR number if: | - github.event_name == 'pull_request' && - steps.changed-files-specific.outputs.any_changed == 'true' + (github.event_name == 'pull_request' + && steps.changed-files-specific.outputs.any_changed == 'true') run: | cd TDengine/src/kit/taos-tools git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge diff --git a/.github/workflows/2.x-taosbenchmark-debug.yml b/.github/workflows/2.x-taosbenchmark-debug.yml index bec9d897..71d28ff1 100644 --- a/.github/workflows/2.x-taosbenchmark-debug.yml +++ b/.github/workflows/2.x-taosbenchmark-debug.yml @@ -73,8 +73,8 @@ jobs: - name: Checkout TDengine's taos-tools to PR number if: | - steps.changed-files-specific.outputs.any_changed == 'true' && - github.event_name == 'pull_request' + (steps.changed-files-specific.outputs.any_changed == 'true' && + github.event_name == 'pull_request') run: | cd TDengine/src/kit/taos-tools git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge diff --git a/.github/workflows/2.x-taosbenchmark-release.yml b/.github/workflows/2.x-taosbenchmark-release.yml index 6c16bc82..51796d31 100644 --- a/.github/workflows/2.x-taosbenchmark-release.yml +++ b/.github/workflows/2.x-taosbenchmark-release.yml @@ -73,8 +73,8 @@ jobs: - name: Checkout TDengine's taos-tools to PR number if: | - steps.changed-files-specific.outputs.any_changed == 'true' && - github.event_name == 'pull_request' + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') run: | cd TDengine/src/kit/taos-tools git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge diff --git a/.github/workflows/2.x-taosdump-debug.yml b/.github/workflows/2.x-taosdump-debug.yml index 115b334b..e6f8f444 100644 --- a/.github/workflows/2.x-taosdump-debug.yml +++ b/.github/workflows/2.x-taosdump-debug.yml @@ -74,8 +74,8 @@ jobs: - name: Checkout TDengine's taos-tools to PR number if: | - steps.changed-files-specific.outputs.any_changed == 'true' && - github.event_name == 'pull_request' + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') run: | cd TDengine/src/kit/taos-tools git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge diff --git a/.github/workflows/2.x-taosdump-release.yml b/.github/workflows/2.x-taosdump-release.yml index a684a395..52d8f98f 100644 --- a/.github/workflows/2.x-taosdump-release.yml +++ b/.github/workflows/2.x-taosdump-release.yml @@ -83,8 +83,8 @@ jobs: - name: Checkout TDengine's taos-tools to PR number if: | - steps.changed-files-specific.outputs.any_changed == 'true' && - github.event_name == 'pull_request' + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') run: | cd TDengine/src/kit/taos-tools git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge diff --git a/.github/workflows/2.x-windows-build.yml b/.github/workflows/2.x-windows-build.yml index 99e17fe4..0726feac 100644 --- a/.github/workflows/2.x-windows-build.yml +++ b/.github/workflows/2.x-windows-build.yml @@ -60,6 +60,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: recursive @@ -69,8 +70,8 @@ jobs: - name: Checkout TDengine's taos-tools to PR number if: | - github.event_name == 'pull_request' && - steps.changed-files-specific.outputs.any_changed == 'true' + (github.event_name == 'pull_request' + && steps.changed-files-specific.outputs.any_changed == 'true') run: | cd TDengine/src/kit/taos-tools git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge @@ -82,6 +83,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | choco install jom gawk grep -y #cd TDengine && mkdir debug && cd debug && cmake .. -G "NMake Makefiles JOM" -DBUILD_JDBC=false -DBUILD_TOOLS=true && jom && jom install diff --git a/.github/workflows/3.0-alpine.yml b/.github/workflows/3.0-alpine.yml index 94647343..360ccbf8 100644 --- a/.github/workflows/3.0-alpine.yml +++ b/.github/workflows/3.0-alpine.yml @@ -78,6 +78,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: jirutka/setup-alpine@v1 @@ -86,6 +87,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cat /etc/alpine-release apk add argp-standalone bash curl cmake gcc make g++ git go procps lsof valgrind linux-headers libunwind libunwind-dev tzdata wget jansson-dev snappy-dev xz-dev zlib-dev @@ -96,11 +98,12 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' shell: alpine.sh --root {0} run: | git clone --branch main --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1 - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DBUILD_HTTP=false && make -j8 && make install + cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DBUILD_HTTP=false && make -j2 && make install if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taos ]] then echo "TDengien build failure" diff --git a/.github/workflows/3.0-archlinux-release.yml b/.github/workflows/3.0-archlinux-release.yml index c05585e3..639815cc 100644 --- a/.github/workflows/3.0-archlinux-release.yml +++ b/.github/workflows/3.0-archlinux-release.yml @@ -82,6 +82,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pacman -Syu --noconfirm > /dev/null pacman -Sy git --noconfirm > /dev/null @@ -91,6 +92,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: submodules: recursive @@ -103,6 +105,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "disable timezone changing" #timedatectl set-timezone Asia/Shanghai @@ -113,6 +116,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -122,6 +126,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -131,6 +136,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' id: cache-rust with: path: | @@ -145,11 +151,12 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | - pacman -Sy --noconfirm gcc make cmake zlib snappy gcc-libs pkg-config python3 python-pip > /dev/null + pacman -Sy --noconfirm gcc make cmake gcc-libs gflags pkg-config python3 python-pip snappy zlib > /dev/null gcc --version python3 -m pip install --upgrade pip > /dev/null 2>&1 - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && make install > /dev/null + cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true && make -j2 > /dev/null && make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] then echo "TDengine build failure" @@ -161,6 +168,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v3 with: fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. @@ -179,6 +187,7 @@ jobs: (needs.check-changed.outputs.changedflag == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug diff --git a/.github/workflows/3.0-coveralls.yml b/.github/workflows/3.0-coveralls.yml index 37de1fad..c4252627 100644 --- a/.github/workflows/3.0-coveralls.yml +++ b/.github/workflows/3.0-coveralls.yml @@ -66,6 +66,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: repository: 'taosdata/TDengine' @@ -77,6 +78,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo timedatectl set-timezone Asia/Shanghai sudo date @@ -115,13 +117,16 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | whoami + sudo apt update > /dev/null + sudo apt install libgflags2.2 libgflags-dev -y > /dev/null cd TDengine mkdir debug ||: cd debug cmake .. -DBUILD_JDBC=false -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null - make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + make -j2 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] then echo "TDengine build failure" @@ -130,8 +135,10 @@ jobs: - name: show TDengine's taosws commit if: | - steps.changed-files-specific.outputs.any_changed == 'true' && - github.event_name == 'pull_request' + (steps.changed-files-specific.outputs.any_changed == 'true' + && github.event_name == 'pull_request') + || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd TDengine/tools/taosws-rs git log --oneline | head -1 @@ -141,6 +148,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo apt update > /dev/null sudo apt-get install lzma-dev liblzma-dev libjansson-dev libz-dev zlib1g libsnappy-dev pkg-config lcov expect -y > /dev/null @@ -158,6 +166,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug @@ -175,6 +184,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "TEST: taodump -p" cd tests/taosdump @@ -185,6 +195,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -219,6 +230,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -263,6 +275,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -330,6 +343,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -353,6 +367,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -383,6 +398,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -427,6 +443,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -449,6 +466,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -494,6 +512,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -526,6 +545,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -572,6 +592,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -593,6 +614,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests sudo pkill -9 taosadapter || : @@ -611,6 +633,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests sudo pkill -9 taosadapter || : @@ -628,6 +651,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -651,6 +675,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -668,6 +693,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo pkill -9 taosadapter || : sudo pkill -9 taosd || : @@ -686,6 +712,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -703,6 +730,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -716,6 +744,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | topdir=`pwd` cd debug @@ -739,6 +768,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: coverallsapp/github-action@master with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/3.0-macos-debug.yml b/.github/workflows/3.0-macos-debug.yml index 96b4dc0a..d56eedc1 100644 --- a/.github/workflows/3.0-macos-debug.yml +++ b/.github/workflows/3.0-macos-debug.yml @@ -62,6 +62,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: repository: 'taosdata/TDengine' @@ -73,6 +74,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | #sudo timedatectl set-timezone Asia/Shanghai #sudo date @@ -82,6 +84,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | brew update brew install argp-standalone pkg-config @@ -92,6 +95,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -101,6 +105,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable @@ -110,6 +115,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' id: cache-rust with: path: | @@ -124,8 +130,9 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null + cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/bin/taosd ]] || [[ ! -f /usr/local/bin/taosadapter ]] then echo "TDengine build failure" @@ -137,6 +144,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "install package for taos-tools" @@ -153,6 +161,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug diff --git a/.github/workflows/3.0-macos-release.yml b/.github/workflows/3.0-macos-release.yml index c6323e1b..24af65ac 100644 --- a/.github/workflows/3.0-macos-release.yml +++ b/.github/workflows/3.0-macos-release.yml @@ -62,6 +62,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: repository: 'taosdata/TDengine' @@ -73,6 +74,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | #sudo timedatectl set-timezone Asia/Shanghai #sudo date @@ -82,6 +84,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | brew update brew install argp-standalone pkg-config @@ -92,6 +95,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -101,6 +105,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable @@ -110,6 +115,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' id: cache-rust with: path: | @@ -124,8 +130,9 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null + cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/bin/taosd ]] || [[ ! -f /usr/local/bin/taosadapter ]] then echo "TDengine build failure" @@ -137,6 +144,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | echo "install package for taos-tools" @@ -153,6 +161,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug diff --git a/.github/workflows/3.0-non-x64.yml b/.github/workflows/3.0-non-x64.yml index 67eb7c50..fecc4c39 100644 --- a/.github/workflows/3.0-non-x64.yml +++ b/.github/workflows/3.0-non-x64.yml @@ -57,6 +57,7 @@ jobs: env: BASE_BRANCH: ${{ github.base_ref }} if: github.event_name == 'pull_request' + || github.event_name == 'schedule' - uses: actions/checkout@v3 with: @@ -125,6 +126,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: uraimo/run-on-arch-action@v2.1.1 with: arch: aarch64 @@ -142,7 +144,7 @@ jobs: cd TDengine || exit 1 mkdir debug ||: cd debug || exit 1 - cmake .. > /dev/null && make -j8 > /dev/null && make install > /dev/null + cmake .. > /dev/null && make -j2 > /dev/null && make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosd ]] then echo "TDengine build failure" @@ -160,7 +162,7 @@ jobs: mkdir debug ||: cd debug - cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} > /dev/null && make -j8 > /dev/null && make install > /dev/null + cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} > /dev/null && make -j2 > /dev/null && make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] then echo "taos-tools build failure" diff --git a/.github/workflows/3.0-taosbenchmark-debug.yml b/.github/workflows/3.0-taosbenchmark-debug.yml index e5737b5e..0212a4a5 100644 --- a/.github/workflows/3.0-taosbenchmark-debug.yml +++ b/.github/workflows/3.0-taosbenchmark-debug.yml @@ -70,6 +70,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo timedatectl set-timezone Asia/Shanghai sudo date @@ -79,6 +80,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -88,6 +90,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -107,8 +110,14 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + sudo apt update > /dev/null + sudo apt install libgflags2.2 libgflags-dev -y > /dev/null + cd TDengine && mkdir debug && cd debug \ + && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null \ + && make -j2 > /dev/null && sudo make install > /dev/null \ + && cd .. && rm -rf debug if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] then echo "TDengine build failure" @@ -120,6 +129,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo apt update > /dev/null sudo apt install lzma-dev liblzma-dev libjansson-dev libz-dev zlib1g libsnappy-dev pkg-config expect -y > /dev/null @@ -137,10 +147,11 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug - cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DTOOLS_SANITIZE=true -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null + cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DTOOLS_SANITIZE=true -DWEBSOCKET=true > /dev/null && make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] then echo "taos-tools build failure" @@ -152,6 +163,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pip3 install decorator numpy fabric2 psutil pandas faker toml taospy > /dev/null @@ -160,6 +172,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -179,6 +192,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' env: TDENGINE_CLOUD_DSN: ${{ secrets.TDENGINE_CLOUD_DSN }} run: | @@ -200,6 +214,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -219,6 +234,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -245,6 +261,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | if [ -f ~/taosbenchmark-release-failed.txt ]; then cat ~/taosbenchmark-release-failed.txt diff --git a/.github/workflows/3.0-taosbenchmark-release.yml b/.github/workflows/3.0-taosbenchmark-release.yml index 3d2110ac..b763cc3c 100644 --- a/.github/workflows/3.0-taosbenchmark-release.yml +++ b/.github/workflows/3.0-taosbenchmark-release.yml @@ -70,6 +70,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo timedatectl set-timezone Asia/Shanghai sudo date @@ -79,6 +80,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -88,6 +90,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -107,8 +110,14 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | - cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + sudo apt update > /dev/null + sudo apt install libgflags2.2 libgflags-dev -y > /dev/null + cd TDengine && mkdir debug && cd debug \ + && cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null \ + && make -j2 > /dev/null && sudo make install > /dev/null \ + && cd .. && rm -rf debug if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] then echo "TDengine build failure" @@ -120,6 +129,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo apt update > /dev/null sudo apt install lzma-dev liblzma-dev libjansson-dev libz-dev zlib1g libsnappy-dev pkg-config expect -y > /dev/null @@ -137,10 +147,11 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug - cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DWEBSOCKET=true > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null + cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DWEBSOCKET=true > /dev/null && make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] then echo "taos-tools build failure" @@ -152,6 +163,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pip3 install decorator numpy fabric2 psutil pandas faker toml taospy > /dev/null @@ -160,6 +172,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -179,6 +192,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' env: TDENGINE_CLOUD_DSN: ${{ secrets.TDENGINE_CLOUD_DSN }} run: | @@ -200,6 +214,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -219,6 +234,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests @@ -252,6 +268,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | if [ -f ~/taosbenchmark-release-failed.txt ]; then cat ~/taosbenchmark-release-failed.txt diff --git a/.github/workflows/3.0-taosdump-debug.yml b/.github/workflows/3.0-taosdump-debug.yml index 32815f89..3ffafe10 100644 --- a/.github/workflows/3.0-taosdump-debug.yml +++ b/.github/workflows/3.0-taosdump-debug.yml @@ -63,6 +63,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: repository: 'taosdata/TDengine' @@ -74,6 +75,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo timedatectl set-timezone Asia/Shanghai sudo date @@ -83,6 +85,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -92,11 +95,12 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd TDengine mkdir debug && cd debug cmake .. -DBUILD_HTTP=false > /dev/null - make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + make -j2 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] then echo "TDengine build failure" @@ -108,6 +112,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: sudo apt-get install lzma-dev liblzma-dev libjansson-dev libz-dev zlib1g libsnappy-dev pkg-config lcov expect -y > /dev/null - name: Checkout taos-tools to PR number @@ -123,11 +128,12 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DTOOLS_SANITIZE=true > /dev/null - make -j8 > /dev/null && sudo make install > /dev/null + make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] then echo "taos-tools build failure" @@ -139,6 +145,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pip3 install decorator numpy fabric2 psutil pandas faker toml taospy > /dev/null @@ -147,6 +154,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests sudo pkill -9 taosadapter || : @@ -175,6 +183,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests sudo pkill -9 taosadapter || : @@ -204,6 +213,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | if [ -f ~/taosdump-failed.txt ]; then cat ~/taosdump-failed.txt; diff --git a/.github/workflows/3.0-taosdump-release-ws.yml b/.github/workflows/3.0-taosdump-release-ws.yml index f2644566..5133622c 100644 --- a/.github/workflows/3.0-taosdump-release-ws.yml +++ b/.github/workflows/3.0-taosdump-release-ws.yml @@ -63,6 +63,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: repository: 'taosdata/TDengine' @@ -74,6 +75,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo timedatectl set-timezone Asia/Shanghai sudo date @@ -83,6 +85,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -92,6 +95,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable @@ -118,12 +122,13 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd TDengine mkdir debug cd debug cmake .. -DBUILD_HTTP=false -DWEBSOCKET=true > /dev/null - make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + make -j2 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] then echo "TDengine build failure" @@ -143,6 +148,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: sudo apt-get install lzma-dev liblzma-dev libjansson-dev libz-dev zlib1g libsnappy-dev pkg-config lcov expect -y > /dev/null - name: Checkout taos-tools to PR number @@ -158,11 +164,12 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DWEBSOCKET=true > /dev/null - make -j8 > /dev/null && sudo make install > /dev/null + make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] then echo "taos-tools build failure" @@ -174,6 +181,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pip3 install decorator numpy fabric2 psutil pandas faker toml taospy > /dev/null @@ -182,6 +190,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests sudo pkill -9 taosadapter || : @@ -203,6 +212,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd debug ./build/bin/taosdump -C https://not-exist.com:80/cloud -D test ||: @@ -214,6 +224,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | if [ -f ~/taosdump-failed.txt ]; then if [ -f ~/taosa.log ]; then diff --git a/.github/workflows/3.0-taosdump-release.yml b/.github/workflows/3.0-taosdump-release.yml index 9118672c..d98cef1e 100644 --- a/.github/workflows/3.0-taosdump-release.yml +++ b/.github/workflows/3.0-taosdump-release.yml @@ -63,6 +63,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: repository: 'taosdata/TDengine' @@ -74,6 +75,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo timedatectl set-timezone Asia/Shanghai sudo date @@ -83,6 +85,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/setup-go@v3 with: go-version: 1.17 @@ -92,11 +95,12 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd TDengine mkdir debug && cd debug cmake .. -DBUILD_HTTP=false > /dev/null - make -j8 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug + make -j2 > /dev/null && sudo make install > /dev/null && cd .. && rm -rf debug if [[ ! -f /usr/local/taos/bin/taosd ]] || [[ ! -f /usr/local/taos/bin/taosadapter ]] then echo "TDengine build failure" @@ -108,6 +112,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: sudo apt-get install lzma-dev liblzma-dev libjansson-dev libz-dev zlib1g libsnappy-dev pkg-config lcov expect -y > /dev/null - name: Checkout taos-tools to PR number @@ -123,11 +128,12 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | mkdir debug ||: cd debug cmake .. -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} > /dev/null - make -j8 > /dev/null && sudo make install > /dev/null + make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] then echo "taos-tools build failure" @@ -139,6 +145,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | pip3 install decorator numpy fabric2 psutil pandas faker toml taospy > /dev/null @@ -147,6 +154,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests sudo pkill -9 taosadapter || : @@ -163,6 +171,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd tests sudo pkill -9 taosadapter || : @@ -192,6 +201,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | if [ -f ~/taosdump-failed.txt ]; then cat ~/taosdump-failed.txt diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml index 59489e82..0d782e0c 100644 --- a/.github/workflows/3.0-windows-build.yml +++ b/.github/workflows/3.0-windows-build.yml @@ -60,6 +60,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions/checkout@v2 with: repository: 'taosdata/TDengine' @@ -71,6 +72,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | choco install jom gawk grep -y cd TDengine @@ -82,6 +84,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd TDengine/debug cmake .. -G "NMake Makefiles JOM" -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} && jom -j 8 && jom install @@ -106,6 +109,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -116,6 +120,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | cd TDengine/debug cmake .. -G "NMake Makefiles JOM" -DBUILD_HTTP=false -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=${{env.TOOLS_BUILD_TYPE}} -DWEBSOCKET=true && jom -j8 && jom install @@ -125,6 +130,7 @@ jobs: #(steps.changed-files-specific.outputs.any_changed == 'true' #&& github.event_name == 'pull_request') #|| github.event_name == 'push' + #|| github.event_name == 'schedule' # run: | #cd TDengine/debug diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a8958d8c..77778846 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -67,7 +67,7 @@ jobs: cd TDengine || exit 1 mkdir debug || exit 1 cd debug || exit 1 - cmake .. > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null + cmake .. > /dev/null && make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosd ]] then echo "TDengine build failure" @@ -80,7 +80,7 @@ jobs: mkdir debug || exit 1 cd debug || exit 1 - cmake .. > /dev/null && make -j8 > /dev/null && sudo make install > /dev/null + cmake .. > /dev/null && make -j2 > /dev/null && sudo make install > /dev/null if [[ ! -f /usr/local/taos/bin/taosdump ]] || [[ ! -f /usr/local/taos/bin/taosBenchmark ]] then echo "taos-tools build failure" diff --git a/.github/workflows/cppcheck.yml b/.github/workflows/cppcheck.yml index c7a5e870..6b8bb56a 100644 --- a/.github/workflows/cppcheck.yml +++ b/.github/workflows/cppcheck.yml @@ -53,6 +53,7 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | sudo apt update -y > /dev/null sudo apt install -y cppcheck @@ -62,5 +63,6 @@ jobs: (steps.changed-files-specific.outputs.any_changed == 'true' && github.event_name == 'pull_request') || github.event_name == 'push' + || github.event_name == 'schedule' run: | for f in `find src inc -type f -name "*.[c,h]" ! -path "src/benchUtilDs.c"`; do cppcheck $f || exit $?; done diff --git a/tests/taosbenchmark/commandline-partial-col-numpy.py b/tests/taosbenchmark/commandline-partial-col-numpy.py index fe384bbc..da6869a9 100644 --- a/tests/taosbenchmark/commandline-partial-col-numpy.py +++ b/tests/taosbenchmark/commandline-partial-col-numpy.py @@ -63,7 +63,7 @@ def run(self): tdSql.query("select * from test.meters") dbresult = tdSql.queryResult for i in range(len(dbresult[0])): - if i in (1, 2) and dbresult[0][i] == None: + if i in (1, 2) and dbresult[0][i] is None: tdLog.exit("result[0][%d] is NULL, which should not be" % i) else: tdLog.info("result[0][{0}] is {1}".format(i, dbresult[0][i])) diff --git a/tests/taosbenchmark/commandline-single-table.py b/tests/taosbenchmark/commandline-single-table.py index d37e290e..7fe1d097 100644 --- a/tests/taosbenchmark/commandline-single-table.py +++ b/tests/taosbenchmark/commandline-single-table.py @@ -11,7 +11,6 @@ # -*- coding: utf-8 -*- import os -import subprocess from util.log import * from util.cases import * diff --git a/tests/taosbenchmark/query_json-with-error-sqlfile.py b/tests/taosbenchmark/query_json-with-error-sqlfile.py index 4fa5e7e7..b5f30981 100644 --- a/tests/taosbenchmark/query_json-with-error-sqlfile.py +++ b/tests/taosbenchmark/query_json-with-error-sqlfile.py @@ -10,9 +10,7 @@ ################################################################### # -*- coding: utf-8 -*- -import ast import os -import re from util.log import * from util.cases import * diff --git a/tests/taosbenchmark/reuse-exist-stb.py b/tests/taosbenchmark/reuse-exist-stb.py index 64eebbf5..cc49570a 100644 --- a/tests/taosbenchmark/reuse-exist-stb.py +++ b/tests/taosbenchmark/reuse-exist-stb.py @@ -10,9 +10,7 @@ ################################################################### # -*- coding: utf-8 -*- -import ast import os -import re from util.log import * from util.cases import * diff --git a/tests/taosbenchmark/taosc_insert_alltypes_json-partial-col.py b/tests/taosbenchmark/taosc_insert_alltypes_json-partial-col.py index 5b618679..2db38187 100644 --- a/tests/taosbenchmark/taosc_insert_alltypes_json-partial-col.py +++ b/tests/taosbenchmark/taosc_insert_alltypes_json-partial-col.py @@ -58,10 +58,6 @@ def getPath(self, tool="taosBenchmark"): return paths[0] def run(self): - tdSql.query("select client_version()") - client_ver = "".join(tdSql.queryResult[0]) - major_ver = client_ver.split(".")[0] - binPath = self.getPath() cmd = ( "%s -f ./taosbenchmark/json/taosc_insert_alltypes-partial-col.json" @@ -114,7 +110,7 @@ def run(self): tdSql.query("select c0,c1,c2 from db.stb limit 1") dbresult = tdSql.queryResult for i in range(len(dbresult[0])): - if i in (0, 1) and dbresult[0][i] == None: + if i in (0, 1) and dbresult[0][i] is None: tdLog.exit("result[0][%d] is NULL, which should not be" % i) else: tdLog.info("result[0][{0}] is {1}".format(i, dbresult[0][i])) diff --git a/tests/taosbenchmark/taosdemoTestInsertWithJsonStmt-otherPara.py b/tests/taosbenchmark/taosdemoTestInsertWithJsonStmt-otherPara.py index 9ee9cb16..6aefcb56 100644 --- a/tests/taosbenchmark/taosdemoTestInsertWithJsonStmt-otherPara.py +++ b/tests/taosbenchmark/taosdemoTestInsertWithJsonStmt-otherPara.py @@ -11,7 +11,6 @@ # -*- coding: utf-8 -*- -import sys import os from util.log import * from util.cases import * @@ -216,7 +215,6 @@ def run(self): ) # child_table_exists = yes, auto_create_table varies = yes tdSql.checkRows(20) - testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") def stop(self): diff --git a/tests/taosbenchmark/taosdemoTestQueryWithJson.py b/tests/taosbenchmark/taosdemoTestQueryWithJson.py index e6bb1c6f..b3341f21 100644 --- a/tests/taosbenchmark/taosdemoTestQueryWithJson.py +++ b/tests/taosbenchmark/taosdemoTestQueryWithJson.py @@ -11,7 +11,6 @@ # -*- coding: utf-8 -*- -import sys import os from util.log import * from util.cases import * diff --git a/tests/taosbenchmark/v3/commandline-vgroups.py b/tests/taosbenchmark/v3/commandline-vgroups.py index e2a7a90e..4292e227 100644 --- a/tests/taosbenchmark/v3/commandline-vgroups.py +++ b/tests/taosbenchmark/v3/commandline-vgroups.py @@ -11,7 +11,6 @@ # -*- coding: utf-8 -*- import os -import subprocess from util.log import * from util.cases import * diff --git a/tests/taosbenchmark/v3/default_tmq_json.py b/tests/taosbenchmark/v3/default_tmq_json.py index 7b31818e..f3b0dc0c 100644 --- a/tests/taosbenchmark/v3/default_tmq_json.py +++ b/tests/taosbenchmark/v3/default_tmq_json.py @@ -10,7 +10,8 @@ ################################################################### # -*- coding: utf-8 -*- -import os, signal +# import os, signal +import os from time import sleep from util.log import * from util.cases import * @@ -61,24 +62,25 @@ def run(self): tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") - + tdSql.execute("alter database db WAL_RETENTION_PERIOD 3600000") - + cmd = "%s -f ./taosbenchmark/json/tmq.json " % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) - time.sleep(15) -# try: -# for line in os.popen("ps ax | grep taosBenchmark | grep -v grep"): -# fields = line.split() - -# pid = fields[0] - -# os.kill(int(pid), signal.SIGINT) -# time.sleep(3) -# print("taosBenchmark be killed on purpose") -# except: -# tdLog.exit("failed to kill taosBenchmark") + sleep(15) + + # try: + # for line in os.popen("ps ax | grep taosBenchmark | grep -v grep"): + # fields = line.split() + + # pid = fields[0] + + # os.kill(int(pid), signal.SIGINT) + # time.sleep(3) + # print("taosBenchmark be killed on purpose") + # except: + # tdLog.exit("failed to kill taosBenchmark") def stop(self): tdSql.close() diff --git a/tests/taosbenchmark/v3/query_json-kill-slow-query.py b/tests/taosbenchmark/v3/query_json-kill-slow-query.py index 9b37abb5..cc2075c8 100644 --- a/tests/taosbenchmark/v3/query_json-kill-slow-query.py +++ b/tests/taosbenchmark/v3/query_json-kill-slow-query.py @@ -11,7 +11,6 @@ # -*- coding: utf-8 -*- import os -import re import subprocess from util.log import * From 411604c546728f9044299d224386ee86f12935fb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 May 2023 17:32:28 +0800 Subject: [PATCH 16/25] fix: cancel and no wait thread exit --- src/benchInsert.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/benchInsert.c b/src/benchInsert.c index d66c3b1d..be34a76d 100644 --- a/src/benchInsert.c +++ b/src/benchInsert.c @@ -1002,7 +1002,7 @@ static int startMultiThreadCreateChildTable( pthread_create(pids + i, NULL, createTable, pThreadInfo); } - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { + for (int i = 0; (i < threads && pids[i] !=0); i++) { pthread_join(pids[i], NULL); } @@ -3179,8 +3179,13 @@ static int startMultiThreadInsertData(SDataBase* database, int64_t start = toolsGetTimestampUs(); // wait threads - for (int i = 0; (i < threads && !g_arguments->terminate); i++) { - pthread_join(pids[i], NULL); + for (int i = 0; (i < threads); i++) { + if(pids[i] != 0) { + infoPrint(" pthread_join %d ...\n", i); + pthread_join(pids[i], NULL); + } else { + infoPrint(" pthread_join %d is null , not wait.\n", i); + } } int64_t end = toolsGetTimestampUs()+1; From d39a301dd6074630df7b9fc379ec9639b3c623bd Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 May 2023 15:29:28 +0800 Subject: [PATCH 17/25] chore: taos tools mac (#658) * fix: taos-tools build on mac * chore: add ../.github/workflows/3.0-macos-release.yml * fix: 3.0-macos-release.yml * chore: taos tools install script for mac * chore: add packaging/tools for action * fix: include typo * fix: update version --- VERSION | 6 +++--- src/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 64ca3db9..94933678 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ -taosbenchmark-3.2.0 -taosdump-2.5.0 -2.5.0 +taosbenchmark-3.2.1 +taosdump-2.5.1 +2.5.1 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f78a52e8..07174fb7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -317,7 +317,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin INCLUDE_DIRECTORIES(/opt/homebrew/opt/argp-standalone/include/) ELSE () SET_PROPERTY(TARGET argp PROPERTY IMPORTED_LOCATION "/usr/local/lib/libargp.a") - INCLUDE_DIRECTORIES(/usr/local/include/include/) + INCLUDE_DIRECTORIES(/usr/local/include/) ENDIF () TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${WEBSOCKET_LINK_FLAGS}) ElSE () From 1b68fc4fe1d104aedc204da1b5cbe33fb90651b0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 23 May 2023 14:03:21 +0800 Subject: [PATCH 18/25] fix: commit fix code --- src/benchQuery.c | 295 +++++++++++++++++++++++++++-------------------- src/benchUtil.c | 5 + 2 files changed, 177 insertions(+), 123 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 87096796..9da0241d 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -184,9 +184,13 @@ static void *specifiedTableQuery(void *sarg) { } while (index < queryTimes) { - if (g_queryInfo.specifiedQueryInfo.queryInterval - && (et - st) < - (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval*1000) { + // check cancel + if (g_arguments->terminate) { + return NULL; + } + + if (g_queryInfo.specifiedQueryInfo.queryInterval && + (et - st) < (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval * 1000) { toolsMsleep((int32_t)( g_queryInfo.specifiedQueryInfo.queryInterval*1000 - (et - st))); // ms @@ -392,144 +396,187 @@ static int multi_thread_super_table_query(uint16_t iface, char* dbName) { return ret; } +// free g_queryInfo.specailQueryInfo memory , can re-call +void freeSpecialQueryInfo() { + // can re-call + if (g_queryInfo.specifiedQueryInfo.sqls == NULL) { + return; + } + + // loop free each item memory + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqls->size; ++i) { + SSQL *sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); + tmfree(sql->command); + tmfree(sql->delay_list); + } + + // free Array + benchArrayDestroy(g_queryInfo.specifiedQueryInfo.sqls); + g_queryInfo.specifiedQueryInfo.sqls = NULL; +} + + static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { pthread_t * pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from specify table int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqls->size; - if ((nSqlCount > 0) && (nConcurrent > 0)) { - pids = benchCalloc(1, nConcurrent*nSqlCount*sizeof(pthread_t), false); - infos = benchCalloc(1, nConcurrent*nSqlCount*sizeof(threadInfo), false); - for (uint64_t i = 0; i < nSqlCount; i++) { - SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); - for (int j = 0; j < nConcurrent; j++) { - uint64_t seq = i * nConcurrent + j; - threadInfo *pThreadInfo = infos + seq; - pThreadInfo->threadID = (int)seq; - pThreadInfo->querySeq = i; - if (iface == REST_IFACE) { - int sockfd = createSockFd(); - // int iMode = 1; - // ioctl(sockfd, FIONBIO, &iMode); - if (sockfd < 0) { - tmfree((char *)pids); - tmfree((char *)infos); - return -1; - } - pThreadInfo->sockfd = sockfd; - } else { - pThreadInfo->conn = initBenchConn(); - if (pThreadInfo->conn == NULL) { - destroySockFd(pThreadInfo->sockfd); - tmfree((char *)pids); - tmfree((char *)infos); - return -1; - } + + // check invaid + if(nSqlCount == 0 || nConcurrent == 0 ) { + if(nSqlCount == 0) + errorPrint(" query sql count is %d. must set query sqls. \n", nSqlCount); + if(nConcurrent == 0) + errorPrint(" concurrent is %d , specified_table_query->concurrent must not zero. \n", nConcurrent); + return -1; + } + + // malloc funciton global memory + pids = benchCalloc(1, nConcurrent * sizeof(pthread_t), false); + infos = benchCalloc(1, nConcurrent * sizeof(threadInfo), false); + + bool exeError = false; + for (uint64_t i = 0; i < nSqlCount; i++) { + // reset + memset(pids, 0, nConcurrent * sizeof(pthread_t)); + memset(infos, 0, nConcurrent * sizeof(threadInfo)); + + // get execute sql + SSQL *sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); + + // create threads + for (int j = 0; j < nConcurrent; j++) { + threadInfo *pThreadInfo = infos + j; + uint64_t seq = i * nConcurrent + j; + pThreadInfo->threadID = (int)seq; + pThreadInfo->querySeq = i; + if (iface == REST_IFACE) { + int sockfd = createSockFd(); + // int iMode = 1; + // ioctl(sockfd, FIONBIO, &iMode); + if (sockfd < 0) { + exeError = true; + + break; + } + pThreadInfo->sockfd = sockfd; + } else { + pThreadInfo->conn = initBenchConn(); + if (pThreadInfo->conn == NULL) { + destroySockFd(pThreadInfo->sockfd); + exeError = true; + break; } + } - pthread_create(pids + seq, NULL, specifiedTableQuery, - pThreadInfo); - } - for (int j = 0; j < nConcurrent; j++) { - uint64_t seq = i * nConcurrent + j; - pthread_join(pids[seq], NULL); - threadInfo *pThreadInfo = infos + seq; - if (iface == REST_IFACE) { + pthread_create(pids + seq, NULL, specifiedTableQuery, pThreadInfo); + } + + // if failed, set termainte flag true like ctrl+c exit + if (exeError) { + errorPrint(" i=%d create thread occur error, so wait exit ...\n", i); + g_arguments->terminate = true; + } + + // wait threads execute finished one by one + for (int j = 0; j < nConcurrent && pids + j > 0 ; j++) { + pthread_join(pids[j], NULL); + threadInfo *pThreadInfo = infos + j; + if (iface == REST_IFACE) { #ifdef WINDOWS - closesocket(pThreadInfo->sockfd); - WSACleanup(); + closesocket(pThreadInfo->sockfd); + WSACleanup(); #else - close(pThreadInfo->sockfd); + close(pThreadInfo->sockfd); #endif - } else { - closeBenchConn(pThreadInfo->conn); - } - if (g_fail) { - tmfree(pThreadInfo->query_delay_list); - } - } - - if (g_fail) { - tmfree((char *)pids); - tmfree((char *)infos); - return -1; - } - uint64_t query_times = g_queryInfo.specifiedQueryInfo.queryTimes; - uint64_t totalQueryTimes = query_times * nConcurrent; - double avg_delay = 0.0; - for (int j = 0; j < nConcurrent; j++) { - uint64_t seq = i * nConcurrent + j; - threadInfo *pThreadInfo = infos + seq; - avg_delay += pThreadInfo->avg_delay; - for (uint64_t k = 0; - k < g_queryInfo.specifiedQueryInfo.queryTimes; k++) { - sql->delay_list[j*query_times + k] = - pThreadInfo->query_delay_list[k]; - } - tmfree(pThreadInfo->query_delay_list); - } - avg_delay /= nConcurrent; - qsort(sql->delay_list, g_queryInfo.specifiedQueryInfo.queryTimes, - sizeof(uint64_t), compare); - infoPrintNoTimestamp("complete query with %d threads and %"PRIu64 - " query delay " - "avg: \t%.6fs " - "min: \t%.6fs " - "max: \t%.6fs " - "p90: \t%.6fs " - "p95: \t%.6fs " - "p99: \t%.6fs " - "SQL command: %s" - "\n", - nConcurrent, query_times, - avg_delay/1E6, /* avg */ - sql->delay_list[0]/1E6, /* min */ - sql->delay_list[totalQueryTimes - 1]/1E6, /* max */ - /* p90 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.90)]/1E6, - /* p95 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.95)]/1E6, - /* p99 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)]/1E6, - sql->command); - infoPrintNoTimestampToFile(g_arguments->fpOfInsertResult, - "complete query with %d threads and %"PRIu64 - " query delay " - "avg: \t%.6fs " - "min: \t%.6fs " - "max: \t%.6fs " - "p90: \t%.6fs " - "p95: \t%.6fs " - "p99: \t%.6fs " - "SQL command: %s" - "\n", - nConcurrent, query_times, - avg_delay/1E6, /* avg */ - sql->delay_list[0]/1E6, /* min */ - sql->delay_list[totalQueryTimes - 1]/1E6, /* max */ - /* p90 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.90)]/1E6, - /* p95 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.95)]/1E6, - /* p99 */ - sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)]/1E6, - sql->command); + } else { + closeBenchConn(pThreadInfo->conn); + } + + // need exit in loop + if (g_fail || g_arguments->terminate) { + // free BArray + benchArrayDestroy(pThreadInfo->query_delay_list); + pThreadInfo->query_delay_list = NULL; + } + } + + // cancel or need exit check + if (g_fail || g_arguments->terminate) { + // free current funciton malloc memory + tmfree((char *)pids); + tmfree((char *)infos); + // free global + freeSpecialQueryInfo(); + return -1; } - } else { - return 0; + + // execute successfully + uint64_t query_times = g_queryInfo.specifiedQueryInfo.queryTimes; + uint64_t totalQueryTimes = query_times * nConcurrent; + double avg_delay = 0.0; + for (int j = 0; j < nConcurrent; j++) { + threadInfo *pThreadInfo = infos + j; + avg_delay += pThreadInfo->avg_delay; + for (uint64_t k = 0; k < g_queryInfo.specifiedQueryInfo.queryTimes; k++) { + sql->delay_list[j * query_times + k] = pThreadInfo->query_delay_list[k]; + } + + // free BArray + benchArrayDestroy(pThreadInfo->query_delay_list); + pThreadInfo->query_delay_list = NULL; + } + avg_delay /= nConcurrent; + qsort(sql->delay_list, g_queryInfo.specifiedQueryInfo.queryTimes, sizeof(uint64_t), compare); + infoPrintNoTimestamp("complete query with %d threads and %" PRIu64 + " query delay " + "avg: \t%.6fs " + "min: \t%.6fs " + "max: \t%.6fs " + "p90: \t%.6fs " + "p95: \t%.6fs " + "p99: \t%.6fs " + "SQL command: %s" + "\n", + nConcurrent, query_times, avg_delay / 1E6, /* avg */ + sql->delay_list[0] / 1E6, /* min */ + sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ + /* p90 */ + sql->delay_list[(uint64_t)(totalQueryTimes * 0.90)] / 1E6, + /* p95 */ + sql->delay_list[(uint64_t)(totalQueryTimes * 0.95)] / 1E6, + /* p99 */ + sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)] / 1E6, sql->command); + infoPrintNoTimestampToFile(g_arguments->fpOfInsertResult, + "complete query with %d threads and %" PRIu64 + " query delay " + "avg: \t%.6fs " + "min: \t%.6fs " + "max: \t%.6fs " + "p90: \t%.6fs " + "p95: \t%.6fs " + "p99: \t%.6fs " + "SQL command: %s" + "\n", + nConcurrent, query_times, avg_delay / 1E6, /* avg */ + sql->delay_list[0] / 1E6, /* min */ + sql->delay_list[totalQueryTimes - 1] / 1E6, /* max */ + /* p90 */ + sql->delay_list[(uint64_t)(totalQueryTimes * 0.90)] / 1E6, + /* p95 */ + sql->delay_list[(uint64_t)(totalQueryTimes * 0.95)] / 1E6, + /* p99 */ + sql->delay_list[(uint64_t)(totalQueryTimes * 0.99)] / 1E6, sql->command); } g_queryInfo.specifiedQueryInfo.totalQueried = g_queryInfo.specifiedQueryInfo.queryTimes * nConcurrent; tmfree((char *)pids); tmfree((char *)infos); - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqls->size; ++i) { - SSQL * sql = benchArrayGet(g_queryInfo.specifiedQueryInfo.sqls, i); - tmfree(sql->command); - tmfree(sql->delay_list); - } - benchArrayDestroy(g_queryInfo.specifiedQueryInfo.sqls); + + // free specialQueryInfo + freeSpecialQueryInfo(); return 0; } @@ -587,6 +634,8 @@ static int multi_thread_specified_mixed_query(uint16_t iface, char* dbName) { pThreadInfo->query_delay_list->size); total_delay += pThreadInfo->total_delay; tmfree(pThreadInfo->query_delay_list); + pThreadInfo->query_delay_list = NULL; + if (iface == REST_IFACE) { #ifdef WINDOWS closesocket(pThreadInfo->sockfd); diff --git a/src/benchUtil.c b/src/benchUtil.c index 9cc86ca9..aa196497 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -1166,6 +1166,11 @@ static void closeSockFd(int sockfd) { } void destroySockFd(int sockfd) { + // check valid + if (sockfd < 0) { + return; + } + // shutdown the connection since no more data will be sent int result; result = shutdown(sockfd, SHUT_WR); From 31d3bb12c2416351c4330f5a2ac62576495f9393 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 23 May 2023 14:16:49 +0800 Subject: [PATCH 19/25] fix: query_delay_list in special table query is int64* --- src/benchQuery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 9da0241d..ad91c99f 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -497,7 +497,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { // need exit in loop if (g_fail || g_arguments->terminate) { // free BArray - benchArrayDestroy(pThreadInfo->query_delay_list); + tmfree(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; } } @@ -524,7 +524,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { } // free BArray - benchArrayDestroy(pThreadInfo->query_delay_list); + tmfree(pThreadInfo->query_delay_list); pThreadInfo->query_delay_list = NULL; } avg_delay /= nConcurrent; From 56ee7eb9f3dd05f7494d3a290d9732543be91f06 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 23 May 2023 14:26:21 +0800 Subject: [PATCH 20/25] fix: check pids[j] is zero --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index ad91c99f..5bdf1636 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -480,7 +480,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { } // wait threads execute finished one by one - for (int j = 0; j < nConcurrent && pids + j > 0 ; j++) { + for (int j = 0; (j < nConcurrent && pids[j] > 0) ; j++) { pthread_join(pids[j], NULL); threadInfo *pThreadInfo = infos + j; if (iface == REST_IFACE) { From bbb17e64c25443b3ec44577cd9f1c3e68ec36ac5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 23 May 2023 15:08:14 +0800 Subject: [PATCH 21/25] fix: conn connect order --- src/benchQuery.c | 1 + src/benchUtil.c | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 5bdf1636..2f47ebbe 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -492,6 +492,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { #endif } else { closeBenchConn(pThreadInfo->conn); + pThreadInfo->conn = NULL; } // need exit in loop diff --git a/src/benchUtil.c b/src/benchUtil.c index aa196497..69aeb47c 100644 --- a/src/benchUtil.c +++ b/src/benchUtil.c @@ -310,12 +310,17 @@ SBenchConn* initBenchConn() { } void closeBenchConn(SBenchConn* conn) { + if(conn == NULL) + return ; #ifdef WEBSOCKET if (g_arguments->websocket) { ws_close(conn->taos_ws); } else { #endif - taos_close(conn->taos); + if(conn->taos) { + taos_close(conn->taos); + conn->taos = NULL; + } if (conn->ctaos) { taos_close(conn->ctaos); conn->ctaos = NULL; From bae299c8cdfb59f43bf080a8ec7c3b40adec943a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 23 May 2023 19:48:00 +0800 Subject: [PATCH 22/25] fix: pthread_create pass first args error --- src/benchQuery.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 2f47ebbe..40ee1d76 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -426,7 +426,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { // check invaid if(nSqlCount == 0 || nConcurrent == 0 ) { if(nSqlCount == 0) - errorPrint(" query sql count is %d. must set query sqls. \n", nSqlCount); + errorPrint(" query sql count is %ld. must set query sqls. \n", nSqlCount); if(nConcurrent == 0) errorPrint(" concurrent is %d , specified_table_query->concurrent must not zero. \n", nConcurrent); return -1; @@ -448,8 +448,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { // create threads for (int j = 0; j < nConcurrent; j++) { threadInfo *pThreadInfo = infos + j; - uint64_t seq = i * nConcurrent + j; - pThreadInfo->threadID = (int)seq; + pThreadInfo->threadID = i * nConcurrent + j; pThreadInfo->querySeq = i; if (iface == REST_IFACE) { int sockfd = createSockFd(); @@ -470,7 +469,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { } } - pthread_create(pids + seq, NULL, specifiedTableQuery, pThreadInfo); + pthread_create(pids + j, NULL, specifiedTableQuery, pThreadInfo); } // if failed, set termainte flag true like ctrl+c exit From f06499c248867941c1616eaba10b2925299426e1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 24 May 2023 10:06:08 +0800 Subject: [PATCH 23/25] fix: build warning --- src/benchQuery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 40ee1d76..3176a122 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -474,7 +474,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { // if failed, set termainte flag true like ctrl+c exit if (exeError) { - errorPrint(" i=%d create thread occur error, so wait exit ...\n", i); + errorPrint(" i=%ld create thread occur error, so wait exit ...\n", i); g_arguments->terminate = true; } From d761edefecc88a34b504ba79282885c956243526 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 24 May 2023 11:39:37 +0800 Subject: [PATCH 24/25] fix: int64 uint64 print format --- src/benchQuery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 3176a122..97c84b9a 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -426,7 +426,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { // check invaid if(nSqlCount == 0 || nConcurrent == 0 ) { if(nSqlCount == 0) - errorPrint(" query sql count is %ld. must set query sqls. \n", nSqlCount); + errorPrint(" query sql count is %" PRIu64 ". must set query sqls. \n", nSqlCount); if(nConcurrent == 0) errorPrint(" concurrent is %d , specified_table_query->concurrent must not zero. \n", nConcurrent); return -1; @@ -474,7 +474,7 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { // if failed, set termainte flag true like ctrl+c exit if (exeError) { - errorPrint(" i=%ld create thread occur error, so wait exit ...\n", i); + errorPrint(" i=%" PRIu64 " create thread occur error, so wait exit ...\n", i); g_arguments->terminate = true; } From 3c11f3a6ec48600939366fddaed83d08842c57a8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Wed, 24 May 2023 17:23:23 +0800 Subject: [PATCH 25/25] fix: count zero is set not error --- src/benchQuery.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/benchQuery.c b/src/benchQuery.c index 97c84b9a..63816b81 100644 --- a/src/benchQuery.c +++ b/src/benchQuery.c @@ -426,10 +426,10 @@ static int multi_thread_specified_table_query(uint16_t iface, char* dbName) { // check invaid if(nSqlCount == 0 || nConcurrent == 0 ) { if(nSqlCount == 0) - errorPrint(" query sql count is %" PRIu64 ". must set query sqls. \n", nSqlCount); + infoPrint(" query sql count is %" PRIu64 ". must set query sqls. \n", nSqlCount); if(nConcurrent == 0) - errorPrint(" concurrent is %d , specified_table_query->concurrent must not zero. \n", nConcurrent); - return -1; + infoPrint(" concurrent is %d , specified_table_query->concurrent must not zero. \n", nConcurrent); + return 0; } // malloc funciton global memory