From daef22ab3a083a0700addf2c41a70a2d2dc1ece9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 15 Aug 2022 17:44:38 +0800 Subject: [PATCH] refactor(tmq): put write logic into taosx.c --- examples/c/tmq.c | 34 +- include/client/taos.h | 40 +- include/util/taoserror.h | 1 + include/util/tref.h | 4 +- source/client/src/clientMain.c | 1 + source/client/src/taosx.c | 1629 +++++++++++++++++++++++++++ source/client/src/tmq.c | 1913 ++++---------------------------- source/util/src/terror.c | 1 + 8 files changed, 1853 insertions(+), 1770 deletions(-) create mode 100644 source/client/src/taosx.c diff --git a/examples/c/tmq.c b/examples/c/tmq.c index fc34915fe754..19adaad116ef 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -45,10 +45,9 @@ static int32_t msg_process(TAOS_RES* msg) { int32_t numOfFields = taos_field_count(msg); int32_t* length = taos_fetch_lengths(msg); int32_t precision = taos_result_precision(msg); - const char* tbName = tmq_get_table_name(msg); rows++; taos_print_row(buf, row, fields, numOfFields); - printf("row content from %s: %s\n", (tbName != NULL ? tbName : "table null"), buf); + printf("row content: %s\n", buf); } return rows; @@ -167,7 +166,7 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1"); + pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3, tbname from tmqdb.stb where c1 > 1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topicname, reason:%s\n", taos_errstr(pRes)); return -1; @@ -199,9 +198,7 @@ tmq_t* build_consumer() { if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); if (TMQ_CONF_OK != code) return NULL; - code = tmq_conf_set(conf, "experimental.snapshot.enable", "true"); - if (TMQ_CONF_OK != code) return NULL; - code = tmq_conf_set(conf, "msg.with.table.name", "true"); + code = tmq_conf_set(conf, "experimental.snapshot.enable", "false"); if (TMQ_CONF_OK != code) return NULL; tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -220,14 +217,7 @@ tmq_list_t* build_topic_list() { return topicList; } -void basic_consume_loop(tmq_t* tmq, tmq_list_t* topicList) { - int32_t code; - - if ((code = tmq_subscribe(tmq, topicList))) { - fprintf(stderr, "%% Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); - return; - } - +void basic_consume_loop(tmq_t* tmq) { int32_t totalRows = 0; int32_t msgCnt = 0; int32_t timeout = 5000; @@ -237,8 +227,8 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topicList) { msgCnt++; totalRows += msg_process(tmqmsg); taos_free_result(tmqmsg); - /*} else {*/ - /*break;*/ + } else { + break; } } @@ -267,14 +257,12 @@ int main(int argc, char* argv[]) { return -1; } - basic_consume_loop(tmq, topic_list); - - code = tmq_unsubscribe(tmq); - if (code) { - fprintf(stderr, "%% Failed to unsubscribe: %s\n", tmq_err2str(code)); - } else { - fprintf(stderr, "%% unsubscribe\n"); + if ((code = tmq_subscribe(tmq, topic_list))) { + fprintf(stderr, "%% Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); } + tmq_list_destroy(topic_list); + + basic_consume_loop(tmq); code = tmq_consumer_close(tmq); if (code) { diff --git a/include/client/taos.h b/include/client/taos.h index dd7266bd96dc..f260b84f4aaf 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -131,10 +131,10 @@ DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); -DLL_EXPORT void taos_close(TAOS *taos); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +DLL_EXPORT void taos_close(TAOS *taos); -const char *taos_data_type(int type); +const char *taos_data_type(int type); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); @@ -244,33 +244,37 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ +DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); + +/* ------------------------------ TAOSX -----------------------------------*/ +// note: following apis are unstable enum tmq_res_t { TMQ_RES_INVALID = -1, TMQ_RES_DATA = 1, TMQ_RES_TABLE_META = 2, }; -typedef struct tmq_raw_data{ - void* raw; +typedef struct tmq_raw_data { + void *raw; uint32_t raw_len; uint16_t raw_type; } tmq_raw_data; typedef enum tmq_res_t tmq_res_t; -DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); -DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); -DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char* tbname); -DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); -DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed by tmq_free_json_meta -DLL_EXPORT void tmq_free_json_meta(char* jsonMeta); -DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); -DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); -DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); - -/* ------------------------------ TMQ END -------------------------------- */ +DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); +DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); +DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); +DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname); +DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); +// Returning null means error. Returned result need to be freed by tmq_free_json_meta +DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); +DLL_EXPORT void tmq_free_json_meta(char *jsonMeta); + +/* ---------------------------- TAOSX END -------------------------------- */ typedef enum { TSDB_SRV_STATUS_UNAVAILABLE = 0, diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3ca6978156bb..d7ec3697afe6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -622,6 +622,7 @@ int32_t* taosGetErrno(); //tmq #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) #define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) +#define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002) #ifdef __cplusplus } diff --git a/include/util/tref.h b/include/util/tref.h index 7e08bb045b08..c2cc54cb07ac 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -29,11 +29,11 @@ int32_t taosOpenRef(int32_t max, void (*fp)(void *)); // close the reference set, refId is the return value by taosOpenRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately -int32_t taosCloseRef(int32_t refId); +int32_t taosCloseRef(int32_t rsetId); // add ref, p is the pointer to resource or pointer ID // return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately -int64_t taosAddRef(int32_t refId, void *p); +int64_t taosAddRef(int32_t rsetId, void *p); // remove ref, rid is the reference ID returned by taosAddRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0ec724c6d0a3..0e95cd4d999f 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -192,6 +192,7 @@ void taos_free_result(TAOS_RES *res) { if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); + taosMemoryFree(pRsp); } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res; taosMemoryFree(pRspObj->metaRsp.metaRsp); diff --git a/source/client/src/taosx.c b/source/client/src/taosx.c new file mode 100644 index 000000000000..2a88b698140e --- /dev/null +++ b/source/client/src/taosx.c @@ -0,0 +1,1629 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "cJSON.h" +#include "clientInt.h" +#include "clientLog.h" +#include "parser.h" +#include "tdatablock.h" +#include "tdef.h" +#include "tglobal.h" +#include "tmsgtype.h" +#include "tqueue.h" +#include "tref.h" +#include "ttimer.h" + +static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, + int8_t t) { + char* string = NULL; + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { + return string; + } + cJSON* type = cJSON_CreateString("create"); + cJSON_AddItemToObject(json, "type", type); + + // char uid[32] = {0}; + // sprintf(uid, "%"PRIi64, id); + // cJSON* id_ = cJSON_CreateString(uid); + // cJSON_AddItemToObject(json, "id", id_); + cJSON* tableName = cJSON_CreateString(name); + cJSON_AddItemToObject(json, "tableName", tableName); + cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); + cJSON_AddItemToObject(json, "tableType", tableType); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); + + cJSON* columns = cJSON_CreateArray(); + for (int i = 0; i < schemaRow->nCols; i++) { + cJSON* column = cJSON_CreateObject(); + SSchema* s = schemaRow->pSchema + i; + cJSON* cname = cJSON_CreateString(s->name); + cJSON_AddItemToObject(column, "name", cname); + cJSON* ctype = cJSON_CreateNumber(s->type); + cJSON_AddItemToObject(column, "type", ctype); + if (s->type == TSDB_DATA_TYPE_BINARY) { + int32_t length = s->bytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(column, "length", cbytes); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(column, "length", cbytes); + } + cJSON_AddItemToArray(columns, column); + } + cJSON_AddItemToObject(json, "columns", columns); + + cJSON* tags = cJSON_CreateArray(); + for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { + cJSON* tag = cJSON_CreateObject(); + SSchema* s = schemaTag->pSchema + i; + cJSON* tname = cJSON_CreateString(s->name); + cJSON_AddItemToObject(tag, "name", tname); + cJSON* ttype = cJSON_CreateNumber(s->type); + cJSON_AddItemToObject(tag, "type", ttype); + if (s->type == TSDB_DATA_TYPE_BINARY) { + int32_t length = s->bytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(tag, "length", cbytes); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(tag, "length", cbytes); + } + cJSON_AddItemToArray(tags, tag); + } + cJSON_AddItemToObject(json, "tags", tags); + + string = cJSON_PrintUnformatted(json); + cJSON_Delete(json); + return string; +} + +static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { + SMAlterStbReq req = {0}; + cJSON* json = NULL; + char* string = NULL; + + if (tDeserializeSMAlterStbReq(alterData, alterDataLen, &req) != 0) { + goto end; + } + + json = cJSON_CreateObject(); + if (json == NULL) { + goto end; + } + cJSON* type = cJSON_CreateString("alter"); + cJSON_AddItemToObject(json, "type", type); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); + SName name = {0}; + tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + cJSON* tableName = cJSON_CreateString(name.tname); + cJSON_AddItemToObject(json, "tableName", tableName); + cJSON* tableType = cJSON_CreateString("super"); + cJSON_AddItemToObject(json, "tableType", tableType); + + cJSON* alterType = cJSON_CreateNumber(req.alterType); + cJSON_AddItemToObject(json, "alterType", alterType); + switch (req.alterType) { + case TSDB_ALTER_TABLE_ADD_TAG: + case TSDB_ALTER_TABLE_ADD_COLUMN: { + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colType = cJSON_CreateNumber(field->type); + cJSON_AddItemToObject(json, "colType", colType); + + if (field->type == TSDB_DATA_TYPE_BINARY) { + int32_t length = field->bytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } else if (field->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } + break; + } + case TSDB_ALTER_TABLE_DROP_TAG: + case TSDB_ALTER_TABLE_DROP_COLUMN: { + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); + cJSON_AddItemToObject(json, "colName", colName); + break; + } + case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colType = cJSON_CreateNumber(field->type); + cJSON_AddItemToObject(json, "colType", colType); + if (field->type == TSDB_DATA_TYPE_BINARY) { + int32_t length = field->bytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } else if (field->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } + break; + } + case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { + TAOS_FIELD* oldField = taosArrayGet(req.pFields, 0); + TAOS_FIELD* newField = taosArrayGet(req.pFields, 1); + cJSON* colName = cJSON_CreateString(oldField->name); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colNewName = cJSON_CreateString(newField->name); + cJSON_AddItemToObject(json, "colNewName", colNewName); + break; + } + default: + break; + } + string = cJSON_PrintUnformatted(json); + +end: + cJSON_Delete(json); + tFreeSMAltertbReq(&req); + return string; +} + +static char* processCreateStb(SMqMetaRsp* metaRsp) { + SVCreateStbReq req = {0}; + SDecoder coder; + char* string = NULL; + + // decode and process req + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + + if (tDecodeSVCreateStbReq(&coder, &req) < 0) { + goto _err; + } + string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE); + tDecoderClear(&coder); + return string; + +_err: + tDecoderClear(&coder); + return string; +} + +static char* processAlterStb(SMqMetaRsp* metaRsp) { + SVCreateStbReq req = {0}; + SDecoder coder; + char* string = NULL; + + // decode and process req + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + + if (tDecodeSVCreateStbReq(&coder, &req) < 0) { + goto _err; + } + string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); + tDecoderClear(&coder); + return string; + +_err: + tDecoderClear(&coder); + return string; +} + +static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id, uint8_t tagNum) { + char* string = NULL; + SArray* pTagVals = NULL; + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { + return string; + } + cJSON* type = cJSON_CreateString("create"); + cJSON_AddItemToObject(json, "type", type); + // char cid[32] = {0}; + // sprintf(cid, "%"PRIi64, id); + // cJSON* cid_ = cJSON_CreateString(cid); + // cJSON_AddItemToObject(json, "id", cid_); + + cJSON* tableName = cJSON_CreateString(name); + cJSON_AddItemToObject(json, "tableName", tableName); + cJSON* tableType = cJSON_CreateString("child"); + cJSON_AddItemToObject(json, "tableType", tableType); + cJSON* using = cJSON_CreateString(sname); + cJSON_AddItemToObject(json, "using", using); + cJSON* tagNumJson = cJSON_CreateNumber(tagNum); + cJSON_AddItemToObject(json, "tagNum", tagNumJson); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); + + cJSON* tags = cJSON_CreateArray(); + int32_t code = tTagToValArray(pTag, &pTagVals); + if (code) { + goto end; + } + + if (tTagIsJson(pTag)) { + STag* p = (STag*)pTag; + if (p->nTag == 0) { + goto end; + } + char* pJson = parseTagDatatoJson(pTag); + cJSON* tag = cJSON_CreateObject(); + STagVal* pTagVal = taosArrayGet(pTagVals, 0); + + char* ptname = taosArrayGet(tagName, 0); + cJSON* tname = cJSON_CreateString(ptname); + cJSON_AddItemToObject(tag, "name", tname); + // cJSON* cid_ = cJSON_CreateString(""); + // cJSON_AddItemToObject(tag, "cid", cid_); + cJSON* ttype = cJSON_CreateNumber(TSDB_DATA_TYPE_JSON); + cJSON_AddItemToObject(tag, "type", ttype); + cJSON* tvalue = cJSON_CreateString(pJson); + cJSON_AddItemToObject(tag, "value", tvalue); + cJSON_AddItemToArray(tags, tag); + taosMemoryFree(pJson); + goto end; + } + + for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { + STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); + + cJSON* tag = cJSON_CreateObject(); + + char* ptname = taosArrayGet(tagName, i); + cJSON* tname = cJSON_CreateString(ptname); + cJSON_AddItemToObject(tag, "name", tname); + // cJSON* cid = cJSON_CreateNumber(pTagVal->cid); + // cJSON_AddItemToObject(tag, "cid", cid); + cJSON* ttype = cJSON_CreateNumber(pTagVal->type); + cJSON_AddItemToObject(tag, "type", ttype); + + cJSON* tvalue = NULL; + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + char* buf = taosMemoryCalloc(pTagVal->nData + 3, 1); + if (!buf) goto end; + dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL); + tvalue = cJSON_CreateString(buf); + taosMemoryFree(buf); + } else { + double val = 0; + GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); + tvalue = cJSON_CreateNumber(val); + } + + cJSON_AddItemToObject(tag, "value", tvalue); + cJSON_AddItemToArray(tags, tag); + } + +end: + cJSON_AddItemToObject(json, "tags", tags); + string = cJSON_PrintUnformatted(json); + cJSON_Delete(json); + taosArrayDestroy(pTagVals); + return string; +} + +static char* processCreateTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVCreateTbBatchReq req = {0}; + SVCreateTbReq* pCreateReq; + char* string = NULL; + // decode + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); + tDecoderInit(&decoder, data, len); + if (tDecodeSVCreateTbBatchReq(&decoder, &req) < 0) { + goto _exit; + } + + // loop to create table + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + if (pCreateReq->type == TSDB_CHILD_TABLE) { + string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name, + pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum); + } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { + string = + buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); + } + } + + tDecoderClear(&decoder); + +_exit: + tDecoderClear(&decoder); + return string; +} + +static char* processAlterTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVAlterTbReq vAlterTbReq = {0}; + char* string = NULL; + + // decode + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); + tDecoderInit(&decoder, data, len); + if (tDecodeSVAlterTbReq(&decoder, &vAlterTbReq) < 0) { + goto _exit; + } + + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { + goto _exit; + } + cJSON* type = cJSON_CreateString("alter"); + cJSON_AddItemToObject(json, "type", type); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); + cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); + cJSON_AddItemToObject(json, "tableName", tableName); + cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL ? "child" : "normal"); + cJSON_AddItemToObject(json, "tableType", tableType); + cJSON* alterType = cJSON_CreateNumber(vAlterTbReq.action); + cJSON_AddItemToObject(json, "alterType", alterType); + + switch (vAlterTbReq.action) { + case TSDB_ALTER_TABLE_ADD_COLUMN: { + cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); + cJSON_AddItemToObject(json, "colType", colType); + + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY) { + int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } + break; + } + case TSDB_ALTER_TABLE_DROP_COLUMN: { + cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); + cJSON_AddItemToObject(json, "colName", colName); + break; + } + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { + cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); + cJSON_AddItemToObject(json, "colType", colType); + if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY) { + int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } + break; + } + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { + cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colNewName = cJSON_CreateString(vAlterTbReq.colNewName); + cJSON_AddItemToObject(json, "colNewName", colNewName); + break; + } + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: { + cJSON* tagName = cJSON_CreateString(vAlterTbReq.tagName); + cJSON_AddItemToObject(json, "colName", tagName); + + bool isNull = vAlterTbReq.isNull; + if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { + STag* jsonTag = (STag*)vAlterTbReq.pTagVal; + if (jsonTag->nTag == 0) isNull = true; + } + if (!isNull) { + char* buf = NULL; + + if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { + ASSERT(tTagIsJson(vAlterTbReq.pTagVal) == true); + buf = parseTagDatatoJson(vAlterTbReq.pTagVal); + } else { + buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); + dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL); + } + + cJSON* colValue = cJSON_CreateString(buf); + cJSON_AddItemToObject(json, "colValue", colValue); + taosMemoryFree(buf); + } + + cJSON* isNullCJson = cJSON_CreateBool(isNull); + cJSON_AddItemToObject(json, "colValueNull", isNullCJson); + break; + } + default: + break; + } + string = cJSON_PrintUnformatted(json); + +_exit: + tDecoderClear(&decoder); + return string; +} + +static char* processDropSTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropStbReq req = {0}; + char* string = NULL; + + // decode + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); + tDecoderInit(&decoder, data, len); + if (tDecodeSVDropStbReq(&decoder, &req) < 0) { + goto _exit; + } + + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { + goto _exit; + } + cJSON* type = cJSON_CreateString("drop"); + cJSON_AddItemToObject(json, "type", type); + cJSON* tableName = cJSON_CreateString(req.name); + cJSON_AddItemToObject(json, "tableName", tableName); + cJSON* tableType = cJSON_CreateString("super"); + cJSON_AddItemToObject(json, "tableType", tableType); + + string = cJSON_PrintUnformatted(json); + +_exit: + tDecoderClear(&decoder); + return string; +} + +static char* processDropTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropTbBatchReq req = {0}; + char* string = NULL; + + // decode + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); + tDecoderInit(&decoder, data, len); + if (tDecodeSVDropTbBatchReq(&decoder, &req) < 0) { + goto _exit; + } + + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { + goto _exit; + } + cJSON* type = cJSON_CreateString("drop"); + cJSON_AddItemToObject(json, "type", type); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); + // cJSON* tableType = cJSON_CreateString("normal"); + // cJSON_AddItemToObject(json, "tableType", tableType); + + cJSON* tableNameList = cJSON_CreateArray(); + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + SVDropTbReq* pDropTbReq = req.pReqs + iReq; + + cJSON* tableName = cJSON_CreateString(pDropTbReq->name); + cJSON_AddItemToArray(tableNameList, tableName); + } + cJSON_AddItemToObject(json, "tableNameList", tableNameList); + + string = cJSON_PrintUnformatted(json); + +_exit: + tDecoderClear(&decoder); + return string; +} + +static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { + SVCreateStbReq req = {0}; + SDecoder coder; + SMCreateStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVCreateStbReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + // build create stable + pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField)); + for (int32_t i = 0; i < req.schemaRow.nCols; i++) { + SSchema* pSchema = req.schemaRow.pSchema + i; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + strcpy(field.name, pSchema->name); + taosArrayPush(pReq.pColumns, &field); + } + pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); + for (int32_t i = 0; i < req.schemaTag.nCols; i++) { + SSchema* pSchema = req.schemaTag.pSchema + i; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + strcpy(field.name, pSchema->name); + taosArrayPush(pReq.pTags, &field); + } + + pReq.colVer = req.schemaRow.version; + pReq.tagVer = req.schemaTag.version; + pReq.numOfColumns = req.schemaRow.nCols; + pReq.numOfTags = req.schemaTag.nCols; + pReq.commentLen = -1; + pReq.suid = req.suid; + pReq.source = TD_REQ_FROM_TAOX; + pReq.igExists = true; + + STscObj* pTscObj = pRequest->pTscObj; + SName tableName; + tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); + + SCmdMsgInfo pCmdMsg = {0}; + pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_CREATE_STB; + pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + + SQuery pQuery = {0}; + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; + + launchQueryImpl(pRequest, &pQuery, true, NULL); + + if (pRequest->code == TSDB_CODE_SUCCESS) { + SCatalog* pCatalog = NULL; + catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + catalogRemoveTableMeta(pCatalog, &tableName); + } + + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); + +end: + destroyRequest(pRequest); + tFreeSMCreateStbReq(&pReq); + tDecoderClear(&coder); + return code; +} + +static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { + SVDropStbReq req = {0}; + SDecoder coder; + SMDropStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVDropStbReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + // build drop stable + pReq.igNotExists = true; + pReq.source = TD_REQ_FROM_TAOX; + pReq.suid = req.suid; + + STscObj* pTscObj = pRequest->pTscObj; + SName tableName = {0}; + tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); + + SCmdMsgInfo pCmdMsg = {0}; + pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_DROP_STB; + pCmdMsg.msgLen = tSerializeSMDropStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + + SQuery pQuery = {0}; + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; + + launchQueryImpl(pRequest, &pQuery, true, NULL); + + if (pRequest->code == TSDB_CODE_SUCCESS) { + SCatalog* pCatalog = NULL; + catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + catalogRemoveTableMeta(pCatalog, &tableName); + } + + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); + +end: + destroyRequest(pRequest); + tDecoderClear(&coder); + return code; +} + +typedef struct SVgroupCreateTableBatch { + SVCreateTbBatchReq req; + SVgroupInfo info; + char dbName[TSDB_DB_NAME_LEN]; +} SVgroupCreateTableBatch; + +static void destroyCreateTbReqBatch(void* data) { + SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*)data; + taosArrayDestroy(pTbBatch->req.pArray); +} + +static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { + SVCreateTbBatchReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SHashObj* pVgroupHashmap = NULL; + + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + STscObj* pTscObj = pRequest->pTscObj; + + SVCreateTbReq* pCreateReq = NULL; + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pVgroupHashmap) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + + pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); + // loop to create table + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + + SVgroupInfo pInfo = {0}; + SName pName = {0}; + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + taosArrayPush(pRequest->tableList, &pName); + + SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); + if (pTableBatch == NULL) { + SVgroupCreateTableBatch tBatch = {0}; + tBatch.info = pInfo; + strcpy(tBatch.dbName, pRequest->pDb); + + tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); + taosArrayPush(tBatch.req.pArray, pCreateReq); + + taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); + } else { // add to the correct vgroup + taosArrayPush(pTableBatch->req.pArray, pCreateReq); + } + } + + SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap); + if (NULL == pBufArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->msgType = TDMT_VND_CREATE_TABLE; + pQuery->stableQuery = false; + pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); + + code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + launchQueryImpl(pRequest, pQuery, true, NULL); + if (pRequest->code == TSDB_CODE_SUCCESS) { + removeMeta(pTscObj, pRequest->tableList); + } + + code = pRequest->code; + +end: + taosHashCleanup(pVgroupHashmap); + destroyRequest(pRequest); + tDecoderClear(&coder); + qDestroyQuery(pQuery); + return code; +} + +typedef struct SVgroupDropTableBatch { + SVDropTbBatchReq req; + SVgroupInfo info; + char dbName[TSDB_DB_NAME_LEN]; +} SVgroupDropTableBatch; + +static void destroyDropTbReqBatch(void* data) { + SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data; + taosArrayDestroy(pTbBatch->req.pArray); +} + +static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { + SVDropTbBatchReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SHashObj* pVgroupHashmap = NULL; + + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + STscObj* pTscObj = pRequest->pTscObj; + + SVDropTbReq* pDropReq = NULL; + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pVgroupHashmap) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); + // loop to create table + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pDropReq = req.pReqs + iReq; + pDropReq->igNotExists = true; + + SVgroupInfo pInfo = {0}; + SName pName = {0}; + toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + taosArrayPush(pRequest->tableList, &pName); + SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); + if (pTableBatch == NULL) { + SVgroupDropTableBatch tBatch = {0}; + tBatch.info = pInfo; + tBatch.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); + taosArrayPush(tBatch.req.pArray, pDropReq); + + taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); + } else { // add to the correct vgroup + taosArrayPush(pTableBatch->req.pArray, pDropReq); + } + } + + SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); + if (NULL == pBufArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->msgType = TDMT_VND_DROP_TABLE; + pQuery->stableQuery = false; + pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); + + code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + launchQueryImpl(pRequest, pQuery, true, NULL); + if (pRequest->code == TSDB_CODE_SUCCESS) { + removeMeta(pTscObj, pRequest->tableList); + } + code = pRequest->code; + +end: + taosHashCleanup(pVgroupHashmap); + destroyRequest(pRequest); + tDecoderClear(&coder); + qDestroyQuery(pQuery); + return code; +} + +// delete from db.tabl where .. -> delete from tabl where .. +// delete from db .tabl where .. -> delete from tabl where .. +// static void getTbName(char *sql){ +// char *ch = sql; +// +// bool inBackQuote = false; +// int8_t dotIndex = 0; +// while(*ch != '\0'){ +// if(!inBackQuote && *ch == '`'){ +// inBackQuote = true; +// ch++; +// continue; +// } +// +// if(inBackQuote && *ch == '`'){ +// inBackQuote = false; +// ch++; +// +// continue; +// } +// +// if(!inBackQuote && *ch == '.'){ +// dotIndex ++; +// if(dotIndex == 2){ +// memmove(sql, ch + 1, strlen(ch + 1) + 1); +// break; +// } +// } +// ch++; +// } +//} + +static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { + SDeleteRes req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeDeleteRes(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + // getTbName(req.tableFName); + char sql[256] = {0}; + sprintf(sql, "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, req.tsColName, + req.skey, req.tsColName, req.ekey); + printf("delete sql:%s\n", sql); + + TAOS_RES* res = taos_query(taos, sql); + SRequestObj* pRequest = (SRequestObj*)res; + code = pRequest->code; + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + code = TSDB_CODE_SUCCESS; + } + taos_free_result(res); + +end: + tDecoderClear(&coder); + return code; +} + +static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SArray* pArray = NULL; + SVgDataBlocks* pVgData = NULL; + + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); + + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVAlterTbReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + // do not deal TSDB_ALTER_TABLE_UPDATE_OPTIONS + if (req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS) { + goto end; + } + + STscObj* pTscObj = pRequest->pTscObj; + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + + SVgroupInfo pInfo = {0}; + SName pName = {0}; + toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + pArray = taosArrayInit(1, sizeof(void*)); + if (NULL == pArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == pVgData) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgData->vg = pInfo; + pVgData->pData = taosMemoryMalloc(metaLen); + if (NULL == pVgData->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + memcpy(pVgData->pData, meta, metaLen); + ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId); + pVgData->size = metaLen; + pVgData->numOfTables = 1; + taosArrayPush(pArray, &pVgData); + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->msgType = TDMT_VND_ALTER_TABLE; + pQuery->stableQuery = false; + pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + + code = rewriteToVnodeModifyOpStmt(pQuery, pArray); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + launchQueryImpl(pRequest, pQuery, true, NULL); + + pVgData = NULL; + pArray = NULL; + code = pRequest->code; + if (code == TSDB_CODE_VND_TABLE_NOT_EXIST) { + code = TSDB_CODE_SUCCESS; + } + + if (pRequest->code == TSDB_CODE_SUCCESS) { + SExecResult* pRes = &pRequest->body.resInfo.execRes; + if (pRes->res != NULL) { + code = handleAlterTbExecRes(pRes->res, pCatalog); + } + } +end: + taosArrayDestroy(pArray); + if (pVgData) taosMemoryFreeClear(pVgData->pData); + taosMemoryFreeClear(pVgData); + destroyRequest(pRequest); + tDecoderClear(&coder); + qDestroyQuery(pQuery); + return code; +} + +typedef struct { + SVgroupInfo vg; + void* data; +} VgData; + +static void destroyVgHash(void* data) { + VgData* vgData = (VgData*)data; + taosMemoryFreeClear(vgData->data); +} + +int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { + int32_t code = TSDB_CODE_SUCCESS; + STableMeta* pTableMeta = NULL; + SQuery* pQuery = NULL; + + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); + if (!pRequest) { + uError("WriteRaw:createRequest error request is null"); + code = terrno; + goto end; + } + + if (!pRequest->pDb) { + uError("WriteRaw:not use db"); + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + strcpy(pName.dbname, pRequest->pDb); + strcpy(pName.tname, tbname); + + struct SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw: get gatlog error"); + goto end; + } + + SRequestConnInfo conn = {0}; + conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; + conn.requestId = pRequest->requestId; + conn.requestObjRefId = pRequest->self; + conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + SVgroupInfo vgData = {0}; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgData); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbname); + goto end; + } + + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); + goto end; + } + uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); + uint64_t uid = pTableMeta->uid; + int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; + + uint16_t fLen = 0; + int32_t rowSize = 0; + int16_t nVar = 0; + for (int i = 0; i < numOfCols; i++) { + SSchema* schema = pTableMeta->schema + i; + fLen += TYPE_BYTES[schema->type]; + rowSize += schema->bytes; + if (IS_VAR_DATA_TYPE(schema->type)) { + nVar++; + } + } + + int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + + (int32_t)TD_BITMAP_BYTES(numOfCols - 1); + int32_t schemaLen = 0; + int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; + + int32_t totalLen = sizeof(SSubmitReq) + submitLen; + SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); + SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); + void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); + STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); + + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTableMeta->sversion); + tdSRowSetTpInfo(&rb, numOfCols, fLen); + int32_t dataLen = 0; + + char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); + int32_t* colLength = (int32_t*)pStart; + pStart += sizeof(int32_t) * numOfCols; + + SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); + + for (int32_t i = 0; i < numOfCols; ++i) { + if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { + pCol[i].offset = (int32_t*)pStart; + pStart += rows * sizeof(int32_t); + } else { + pCol[i].nullbitmap = pStart; + pStart += BitmapLen(rows); + } + + pCol[i].pData = pStart; + pStart += colLength[i]; + } + + for (int32_t j = 0; j < rows; j++) { + tdSRowResetBuf(&rb, rowData); + int32_t offset = 0; + for (int32_t k = 0; k < numOfCols; k++) { + const SSchema* pColumn = &pTableMeta->schema[k]; + + if (IS_VAR_DATA_TYPE(pColumn->type)) { + if (pCol[k].offset[j] != -1) { + char* data = pCol[k].pData + pCol[k].offset[j]; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } else { + if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { + char* data = pCol[k].pData + pColumn->bytes * j; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } + } + + offset += TYPE_BYTES[pColumn->type]; + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + + taosMemoryFree(pCol); + + blk->uid = htobe64(uid); + blk->suid = htobe64(suid); + blk->sversion = htonl(pTableMeta->sversion); + blk->schemaLen = htonl(schemaLen); + blk->numOfRows = htonl(rows); + blk->dataLen = htonl(dataLen); + subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; + subReq->numOfBlocks = 1; + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create SQuery error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pQuery->pRoot) { + uError("create pQuery->pRoot error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); + nodeStmt->payloadType = PAYLOAD_TYPE_KV; + nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); + + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + dst->vg = vgData; + dst->numOfTables = subReq->numOfBlocks; + dst->size = subReq->length; + dst->pData = (char*)subReq; + subReq->header.vgId = htonl(dst->vg.vgId); + subReq->version = htonl(1); + subReq->header.contLen = htonl(subReq->length); + subReq->length = htonl(subReq->length); + subReq->numOfBlocks = htonl(subReq->numOfBlocks); + subReq = NULL; // no need free + taosArrayPush(nodeStmt->pDataBlocks, &dst); + + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + +end: + taosMemoryFreeClear(pTableMeta); + qDestroyQuery(pQuery); + return code; +} + +static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pVgHash = NULL; + SQuery* pQuery = NULL; + SMqRspObj rspObj = {0}; + SDecoder decoder = {0}; + + terrno = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); + if (!pRequest) { + uError("WriteRaw:createRequest error request is null"); + return terrno; + } + + rspObj.resIter = -1; + rspObj.resType = RES_TYPE__TMQ; + + tDecoderInit(&decoder, data, dataLen); + code = tDecodeSMqDataRsp(&decoder, &rspObj.rsp); + if (code != 0) { + uError("WriteRaw:decode smqDataRsp error"); + code = TSDB_CODE_INVALID_MSG; + goto end; + } + + if (!pRequest->pDb) { + uError("WriteRaw:not use db"); + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashSetFreeFp(pVgHash, destroyVgHash); + struct SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw: get gatlog error"); + goto end; + } + + SRequestConnInfo conn = {0}; + conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; + conn.requestId = pRequest->requestId; + conn.requestObjRefId = pRequest->self; + conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + printf("raw data block num:%d\n", rspObj.rsp.blockNum); + while (++rspObj.resIter < rspObj.rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); + if (!rspObj.rsp.withSchema) { + uError("WriteRaw:no schema, iter:%d", rspObj.resIter); + goto end; + } + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); + + code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw: setQueryResultFromRsp error"); + goto end; + } + + uint16_t fLen = 0; + int32_t rowSize = 0; + int16_t nVar = 0; + for (int i = 0; i < pSW->nCols; i++) { + SSchema* schema = pSW->pSchema + i; + fLen += TYPE_BYTES[schema->type]; + rowSize += schema->bytes; + if (IS_VAR_DATA_TYPE(schema->type)) { + nVar++; + } + } + + int32_t rows = rspObj.resInfo.numOfRows; + int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + + (int32_t)TD_BITMAP_BYTES(pSW->nCols - 1); + int32_t schemaLen = 0; + int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; + + const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); + if (!tbName) { + uError("WriteRaw: tbname is null"); + code = TSDB_CODE_TMQ_INVALID_MSG; + goto end; + } + + printf("raw data tbname:%s\n", tbName); + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + strcpy(pName.dbname, pRequest->pDb); + strcpy(pName.tname, tbName); + + VgData vgData = {0}; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; + } + + SSubmitReq* subReq = NULL; + SSubmitBlk* blk = NULL; + void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); + if (hData) { + vgData = *(VgData*)hData; + + int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; + void* tmp = taosMemoryRealloc(vgData.data, totalLen); + if (tmp == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + vgData.data = tmp; + ((VgData*)hData)->data = tmp; + subReq = (SSubmitReq*)(vgData.data); + blk = POINTER_SHIFT(vgData.data, subReq->length); + } else { + int32_t totalLen = sizeof(SSubmitReq) + submitLen; + void* tmp = taosMemoryCalloc(1, totalLen); + if (tmp == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + vgData.data = tmp; + taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); + subReq = (SSubmitReq*)(vgData.data); + subReq->length = sizeof(SSubmitReq); + subReq->numOfBlocks = 0; + + blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); + } + + STableMeta* pTableMeta = NULL; + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbName); + goto end; + } + uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); + uint64_t uid = pTableMeta->uid; + taosMemoryFreeClear(pTableMeta); + + void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); + STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); + + SRowBuilder rb = {0}; + tdSRowInit(&rb, pSW->version); + tdSRowSetTpInfo(&rb, pSW->nCols, fLen); + int32_t dataLen = 0; + + for (int32_t j = 0; j < rows; j++) { + tdSRowResetBuf(&rb, rowData); + + doSetOneRowPtr(&rspObj.resInfo); + rspObj.resInfo.current += 1; + + int32_t offset = 0; + for (int32_t k = 0; k < pSW->nCols; k++) { + const SSchema* pColumn = &pSW->pSchema[k]; + char* data = rspObj.resInfo.row[k]; + if (!data) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); + } else { + if (IS_VAR_DATA_TYPE(pColumn->type)) { + data -= VARSTR_HEADER_SIZE; + } + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); + } + offset += TYPE_BYTES[pColumn->type]; + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + + blk->uid = htobe64(uid); + blk->suid = htobe64(suid); + blk->sversion = htonl(pSW->version); + blk->schemaLen = htonl(schemaLen); + blk->numOfRows = htonl(rows); + blk->dataLen = htonl(dataLen); + subReq->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + subReq->numOfBlocks++; + } + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create SQuery error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + if (NULL == pQuery->pRoot) { + uError("create pQuery->pRoot error"); + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); + nodeStmt->payloadType = PAYLOAD_TYPE_KV; + + int32_t numOfVg = taosHashGetSize(pVgHash); + nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + + VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); + while (vData) { + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto end; + } + dst->vg = vData->vg; + SSubmitReq* subReq = (SSubmitReq*)(vData->data); + dst->numOfTables = subReq->numOfBlocks; + dst->size = subReq->length; + dst->pData = (char*)subReq; + vData->data = NULL; // no need free + subReq->header.vgId = htonl(dst->vg.vgId); + subReq->version = htonl(1); + subReq->header.contLen = htonl(subReq->length); + subReq->length = htonl(subReq->length); + subReq->numOfBlocks = htonl(subReq->numOfBlocks); + taosArrayPush(nodeStmt->pDataBlocks, &dst); + vData = (VgData*)taosHashIterate(pVgHash, vData); + } + + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + +end: + tDecoderClear(&decoder); + taos_free_result(&rspObj); + qDestroyQuery(pQuery); + destroyRequest(pRequest); + taosHashCleanup(pVgHash); + return code; +} + +char* tmq_get_json_meta(TAOS_RES* res) { + if (!TD_RES_TMQ_META(res)) { + return NULL; + } + + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { + return processCreateStb(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { + return processAlterStb(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { + return processDropSTable(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { + return processCreateTable(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { + return processAlterTable(&pMetaRspObj->metaRsp); + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { + return processDropTable(&pMetaRspObj->metaRsp); + } + return NULL; +} + +void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } + +int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { + if (!raw || !res) { + return TSDB_CODE_INVALID_PARA; + } + if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + raw->raw = pMetaRspObj->metaRsp.metaRsp; + raw->raw_len = pMetaRspObj->metaRsp.metaRspLen; + raw->raw_type = pMetaRspObj->metaRsp.resMsgType; + } else if (TD_RES_TMQ(res)) { + SMqRspObj* rspObj = ((SMqRspObj*)res); + + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeSMqDataRsp, &rspObj->rsp, len, code); + if (code < 0) { + return -1; + } + + void* buf = taosMemoryCalloc(1, len); + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, len); + tEncodeSMqDataRsp(&encoder, &rspObj->rsp); + tEncoderClear(&encoder); + + raw->raw = buf; + raw->raw_len = len; + raw->raw_type = RES_TYPE__TMQ; + } else { + return TSDB_CODE_TMQ_INVALID_MSG; + } + return TSDB_CODE_SUCCESS; +} + +void tmq_free_raw(tmq_raw_data raw) { + if (raw.raw_type == RES_TYPE__TMQ) { + taosMemoryFree(raw.raw); + } +} + +int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { + if (!taos) { + return TSDB_CODE_INVALID_PARA; + } + + if (raw.raw_type == TDMT_VND_CREATE_STB) { + return taosCreateStb(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == TDMT_VND_ALTER_STB) { + return taosCreateStb(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == TDMT_VND_DROP_STB) { + return taosDropStb(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == TDMT_VND_CREATE_TABLE) { + return taosCreateTable(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == TDMT_VND_ALTER_TABLE) { + return taosAlterTable(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == TDMT_VND_DROP_TABLE) { + return taosDropTable(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == TDMT_VND_DELETE) { + return taosDeleteData(taos, raw.raw, raw.raw_len); + } else if (raw.raw_type == RES_TYPE__TMQ) { + return tmqWriteRaw(taos, raw.raw, raw.raw_len); + } + return TSDB_CODE_INVALID_PARA; +} diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 436a9c2cfd08..023e21125616 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -28,8 +28,9 @@ int32_t tmqAskEp(tmq_t* tmq, bool async); typedef struct { - int8_t inited; - tmr_h timer; + int8_t inited; + tmr_h timer; + int32_t rsetId; } SMqMgmt; static SMqMgmt tmqMgmt = {0}; @@ -70,6 +71,7 @@ struct tmq_conf_t { }; struct tmq_t { + int64_t refId; // conf char groupId[TSDB_CGROUP_LEN]; char clientId[256]; @@ -146,8 +148,8 @@ typedef struct { typedef struct { // subscribe info - char* topicName; - char db[TSDB_DB_FNAME_LEN]; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; SArray* vgs; // SArray @@ -166,29 +168,32 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - tmq_t* tmq; + int64_t refId; + int32_t epoch; tsem_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; typedef struct { - tmq_t* tmq; + int64_t refId; + int32_t epoch; int32_t code; int32_t async; tsem_t rspSem; } SMqAskEpCbParam; typedef struct { - tmq_t* tmq; + int64_t refId; + int32_t epoch; SMqClientVg* pVg; SMqClientTopic* pTopic; - int32_t epoch; int32_t vgId; tsem_t rspSem; } SMqPollCbParam; typedef struct { - tmq_t* tmq; + int64_t refId; + int32_t epoch; int8_t automatic; int8_t async; int32_t waitingRspNum; @@ -369,6 +374,38 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } +int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParamSet->refId); + if (tmq == NULL) { + if (!pParamSet->async) { + tsem_destroy(&pParamSet->rspSem); + } + taosMemoryFree(pParamSet); + terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; + return -1; + } + + // if no more waiting rsp + if (pParamSet->async) { + // call async cb func + if (pParamSet->automatic && tmq->commitCb) { + tmq->commitCb(tmq, pParamSet->rspErr, tmq->commitCbUserParam); + } else if (!pParamSet->automatic && pParamSet->userCb) { + // sem post + pParamSet->userCb(tmq, pParamSet->rspErr, pParamSet->userParam); + } + taosMemoryFree(pParamSet); + } else { + tsem_post(&pParamSet->rspSem); + } + +#if 0 + taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); + taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); +#endif + return 0; +} + int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; @@ -392,25 +429,7 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { ASSERT(waitingRspNum >= 0); if (waitingRspNum == 0) { - // if no more waiting rsp - if (pParamSet->async) { - // call async cb func - if (pParamSet->automatic && pParamSet->tmq->commitCb) { - pParamSet->tmq->commitCb(pParamSet->tmq, pParamSet->rspErr, pParamSet->tmq->commitCbUserParam); - } else if (!pParamSet->automatic && pParamSet->userCb) { - // sem post - pParamSet->userCb(pParamSet->tmq, pParamSet->rspErr, pParamSet->userParam); - } - } else { - tsem_post(&pParamSet->rspSem); - } - - taosMemoryFree(pParamSet); - -#if 0 - taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); - taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); -#endif + tmqCommitDone(pParamSet); } return 0; } @@ -504,7 +523,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pParamSet->tmq = tmq; + pParamSet->refId = tmq->refId; + pParamSet->epoch = tmq->epoch; pParamSet->automatic = 0; pParamSet->async = async; pParamSet->userCb = userCb; @@ -565,13 +585,19 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pParamSet->tmq = tmq; + + pParamSet->refId = tmq->refId; + pParamSet->epoch = tmq->epoch; + pParamSet->automatic = automatic; pParamSet->async = async; pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); + // init as 1 to prevent concurrency issue + pParamSet->waitingRspNum = 1; + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -600,10 +626,17 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t return 0; } + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + ASSERT(waitingRspNum >= 0); + if (waitingRspNum == 0) { + tmqCommitDone(pParamSet); + } + if (!async) { tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); } else { code = 0; } @@ -616,38 +649,50 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t } } - if (!async) { #if 0 + if (!async) { taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); -#endif } +#endif return 0; } void tmqAssignAskEpTask(void* param, void* tmrId) { - tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); - *pTaskType = TMQ_DELAYED_TASK__ASK_EP; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); + int64_t refId = *(int64_t*)param; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq != NULL) { + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); + *pTaskType = TMQ_DELAYED_TASK__ASK_EP; + taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); + } + taosMemoryFree(param); } void tmqAssignDelayedCommitTask(void* param, void* tmrId) { - tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); - *pTaskType = TMQ_DELAYED_TASK__COMMIT; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); + int64_t refId = *(int64_t*)param; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq != NULL) { + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); + *pTaskType = TMQ_DELAYED_TASK__COMMIT; + taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); + } + taosMemoryFree(param); } void tmqAssignDelayedReportTask(void* param, void* tmrId) { - tmq_t* tmq = (tmq_t*)param; - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); - *pTaskType = TMQ_DELAYED_TASK__REPORT; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); + int64_t refId = *(int64_t*)param; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq != NULL) { + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); + *pTaskType = TMQ_DELAYED_TASK__REPORT; + taosWriteQitem(tmq->delayedTask, pTaskType); + tsem_post(&tmq->rspSem); + } + taosMemoryFree(param); } int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { @@ -656,8 +701,11 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { } void tmqSendHbReq(void* param, void* tmrId) { - // TODO replace with ref - tmq_t* tmq = (tmq_t*)param; + int64_t refId = *(int64_t*)param; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq == NULL) { + return; + } int64_t consumerId = tmq->consumerId; int32_t epoch = tmq->epoch; SMqHbReq* pReq = taosMemoryMalloc(sizeof(SMqHbReq)); @@ -687,7 +735,7 @@ void tmqSendHbReq(void* param, void* tmrId) { asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); OVER: - taosTmrReset(tmqSendHbReq, 1000, tmq, tmqMgmt.timer, &tmq->hbLiveTimer); + taosTmrReset(tmqSendHbReq, 1000, param, tmqMgmt.timer, &tmq->hbLiveTimer); } int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { @@ -700,10 +748,18 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { tmqAskEp(tmq, true); - taosTmrReset(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer, &tmq->epTimer); + + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = tmq->refId; + + taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &tmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); - taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); + + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = tmq->refId; + + taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, pRefId, tmqMgmt.timer, &tmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { } else { ASSERT(0); @@ -738,7 +794,6 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; - /*tmq_t* tmq = pParam->tmq;*/ tsem_post(&pParam->rspSem); return 0; } @@ -761,40 +816,26 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { return rsp; } -#if 0 -tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - tmq_t* pTmq = taosMemoryCalloc(sizeof(tmq_t), 1); - if (pTmq == NULL) { - return NULL; - } - pTmq->pTscObj = (STscObj*)conn; - pTmq->status = 0; - pTmq->pollCnt = 0; - pTmq->epoch = 0; - pTmq->epStatus = 0; - pTmq->epSkipCnt = 0; - // set conf - strcpy(pTmq->clientId, conf->clientId); - strcpy(pTmq->groupId, conf->groupId); - pTmq->autoCommit = conf->autoCommit; - pTmq->commit_cb = conf->commit_cb; - pTmq->resetOffsetCfg = conf->resetOffset; +void tmqFreeImpl(void* handle) { + tmq_t* tmq = (tmq_t*)handle; - pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); - pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); - if (pTmq->clientTopics == NULL) { - taosMemoryFree(pTmq); - return NULL; - } - - pTmq->mqueue = taosOpenQueue(); - pTmq->qall = taosAllocateQall(); + // TODO stop timer + if (tmq->mqueue) taosCloseQueue(tmq->mqueue); + if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask); + if (tmq->qall) taosFreeQall(tmq->qall); - tsem_init(&pTmq->rspSem, 0, 0); + tsem_destroy(&tmq->rspSem); - return pTmq; + int32_t sz = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (pTopic->schema.nCols) taosMemoryFree(pTopic->schema.pSchema); + int32_t vgSz = taosArrayGetSize(pTopic->vgs); + taosArrayDestroy(pTopic->vgs); + } + taosArrayDestroy(tmq->clientTopics); + taosMemoryFree(tmq); } -#endif tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // init timer @@ -806,6 +847,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + tmqMgmt.rsetId = taosOpenRef(10000, tmqFreeImpl); } tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); @@ -874,8 +916,17 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { goto FAIL; } + pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq); + if (pTmq->refId < 0) { + tmqFreeImpl(pTmq); + return NULL; + } + + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = pTmq->refId; + if (pTmq->hbBgEnable) { - pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pTmq, tmqMgmt.timer); + pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } tscInfo("consumer %" PRId64 " is setup, consumer group %s", pTmq->consumerId, pTmq->groupId); @@ -933,7 +984,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SMqSubscribeCbParam param = { .rspErr = 0, - .tmq = tmq, + .refId = tmq->refId, + .epoch = tmq->epoch, }; if (tsem_init(¶m.rspSem, 0, 0) != 0) goto FAIL; @@ -975,12 +1027,16 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { // init ep timer if (tmq->epTimer == NULL) { - tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer); + int64_t* pRefId1 = taosMemoryMalloc(sizeof(int64_t)); + *pRefId1 = tmq->refId; + tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, 1000, pRefId1, tmqMgmt.timer); } // init auto commit timer if (tmq->autoCommit && tmq->commitTimer == NULL) { - tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer); + int64_t* pRefId2 = taosMemoryMalloc(sizeof(int64_t)); + *pRefId2 = tmq->refId; + tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, pRefId2, tmqMgmt.timer); } code = 0; @@ -1002,9 +1058,18 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; SMqClientTopic* pTopic = pParam->pTopic; - tmq_t* tmq = pParam->tmq; - int32_t vgId = pParam->vgId; - int32_t epoch = pParam->epoch; + + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); + if (tmq == NULL) { + tsem_destroy(&pParam->rspSem); + taosMemoryFree(pParam); + taosMemoryFree(pMsg->pData); + terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; + return -1; + } + + int32_t epoch = pParam->epoch; + int32_t vgId = pParam->vgId; taosMemoryFree(pParam); if (code != 0) { tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr()); @@ -1129,7 +1194,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientTopic topic = {0}; SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); topic.schema = pTopicEp->schema; - topic.topicName = strdup(pTopicEp->topic); + tstrncpy(topic.topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); tscDebug("consumer:%" PRId64 ", update topic: %s", tmq->consumerId, topic.topicName); @@ -1158,7 +1223,16 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { } taosArrayPush(newTopics, &topic); } - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + if (tmq->clientTopics) { + int32_t sz = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (pTopic->schema.nCols) taosMemoryFree(pTopic->schema.pSchema); + int32_t vgSz = taosArrayGetSize(pTopic->vgs); + taosArrayDestroy(pTopic->vgs); + } + taosArrayDestroy(tmq->clientTopics); + } taosHashCleanup(pHash); tmq->clientTopics = newTopics; @@ -1173,8 +1247,20 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; - tmq_t* tmq = pParam->tmq; int8_t async = pParam->async; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); + + if (tmq == NULL) { + if (!async) { + tsem_destroy(&pParam->rspSem); + } else { + taosMemoryFree(pParam); + } + taosMemoryFree(pMsg->pData); + terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; + return -1; + } + pParam->code = code; if (code != 0) { tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); @@ -1254,7 +1340,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { /*atomic_store_8(&tmq->epStatus, 0);*/ return -1; } - pParam->tmq = tmq; + pParam->refId = tmq->refId; + pParam->epoch = tmq->epoch; pParam->async = async; tsem_init(&pParam->rspSem, 0, 0); @@ -1294,31 +1381,6 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { return code; } -#if 0 -int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { - const SMqOffset* pOffset = &offset->offset; - if (strcmp(pOffset->cgroup, tmq->groupId) != 0) { - return TMQ_RESP_ERR__FAIL; - } - int32_t sz = taosArrayGetSize(tmq->clientTopics); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) { - int32_t vgSz = taosArrayGetSize(clientTopic->vgs); - for (int32_t j = 0; j < vgSz; j++) { - SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j); - if (pVg->vgId == pOffset->vgId) { - pVg->currentOffset = pOffset->offset; - tmqClearUnhandleMsg(tmq); - return TMQ_RESP_ERR__SUCCESS; - } - } - } - } - return TMQ_RESP_ERR__FAIL; -} -#endif - SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); if (pReq == NULL) { @@ -1412,11 +1474,12 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { tsem_post(&tmq->rspSem); return -1; } - pParam->tmq = tmq; + pParam->refId = tmq->refId; + pParam->epoch = tmq->epoch; + pParam->pVg = pVg; pParam->pTopic = pTopic; pParam->vgId = pVg->vgId; - pParam->epoch = tmq->epoch; SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { @@ -1556,7 +1619,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { } #endif - // in no topic status also need process delayed task + // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { return NULL; } @@ -1621,7 +1684,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) { /*return rsp;*/ return 0; } - // TODO: free resources + taosRemoveRef(tmqMgmt.rsetId, tmq->refId); return 0; } @@ -1697,1610 +1760,6 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, - int8_t t) { - char* string = NULL; - cJSON* json = cJSON_CreateObject(); - if (json == NULL) { - return string; - } - cJSON* type = cJSON_CreateString("create"); - cJSON_AddItemToObject(json, "type", type); - - // char uid[32] = {0}; - // sprintf(uid, "%"PRIi64, id); - // cJSON* id_ = cJSON_CreateString(uid); - // cJSON_AddItemToObject(json, "id", id_); - cJSON* tableName = cJSON_CreateString(name); - cJSON_AddItemToObject(json, "tableName", tableName); - cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); - cJSON_AddItemToObject(json, "tableType", tableType); - // cJSON* version = cJSON_CreateNumber(1); - // cJSON_AddItemToObject(json, "version", version); - - cJSON* columns = cJSON_CreateArray(); - for (int i = 0; i < schemaRow->nCols; i++) { - cJSON* column = cJSON_CreateObject(); - SSchema* s = schemaRow->pSchema + i; - cJSON* cname = cJSON_CreateString(s->name); - cJSON_AddItemToObject(column, "name", cname); - cJSON* ctype = cJSON_CreateNumber(s->type); - cJSON_AddItemToObject(column, "type", ctype); - if (s->type == TSDB_DATA_TYPE_BINARY) { - int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(column, "length", cbytes); - } else if (s->type == TSDB_DATA_TYPE_NCHAR) { - int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(column, "length", cbytes); - } - cJSON_AddItemToArray(columns, column); - } - cJSON_AddItemToObject(json, "columns", columns); - - cJSON* tags = cJSON_CreateArray(); - for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { - cJSON* tag = cJSON_CreateObject(); - SSchema* s = schemaTag->pSchema + i; - cJSON* tname = cJSON_CreateString(s->name); - cJSON_AddItemToObject(tag, "name", tname); - cJSON* ttype = cJSON_CreateNumber(s->type); - cJSON_AddItemToObject(tag, "type", ttype); - if (s->type == TSDB_DATA_TYPE_BINARY) { - int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(tag, "length", cbytes); - } else if (s->type == TSDB_DATA_TYPE_NCHAR) { - int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(tag, "length", cbytes); - } - cJSON_AddItemToArray(tags, tag); - } - cJSON_AddItemToObject(json, "tags", tags); - - string = cJSON_PrintUnformatted(json); - cJSON_Delete(json); - return string; -} - -static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { - SMAlterStbReq req = {0}; - cJSON* json = NULL; - char* string = NULL; - - if (tDeserializeSMAlterStbReq(alterData, alterDataLen, &req) != 0) { - goto end; - } - - json = cJSON_CreateObject(); - if (json == NULL) { - goto end; - } - cJSON* type = cJSON_CreateString("alter"); - cJSON_AddItemToObject(json, "type", type); - // cJSON* uid = cJSON_CreateNumber(id); - // cJSON_AddItemToObject(json, "uid", uid); - SName name = {0}; - tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - cJSON* tableName = cJSON_CreateString(name.tname); - cJSON_AddItemToObject(json, "tableName", tableName); - cJSON* tableType = cJSON_CreateString("super"); - cJSON_AddItemToObject(json, "tableType", tableType); - - cJSON* alterType = cJSON_CreateNumber(req.alterType); - cJSON_AddItemToObject(json, "alterType", alterType); - switch (req.alterType) { - case TSDB_ALTER_TABLE_ADD_TAG: - case TSDB_ALTER_TABLE_ADD_COLUMN: { - TAOS_FIELD* field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); - cJSON_AddItemToObject(json, "colName", colName); - cJSON* colType = cJSON_CreateNumber(field->type); - cJSON_AddItemToObject(json, "colType", colType); - - if (field->type == TSDB_DATA_TYPE_BINARY) { - int32_t length = field->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } else if (field->type == TSDB_DATA_TYPE_NCHAR) { - int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } - break; - } - case TSDB_ALTER_TABLE_DROP_TAG: - case TSDB_ALTER_TABLE_DROP_COLUMN: { - TAOS_FIELD* field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); - cJSON_AddItemToObject(json, "colName", colName); - break; - } - case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { - TAOS_FIELD* field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); - cJSON_AddItemToObject(json, "colName", colName); - cJSON* colType = cJSON_CreateNumber(field->type); - cJSON_AddItemToObject(json, "colType", colType); - if (field->type == TSDB_DATA_TYPE_BINARY) { - int32_t length = field->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } else if (field->type == TSDB_DATA_TYPE_NCHAR) { - int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } - break; - } - case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { - TAOS_FIELD* oldField = taosArrayGet(req.pFields, 0); - TAOS_FIELD* newField = taosArrayGet(req.pFields, 1); - cJSON* colName = cJSON_CreateString(oldField->name); - cJSON_AddItemToObject(json, "colName", colName); - cJSON* colNewName = cJSON_CreateString(newField->name); - cJSON_AddItemToObject(json, "colNewName", colNewName); - break; - } - default: - break; - } - string = cJSON_PrintUnformatted(json); - -end: - cJSON_Delete(json); - tFreeSMAltertbReq(&req); - return string; -} - -static char* processCreateStb(SMqMetaRsp* metaRsp) { - SVCreateStbReq req = {0}; - SDecoder coder; - char* string = NULL; - - // decode and process req - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); - int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - - if (tDecodeSVCreateStbReq(&coder, &req) < 0) { - goto _err; - } - string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE); - tDecoderClear(&coder); - return string; - -_err: - tDecoderClear(&coder); - return string; -} - -static char* processAlterStb(SMqMetaRsp* metaRsp) { - SVCreateStbReq req = {0}; - SDecoder coder; - char* string = NULL; - - // decode and process req - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); - int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - - if (tDecodeSVCreateStbReq(&coder, &req) < 0) { - goto _err; - } - string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); - tDecoderClear(&coder); - return string; - -_err: - tDecoderClear(&coder); - return string; -} - -static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id, uint8_t tagNum) { - char* string = NULL; - SArray* pTagVals = NULL; - cJSON* json = cJSON_CreateObject(); - if (json == NULL) { - return string; - } - cJSON* type = cJSON_CreateString("create"); - cJSON_AddItemToObject(json, "type", type); - // char cid[32] = {0}; - // sprintf(cid, "%"PRIi64, id); - // cJSON* cid_ = cJSON_CreateString(cid); - // cJSON_AddItemToObject(json, "id", cid_); - - cJSON* tableName = cJSON_CreateString(name); - cJSON_AddItemToObject(json, "tableName", tableName); - cJSON* tableType = cJSON_CreateString("child"); - cJSON_AddItemToObject(json, "tableType", tableType); - cJSON* using = cJSON_CreateString(sname); - cJSON_AddItemToObject(json, "using", using); - cJSON* tagNumJson = cJSON_CreateNumber(tagNum); - cJSON_AddItemToObject(json, "tagNum", tagNumJson); - // cJSON* version = cJSON_CreateNumber(1); - // cJSON_AddItemToObject(json, "version", version); - - cJSON* tags = cJSON_CreateArray(); - int32_t code = tTagToValArray(pTag, &pTagVals); - if (code) { - goto end; - } - - if (tTagIsJson(pTag)) { - STag* p = (STag*)pTag; - if (p->nTag == 0) { - goto end; - } - char* pJson = parseTagDatatoJson(pTag); - cJSON* tag = cJSON_CreateObject(); - STagVal* pTagVal = taosArrayGet(pTagVals, 0); - - char* ptname = taosArrayGet(tagName, 0); - cJSON* tname = cJSON_CreateString(ptname); - cJSON_AddItemToObject(tag, "name", tname); - // cJSON* cid_ = cJSON_CreateString(""); - // cJSON_AddItemToObject(tag, "cid", cid_); - cJSON* ttype = cJSON_CreateNumber(TSDB_DATA_TYPE_JSON); - cJSON_AddItemToObject(tag, "type", ttype); - cJSON* tvalue = cJSON_CreateString(pJson); - cJSON_AddItemToObject(tag, "value", tvalue); - cJSON_AddItemToArray(tags, tag); - taosMemoryFree(pJson); - goto end; - } - - for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { - STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); - - cJSON* tag = cJSON_CreateObject(); - - char* ptname = taosArrayGet(tagName, i); - cJSON* tname = cJSON_CreateString(ptname); - cJSON_AddItemToObject(tag, "name", tname); - // cJSON* cid = cJSON_CreateNumber(pTagVal->cid); - // cJSON_AddItemToObject(tag, "cid", cid); - cJSON* ttype = cJSON_CreateNumber(pTagVal->type); - cJSON_AddItemToObject(tag, "type", ttype); - - cJSON* tvalue = NULL; - if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char* buf = taosMemoryCalloc(pTagVal->nData + 3, 1); - if (!buf) goto end; - dataConverToStr(buf, pTagVal->type, pTagVal->pData, pTagVal->nData, NULL); - tvalue = cJSON_CreateString(buf); - taosMemoryFree(buf); - } else { - double val = 0; - GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); - tvalue = cJSON_CreateNumber(val); - } - - cJSON_AddItemToObject(tag, "value", tvalue); - cJSON_AddItemToArray(tags, tag); - } - -end: - cJSON_AddItemToObject(json, "tags", tags); - string = cJSON_PrintUnformatted(json); - cJSON_Delete(json); - taosArrayDestroy(pTagVals); - return string; -} - -static char* processCreateTable(SMqMetaRsp* metaRsp) { - SDecoder decoder = {0}; - SVCreateTbBatchReq req = {0}; - SVCreateTbReq* pCreateReq; - char* string = NULL; - // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); - int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); - tDecoderInit(&decoder, data, len); - if (tDecodeSVCreateTbBatchReq(&decoder, &req) < 0) { - goto _exit; - } - - // loop to create table - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pCreateReq = req.pReqs + iReq; - if (pCreateReq->type == TSDB_CHILD_TABLE) { - string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name, - pCreateReq->ctb.tagName, pCreateReq->uid, pCreateReq->ctb.tagNum); - } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { - string = - buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); - } - } - - tDecoderClear(&decoder); - -_exit: - tDecoderClear(&decoder); - return string; -} - -static char* processAlterTable(SMqMetaRsp* metaRsp) { - SDecoder decoder = {0}; - SVAlterTbReq vAlterTbReq = {0}; - char* string = NULL; - - // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); - int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); - tDecoderInit(&decoder, data, len); - if (tDecodeSVAlterTbReq(&decoder, &vAlterTbReq) < 0) { - goto _exit; - } - - cJSON* json = cJSON_CreateObject(); - if (json == NULL) { - goto _exit; - } - cJSON* type = cJSON_CreateString("alter"); - cJSON_AddItemToObject(json, "type", type); - // cJSON* uid = cJSON_CreateNumber(id); - // cJSON_AddItemToObject(json, "uid", uid); - cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); - cJSON_AddItemToObject(json, "tableName", tableName); - cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL ? "child" : "normal"); - cJSON_AddItemToObject(json, "tableType", tableType); - cJSON* alterType = cJSON_CreateNumber(vAlterTbReq.action); - cJSON_AddItemToObject(json, "alterType", alterType); - - switch (vAlterTbReq.action) { - case TSDB_ALTER_TABLE_ADD_COLUMN: { - cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); - cJSON_AddItemToObject(json, "colName", colName); - cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); - cJSON_AddItemToObject(json, "colType", colType); - - if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY) { - int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { - int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } - break; - } - case TSDB_ALTER_TABLE_DROP_COLUMN: { - cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); - cJSON_AddItemToObject(json, "colName", colName); - break; - } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { - cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); - cJSON_AddItemToObject(json, "colName", colName); - cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); - cJSON_AddItemToObject(json, "colType", colType); - if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY) { - int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR) { - int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); - cJSON_AddItemToObject(json, "colLength", cbytes); - } - break; - } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { - cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); - cJSON_AddItemToObject(json, "colName", colName); - cJSON* colNewName = cJSON_CreateString(vAlterTbReq.colNewName); - cJSON_AddItemToObject(json, "colNewName", colNewName); - break; - } - case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: { - cJSON* tagName = cJSON_CreateString(vAlterTbReq.tagName); - cJSON_AddItemToObject(json, "colName", tagName); - - bool isNull = vAlterTbReq.isNull; - if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { - STag* jsonTag = (STag*)vAlterTbReq.pTagVal; - if (jsonTag->nTag == 0) isNull = true; - } - if (!isNull) { - char* buf = NULL; - - if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { - ASSERT(tTagIsJson(vAlterTbReq.pTagVal) == true); - buf = parseTagDatatoJson(vAlterTbReq.pTagVal); - } else { - buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); - dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL); - } - - cJSON* colValue = cJSON_CreateString(buf); - cJSON_AddItemToObject(json, "colValue", colValue); - taosMemoryFree(buf); - } - - cJSON* isNullCJson = cJSON_CreateBool(isNull); - cJSON_AddItemToObject(json, "colValueNull", isNullCJson); - break; - } - default: - break; - } - string = cJSON_PrintUnformatted(json); - -_exit: - tDecoderClear(&decoder); - return string; -} - -static char* processDropSTable(SMqMetaRsp* metaRsp) { - SDecoder decoder = {0}; - SVDropStbReq req = {0}; - char* string = NULL; - - // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); - int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); - tDecoderInit(&decoder, data, len); - if (tDecodeSVDropStbReq(&decoder, &req) < 0) { - goto _exit; - } - - cJSON* json = cJSON_CreateObject(); - if (json == NULL) { - goto _exit; - } - cJSON* type = cJSON_CreateString("drop"); - cJSON_AddItemToObject(json, "type", type); - cJSON* tableName = cJSON_CreateString(req.name); - cJSON_AddItemToObject(json, "tableName", tableName); - cJSON* tableType = cJSON_CreateString("super"); - cJSON_AddItemToObject(json, "tableType", tableType); - - string = cJSON_PrintUnformatted(json); - -_exit: - tDecoderClear(&decoder); - return string; -} - -static char* processDropTable(SMqMetaRsp* metaRsp) { - SDecoder decoder = {0}; - SVDropTbBatchReq req = {0}; - char* string = NULL; - - // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); - int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); - tDecoderInit(&decoder, data, len); - if (tDecodeSVDropTbBatchReq(&decoder, &req) < 0) { - goto _exit; - } - - cJSON* json = cJSON_CreateObject(); - if (json == NULL) { - goto _exit; - } - cJSON* type = cJSON_CreateString("drop"); - cJSON_AddItemToObject(json, "type", type); - // cJSON* uid = cJSON_CreateNumber(id); - // cJSON_AddItemToObject(json, "uid", uid); - // cJSON* tableType = cJSON_CreateString("normal"); - // cJSON_AddItemToObject(json, "tableType", tableType); - - cJSON* tableNameList = cJSON_CreateArray(); - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - SVDropTbReq* pDropTbReq = req.pReqs + iReq; - - cJSON* tableName = cJSON_CreateString(pDropTbReq->name); - cJSON_AddItemToArray(tableNameList, tableName); - } - cJSON_AddItemToObject(json, "tableNameList", tableNameList); - - string = cJSON_PrintUnformatted(json); - -_exit: - tDecoderClear(&decoder); - return string; -} - -static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { - SVCreateStbReq req = {0}; - SDecoder coder; - SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; - - code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeSVCreateStbReq(&coder, &req) < 0) { - code = TSDB_CODE_INVALID_PARA; - goto end; - } - // build create stable - pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField)); - for (int32_t i = 0; i < req.schemaRow.nCols; i++) { - SSchema* pSchema = req.schemaRow.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; - strcpy(field.name, pSchema->name); - taosArrayPush(pReq.pColumns, &field); - } - pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); - for (int32_t i = 0; i < req.schemaTag.nCols; i++) { - SSchema* pSchema = req.schemaTag.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; - strcpy(field.name, pSchema->name); - taosArrayPush(pReq.pTags, &field); - } - - pReq.colVer = req.schemaRow.version; - pReq.tagVer = req.schemaTag.version; - pReq.numOfColumns = req.schemaRow.nCols; - pReq.numOfTags = req.schemaTag.nCols; - pReq.commentLen = -1; - pReq.suid = req.suid; - pReq.source = TD_REQ_FROM_TAOX; - pReq.igExists = true; - - STscObj* pTscObj = pRequest->pTscObj; - SName tableName; - tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); - - SCmdMsgInfo pCmdMsg = {0}; - pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - pCmdMsg.msgType = TDMT_MND_CREATE_STB; - pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - - SQuery pQuery = {0}; - pQuery.execMode = QUERY_EXEC_MODE_RPC; - pQuery.pCmdMsg = &pCmdMsg; - pQuery.msgType = pQuery.pCmdMsg->msgType; - pQuery.stableQuery = true; - - launchQueryImpl(pRequest, &pQuery, true, NULL); - - if (pRequest->code == TSDB_CODE_SUCCESS) { - SCatalog* pCatalog = NULL; - catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - catalogRemoveTableMeta(pCatalog, &tableName); - } - - code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); - -end: - destroyRequest(pRequest); - tFreeSMCreateStbReq(&pReq); - tDecoderClear(&coder); - return code; -} - -static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { - SVDropStbReq req = {0}; - SDecoder coder; - SMDropStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; - - code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeSVDropStbReq(&coder, &req) < 0) { - code = TSDB_CODE_INVALID_PARA; - goto end; - } - - // build drop stable - pReq.igNotExists = true; - pReq.source = TD_REQ_FROM_TAOX; - pReq.suid = req.suid; - - STscObj* pTscObj = pRequest->pTscObj; - SName tableName = {0}; - tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); - - SCmdMsgInfo pCmdMsg = {0}; - pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - pCmdMsg.msgType = TDMT_MND_DROP_STB; - pCmdMsg.msgLen = tSerializeSMDropStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - - SQuery pQuery = {0}; - pQuery.execMode = QUERY_EXEC_MODE_RPC; - pQuery.pCmdMsg = &pCmdMsg; - pQuery.msgType = pQuery.pCmdMsg->msgType; - pQuery.stableQuery = true; - - launchQueryImpl(pRequest, &pQuery, true, NULL); - - if (pRequest->code == TSDB_CODE_SUCCESS) { - SCatalog* pCatalog = NULL; - catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - catalogRemoveTableMeta(pCatalog, &tableName); - } - - code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); - -end: - destroyRequest(pRequest); - tDecoderClear(&coder); - return code; -} - -typedef struct SVgroupCreateTableBatch { - SVCreateTbBatchReq req; - SVgroupInfo info; - char dbName[TSDB_DB_NAME_LEN]; -} SVgroupCreateTableBatch; - -static void destroyCreateTbReqBatch(void* data) { - SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*)data; - taosArrayDestroy(pTbBatch->req.pArray); -} - -static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { - SVCreateTbBatchReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; - SQuery* pQuery = NULL; - SHashObj* pVgroupHashmap = NULL; - - code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { - code = TSDB_CODE_INVALID_PARA; - goto end; - } - - STscObj* pTscObj = pRequest->pTscObj; - - SVCreateTbReq* pCreateReq = NULL; - SCatalog* pCatalog = NULL; - code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (NULL == pVgroupHashmap) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); - - SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; - - pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); - // loop to create table - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pCreateReq = req.pReqs + iReq; - - SVgroupInfo pInfo = {0}; - SName pName = {0}; - toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - taosArrayPush(pRequest->tableList, &pName); - - SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); - if (pTableBatch == NULL) { - SVgroupCreateTableBatch tBatch = {0}; - tBatch.info = pInfo; - strcpy(tBatch.dbName, pRequest->pDb); - - tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); - taosArrayPush(tBatch.req.pArray, pCreateReq); - - taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); - } else { // add to the correct vgroup - taosArrayPush(pTableBatch->req.pArray, pCreateReq); - } - } - - SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap); - if (NULL == pBufArray) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->msgType = TDMT_VND_CREATE_TABLE; - pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); - - code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - launchQueryImpl(pRequest, pQuery, true, NULL); - if (pRequest->code == TSDB_CODE_SUCCESS) { - removeMeta(pTscObj, pRequest->tableList); - } - - code = pRequest->code; - -end: - taosHashCleanup(pVgroupHashmap); - destroyRequest(pRequest); - tDecoderClear(&coder); - qDestroyQuery(pQuery); - return code; -} - -typedef struct SVgroupDropTableBatch { - SVDropTbBatchReq req; - SVgroupInfo info; - char dbName[TSDB_DB_NAME_LEN]; -} SVgroupDropTableBatch; - -static void destroyDropTbReqBatch(void* data) { - SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data; - taosArrayDestroy(pTbBatch->req.pArray); -} - -static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { - SVDropTbBatchReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; - SQuery* pQuery = NULL; - SHashObj* pVgroupHashmap = NULL; - - code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) { - code = TSDB_CODE_INVALID_PARA; - goto end; - } - - STscObj* pTscObj = pRequest->pTscObj; - - SVDropTbReq* pDropReq = NULL; - SCatalog* pCatalog = NULL; - code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (NULL == pVgroupHashmap) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); - - SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; - pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); - // loop to create table - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pDropReq = req.pReqs + iReq; - pDropReq->igNotExists = true; - - SVgroupInfo pInfo = {0}; - SName pName = {0}; - toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - taosArrayPush(pRequest->tableList, &pName); - SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); - if (pTableBatch == NULL) { - SVgroupDropTableBatch tBatch = {0}; - tBatch.info = pInfo; - tBatch.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); - taosArrayPush(tBatch.req.pArray, pDropReq); - - taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); - } else { // add to the correct vgroup - taosArrayPush(pTableBatch->req.pArray, pDropReq); - } - } - - SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); - if (NULL == pBufArray) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->msgType = TDMT_VND_DROP_TABLE; - pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); - - code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - launchQueryImpl(pRequest, pQuery, true, NULL); - if (pRequest->code == TSDB_CODE_SUCCESS) { - removeMeta(pTscObj, pRequest->tableList); - } - code = pRequest->code; - -end: - taosHashCleanup(pVgroupHashmap); - destroyRequest(pRequest); - tDecoderClear(&coder); - qDestroyQuery(pQuery); - return code; -} - -// delete from db.tabl where .. -> delete from tabl where .. -// delete from db .tabl where .. -> delete from tabl where .. -// static void getTbName(char *sql){ -// char *ch = sql; -// -// bool inBackQuote = false; -// int8_t dotIndex = 0; -// while(*ch != '\0'){ -// if(!inBackQuote && *ch == '`'){ -// inBackQuote = true; -// ch++; -// continue; -// } -// -// if(inBackQuote && *ch == '`'){ -// inBackQuote = false; -// ch++; -// -// continue; -// } -// -// if(!inBackQuote && *ch == '.'){ -// dotIndex ++; -// if(dotIndex == 2){ -// memmove(sql, ch + 1, strlen(ch + 1) + 1); -// break; -// } -// } -// ch++; -// } -//} - -static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { - SDeleteRes req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - - // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeDeleteRes(&coder, &req) < 0) { - code = TSDB_CODE_INVALID_PARA; - goto end; - } - - // getTbName(req.tableFName); - char sql[256] = {0}; - sprintf(sql, "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, req.tsColName, - req.skey, req.tsColName, req.ekey); - printf("delete sql:%s\n", sql); - - TAOS_RES* res = taos_query(taos, sql); - SRequestObj* pRequest = (SRequestObj*)res; - code = pRequest->code; - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - code = TSDB_CODE_SUCCESS; - } - taos_free_result(res); - -end: - tDecoderClear(&coder); - return code; -} - -static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { - SVAlterTbReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; - SQuery* pQuery = NULL; - SArray* pArray = NULL; - SVgDataBlocks* pVgData = NULL; - - code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); - - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); - tDecoderInit(&coder, data, len); - if (tDecodeSVAlterTbReq(&coder, &req) < 0) { - code = TSDB_CODE_INVALID_PARA; - goto end; - } - - // do not deal TSDB_ALTER_TABLE_UPDATE_OPTIONS - if (req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS) { - goto end; - } - - STscObj* pTscObj = pRequest->pTscObj; - SCatalog* pCatalog = NULL; - code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; - - SVgroupInfo pInfo = {0}; - SName pName = {0}; - toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - pArray = taosArrayInit(1, sizeof(void*)); - if (NULL == pArray) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - - pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == pVgData) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pVgData->vg = pInfo; - pVgData->pData = taosMemoryMalloc(metaLen); - if (NULL == pVgData->pData) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - memcpy(pVgData->pData, meta, metaLen); - ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId); - pVgData->size = metaLen; - pVgData->numOfTables = 1; - taosArrayPush(pArray, &pVgData); - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->msgType = TDMT_VND_ALTER_TABLE; - pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); - - code = rewriteToVnodeModifyOpStmt(pQuery, pArray); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - launchQueryImpl(pRequest, pQuery, true, NULL); - - pVgData = NULL; - pArray = NULL; - code = pRequest->code; - if (code == TSDB_CODE_VND_TABLE_NOT_EXIST) { - code = TSDB_CODE_SUCCESS; - } - - if (pRequest->code == TSDB_CODE_SUCCESS) { - SExecResult* pRes = &pRequest->body.resInfo.execRes; - if (pRes->res != NULL) { - code = handleAlterTbExecRes(pRes->res, pCatalog); - } - } -end: - taosArrayDestroy(pArray); - if (pVgData) taosMemoryFreeClear(pVgData->pData); - taosMemoryFreeClear(pVgData); - destroyRequest(pRequest); - tDecoderClear(&coder); - qDestroyQuery(pQuery); - return code; -} - -typedef struct { - SVgroupInfo vg; - void* data; -} VgData; - -static void destroyVgHash(void* data) { - VgData* vgData = (VgData*)data; - taosMemoryFreeClear(vgData->data); -} - -int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { - int32_t code = TSDB_CODE_SUCCESS; - STableMeta* pTableMeta = NULL; - SQuery* pQuery = NULL; - - SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); - if (!pRequest) { - uError("WriteRaw:createRequest error request is null"); - code = terrno; - goto end; - } - - if (!pRequest->pDb) { - uError("WriteRaw:not use db"); - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - - SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - strcpy(pName.dbname, pRequest->pDb); - strcpy(pName.tname, tbname); - - struct SCatalog* pCatalog = NULL; - code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: get gatlog error"); - goto end; - } - - SRequestConnInfo conn = {0}; - conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; - conn.requestId = pRequest->requestId; - conn.requestObjRefId = pRequest->self; - conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - - SVgroupInfo vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vgData); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbname); - goto end; - } - - code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); - goto end; - } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < numOfCols; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numOfCols; - - SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - - offset += TYPE_BYTES[pColumn->type]; - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - nodeStmt->payloadType = PAYLOAD_TYPE_KV; - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto end; - } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); - - launchQueryImpl(pRequest, pQuery, true, NULL); - code = pRequest->code; - -end: - taosMemoryFreeClear(pTableMeta); - qDestroyQuery(pQuery); - return code; -} - -static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) { - int32_t code = TSDB_CODE_SUCCESS; - SHashObj* pVgHash = NULL; - SQuery* pQuery = NULL; - SMqRspObj rspObj = {0}; - SDecoder decoder = {0}; - - terrno = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); - if (!pRequest) { - uError("WriteRaw:createRequest error request is null"); - return terrno; - } - - rspObj.resIter = -1; - rspObj.resType = RES_TYPE__TMQ; - - tDecoderInit(&decoder, data, dataLen); - code = tDecodeSMqDataRsp(&decoder, &rspObj.rsp); - if (code != 0) { - uError("WriteRaw:decode smqDataRsp error"); - code = TSDB_CODE_INVALID_MSG; - goto end; - } - - if (!pRequest->pDb) { - uError("WriteRaw:not use db"); - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); - struct SCatalog* pCatalog = NULL; - code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: get gatlog error"); - goto end; - } - - SRequestConnInfo conn = {0}; - conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; - conn.requestId = pRequest->requestId; - conn.requestObjRefId = pRequest->self; - conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - - printf("raw data block num:%d\n", rspObj.rsp.blockNum); - while (++rspObj.resIter < rspObj.rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); - if (!rspObj.rsp.withSchema) { - uError("WriteRaw:no schema, iter:%d", rspObj.resIter); - goto end; - } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = pSW->pSchema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pSW->nCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); - if (!tbName) { - uError("WriteRaw: tbname is null"); - code = TSDB_CODE_TMQ_INVALID_MSG; - goto end; - } - - printf("raw data tbname:%s\n", tbName); - SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - strcpy(pName.dbname, pRequest->pDb); - strcpy(pName.tname, tbName); - - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); - } - - STableMeta* pTableMeta = NULL; - code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbName); - goto end; - } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - taosMemoryFreeClear(pTableMeta); - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pSW->version); - tdSRowSetTpInfo(&rb, pSW->nCols, fLen); - int32_t dataLen = 0; - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pSW->nCols; k++) { - const SSchema* pColumn = &pSW->pSchema[k]; - char* data = rspObj.resInfo.row[k]; - if (!data) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - data -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } - offset += TYPE_BYTES[pColumn->type]; - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pSW->version); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks++; - } - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - nodeStmt->payloadType = PAYLOAD_TYPE_KV; - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto end; - } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); - } - - launchQueryImpl(pRequest, pQuery, true, NULL); - code = pRequest->code; - -end: - tDecoderClear(&decoder); - taos_free_result(&rspObj); - qDestroyQuery(pQuery); - destroyRequest(pRequest); - taosHashCleanup(pVgHash); - return code; -} - -char* tmq_get_json_meta(TAOS_RES* res) { - if (!TD_RES_TMQ_META(res)) { - return NULL; - } - - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { - return processCreateStb(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { - return processAlterStb(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { - return processDropSTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { - return processCreateTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { - return processAlterTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { - return processDropTable(&pMetaRspObj->metaRsp); - } - return NULL; -} - -void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } - -int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { - if (!raw || !res) { - return TSDB_CODE_INVALID_PARA; - } - if (TD_RES_TMQ_META(res)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - raw->raw = pMetaRspObj->metaRsp.metaRsp; - raw->raw_len = pMetaRspObj->metaRsp.metaRspLen; - raw->raw_type = pMetaRspObj->metaRsp.resMsgType; - } else if (TD_RES_TMQ(res)) { - SMqRspObj* rspObj = ((SMqRspObj*)res); - - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSMqDataRsp, &rspObj->rsp, len, code); - if (code < 0) { - return -1; - } - - void* buf = taosMemoryCalloc(1, len); - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, len); - tEncodeSMqDataRsp(&encoder, &rspObj->rsp); - tEncoderClear(&encoder); - - raw->raw = buf; - raw->raw_len = len; - raw->raw_type = RES_TYPE__TMQ; - } else { - return TSDB_CODE_TMQ_INVALID_MSG; - } - return TSDB_CODE_SUCCESS; -} - -void tmq_free_raw(tmq_raw_data raw) { - if (raw.raw_type == RES_TYPE__TMQ) { - taosMemoryFree(raw.raw); - } -} - -int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { - if (!taos) { - return TSDB_CODE_INVALID_PARA; - } - - if (raw.raw_type == TDMT_VND_CREATE_STB) { - return taosCreateStb(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_ALTER_STB) { - return taosCreateStb(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_DROP_STB) { - return taosDropStb(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_CREATE_TABLE) { - return taosCreateTable(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_ALTER_TABLE) { - return taosAlterTable(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_DROP_TABLE) { - return taosDropTable(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_DELETE) { - return taosDeleteData(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == RES_TYPE__TMQ) { - return tmqWriteRaw(taos, raw.raw, raw.raw_len); - } - return TSDB_CODE_INVALID_PARA; -} - void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { // tmqCommitInner(tmq, msg, 0, 1, cb, param); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6e3067d44edc..7b06967940e4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -624,6 +624,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Invalid index file" //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") #ifdef TAOS_ERROR_C };