Skip to content

Commit

Permalink
merge: v3.3.2.0 3.0 to main
Browse files Browse the repository at this point in the history
  • Loading branch information
DuanKuanJun committed Jun 26, 2024
2 parents 5cbf094 + 156def6 commit e8a6c45
Show file tree
Hide file tree
Showing 23 changed files with 1,396 additions and 476 deletions.
77 changes: 77 additions & 0 deletions case/exportCsv.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{
"filetype": "csvfile",
"csvPath": "/root/csv/",
"num_of_records_per_req": 10000,
"databases": [
{
"dbinfo": {
"name": "csvdb"
},
"super_tables": [
{
"name": "batchTable",
"childtable_count": 10,
"insert_rows": 50,
"childtable_prefix": "d",
"timestamp_step": 10,
"start_timestamp":1600000000000,
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "min": 1},
{ "type": "double", "name": "dc", "min":10, "max":10},
{ "type": "tinyint", "name": "ti"},
{ "type": "smallint", "name": "si"},
{ "type": "int", "name": "ic", "fillNull":"false"},
{ "type": "bigint", "name": "bi"},
{ "type": "utinyint", "name": "uti"},
{ "type": "usmallint", "name": "usi", "min":100, "max":120},
{ "type": "uint", "name": "ui"},
{ "type": "ubigint", "name": "ubi"},
{ "type": "binary", "name": "bin", "len": 16},
{ "type": "nchar", "name": "nch", "len": 16}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 10,"min": 1},
{"type": "binary", "name": "location", "len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
},
{
"name": "interlaceTable",
"childtable_count": 5,
"insert_rows": 100,
"interlace_rows": 10,
"childtable_prefix": "e",
"timestamp_step": 1000,
"start_timestamp":1700000000000,
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "min":16},
{ "type": "double", "name": "dc", "min":16},
{ "type": "tinyint", "name": "ti"},
{ "type": "smallint", "name": "si"},
{ "type": "int", "name": "ic", "fillNull":"false"},
{ "type": "bigint", "name": "bi"},
{ "type": "utinyint", "name": "uti"},
{ "type": "usmallint", "name": "usi"},
{ "type": "uint", "name": "ui"},
{ "type": "ubigint", "name": "ubi"},
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 10,"min": 1},
{"type": "binary", "name": "location", "len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}
58 changes: 58 additions & 0 deletions case/insertBindVGroup.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"num_of_records_per_req": 200,
"thread_count": 20,
"thread_bind_vgroup": "yes",
"create_table_thread_count": 1,
"confirm_parameter_prompt": "no",
"databases": [
{
"dbinfo": {
"name": "binddb",
"drop": "yes",
"vgroups": 2
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 4,
"insert_rows": 100,
"interlace_rows": 10,
"childtable_prefix": "d",
"insert_mode": "taosc",
"timestamp_step": 1000,
"start_timestamp":1500000000000,
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 1, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0 },
{ "type": "int", "name": "ic", "max": 100, "min": 0 },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 100, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0 },
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{"type": "tinyint", "name": "groupid","max": 10,"min": 1},
{"type": "binary", "name": "location", "len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}
2 changes: 1 addition & 1 deletion deps/avro/lang/c/examples/quickstop.c
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ int print_person(avro_file_reader_t db, avro_schema_t reader_schema)
if (rval == 0) {
int64_t id;
int32_t age;
int32_t *p;
char *p = NULL;
size_t size;
avro_value_t id_value;
avro_value_t first_value;
Expand Down
12 changes: 8 additions & 4 deletions inc/bench.h
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,7 @@ enum TEST_MODE {
INSERT_TEST, // 0
QUERY_TEST, // 1
SUBSCRIBE_TEST, // 2
CSVFILE_TEST // 3
};

enum enumSYNC_MODE { SYNC_MODE, ASYNC_MODE, MODE_BUT };
Expand Down Expand Up @@ -646,6 +647,8 @@ typedef struct STSMA {
#define SUIT_DATAPOS_MUL_FILE 4
#define SUIT_DATAPOS_MIX 5

#define VAL_NULL "NULL"

enum CONTINUE_IF_FAIL_MODE {
NO_IF_FAILED, // 0
YES_IF_FAILED, // 1
Expand Down Expand Up @@ -788,14 +791,12 @@ typedef struct SSTREAM_S {
bool drop;
} SSTREAM;

#ifdef TD_VER_COMPATIBLE_3_0_0_0
typedef struct SVGroup_S {
int32_t vgId;
uint64_t tbCountPerVgId;
SChildTable **childTblArray;
uint64_t tbOffset; // internal use
} SVGroup;
#endif // TD_VER_COMPATIBLE_3_0_0_0
//
typedef struct SDataBase_S {
char * dbName;
Expand Down Expand Up @@ -923,7 +924,6 @@ typedef struct SArguments_S {
uint32_t binwidth;
uint32_t intColumnCount;
uint32_t nthreads;
bool nthreads_auto;
uint32_t table_threads;
uint64_t prepared_rand;
uint32_t reqPerReq;
Expand Down Expand Up @@ -965,6 +965,9 @@ typedef struct SArguments_S {
bool mistMode;
bool escape_character;
bool pre_load_tb_meta;
char csvPath[MAX_FILE_NAME_LEN];

bool bind_vgroup;
} SArguments;

typedef struct SBenchConn {
Expand Down Expand Up @@ -1160,7 +1163,7 @@ int32_t benchGetTotalMemory(int64_t *totalKB);
int32_t benchParseArgsNoArgp(int argc, char* argv[]);
#endif

int32_t execInsert(threadInfo *pThreadInfo, uint32_t k);
int32_t execInsert(threadInfo *pThreadInfo, uint32_t k, int64_t* delay3);
// if return true, timestmap must add timestap_step, else timestamp no need changed
bool needChangeTs(SSuperTable * stbInfo, int32_t *pkCur, int32_t *pkCnt);

Expand All @@ -1177,5 +1180,6 @@ uint64_t tmpUint64Impl(Field *field, int32_t angle, int64_t k);
float tmpFloatImpl(Field *field, int i, int32_t angle, int32_t k);
double tmpDoubleImpl(Field *field, int32_t angle, int32_t k);
int tmpStr(char *tmp, int iface, Field *field, int64_t k);
int tmpGeometry(char *tmp, int iface, Field *field, int64_t k);

#endif // INC_BENCH_H_
36 changes: 36 additions & 0 deletions inc/benchCsv.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the MIT license as published by the Free Software
* Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef INC_BENCHCSV_H_
#define INC_BENCHCSV_H_

#include <bench.h>

int csvTestProcess();

int genWithSTable(SDataBase* db, SSuperTable* stb, char* outDir);

char * genTagData(char* buf, SSuperTable* stb, int64_t i, int64_t *k);

char * genColumnData(char* colData, SSuperTable* stb, int64_t ts, int32_t precision, int64_t *k);

int32_t genRowByField(char* buf, BArray* fields, int16_t fieldCnt, char* binanryPrefix, char* ncharPrefix, int64_t *k);

void obtainCsvFile(char * outFile, SDataBase* db, SSuperTable* stb, char* outDir);

int interlaceWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain);
int batchWriteCsv(SDataBase* db, SSuperTable* stb, FILE* fs, char* buf, int bufLen, int minRemain);

#endif // INC_BENCHCSV_H_
6 changes: 4 additions & 2 deletions inc/benchData.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
/***** Global variables ******/
/***** Declare functions *****/
void rand_string(char *str, int size, bool chinese);
int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio,
int disorderRange);
void rand_geometry(char *str, int fieldLen, int maxType);
int geoCalcBufferSize(int fieldLen);
int getGeoMaxType(int fieldLen);
int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, int disorderRange);
int generateRandData(SSuperTable *stbInfo, char *sampleDataBuf,
int64_t bufLen,
int lenOfOneRow, BArray * fields, int64_t loop,
Expand Down
2 changes: 1 addition & 1 deletion inc/benchDataMix.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
#define __BENCHDATAMIX_H_


uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k);
uint32_t dataGenByField(Field* fd, char* pstr, uint32_t len, char* prefix, int64_t *k, char* nullVal);

// data generate by calc ts
uint32_t dataGenByCalcTs(Field* fd, char* pstr, uint32_t len, int64_t ts);
Expand Down
12 changes: 6 additions & 6 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
ADD_DEPENDENCIES(taosdump deps-jansson)
ADD_DEPENDENCIES(taosdump deps-snappy)
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ELSE ()
INCLUDE_DIRECTORIES(/usr/local/include)
Expand All @@ -246,9 +246,9 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
SET(OS_ID "Darwin")

IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ELSE()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF()
ENDIF ()

Expand Down Expand Up @@ -441,9 +441,9 @@ ELSE ()
SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /utf-8")
SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /utf-8")
IF (${TD_VER_COMPATIBLE} STRGREATER_EQUAL "3.0.0.0")
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchTmq.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsString.c toolsSys.c toolsString.c)
ELSE ()
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ADD_EXECUTABLE(taosBenchmark benchMain.c benchSubscribe.c benchQuery.c benchCsv.c benchJsonOpt.c benchInsert.c benchInsertMix.c benchDataMix.c wrapDb.c benchData.c benchCommandOpt.c benchUtil.c benchUtilDs.c benchSys.c toolstime.c toolsSys.c toolsString.c)
ENDIF ()

ADD_EXECUTABLE(taosdump taosdump.c toolsSys.c toolstime.c toolsDir.c toolsString.c)
Expand Down
7 changes: 1 addition & 6 deletions src/benchCommandOpt.c
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ extern char g_configDir[MAX_PATH_LEN];
#define TAOSBENCHMARK_STATUS "unknown"
#endif

// libtaos.so
extern char buildinfo[];

char *g_aggreFuncDemo[] = {"*",
"count(*)",
Expand All @@ -49,9 +47,7 @@ void printVersion() {

// version
printf("taosBenchmark version: %s\ngit: %s\n", taosBenchmark_ver, taosBenchmark_commit);
#ifdef LINUX
printf("build: %s\n", buildinfo);
#endif
printf("build: %s\n", getBuildInfo());
if (strlen(taosBenchmark_status) > 0) {
printf("status: %s\n", taosBenchmark_status);
}
Expand Down Expand Up @@ -242,7 +238,6 @@ void initArgument() {
g_arguments->performance_print = 0;
g_arguments->output_file = DEFAULT_OUTPUT;
g_arguments->nthreads = DEFAULT_NTHREADS;
g_arguments->nthreads_auto = true;
g_arguments->table_threads = DEFAULT_NTHREADS;
g_arguments->prepared_rand = DEFAULT_PREPARED_RAND;
g_arguments->reqPerReq = DEFAULT_REQ_PER_REQ;
Expand Down
Loading

0 comments on commit e8a6c45

Please sign in to comment.