fix: taosbenchmark schemaless distinguish opentsdb json and taosjson (#…

…560) * test: change test branch * fix: optimize data preparing for sml json * fix: use unformat json print in progressive * test: add sml json alltypes interlface test * fix: refactor debug print and echo print * fix: typos in 2.x-centos-release.yml and coveralls.yml * chore: add 3.0-macos-debug.yml * fix: update macos yml * fix: more ymls * fix: workflow env variables * fix: distinguish standard json and taos json * fix: typos * fix: distinguish logic * test: split sml command line test * test: improve test coverage in taosbenchmark/json/sml_taosjson_alltypes.json * test: chang test branch * fix: cmd line coverage test * test: revert test branch for 3.0-alpine.yml * fix: issues codacy reports * fix: support opentsdb standard json * fix: sml json test for 2.x
taosdata · Feb 1, 2023 · 181bcac · 181bcac
1 parent c09bf1e
commit 181bcac
Show file tree

Hide file tree

Showing 32 changed files with 894 additions and 123 deletions.
diff --git a/.github/workflows/3.0-coveralls.yml b/.github/workflows/3.0-coveralls.yml
@@ -70,7 +70,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if:
@@ -181,7 +181,7 @@ jobs:
           cd tests/taosdump
           ./exp.sh
 
-      - name: Command Line Basic Test
+      - name: taosBenchmark Command Line Basic Test
         if:
           (steps.changed-files-specific.outputs.any_changed == 'true'
           && github.event_name == 'pull_request')
@@ -206,11 +206,30 @@ jobs:
           ./build/bin/taosBenchmark -D NotANumber > /dev/null || :
           echo "TEST: taosBenchmark -v NotANumber"
           ./build/bin/taosBenchmark -v NotANumber > /dev/null || :
+          echo "TEST: taosBenchmark -o out.txt"
+          ./build/bin/taosBenchmark -t 1 -n 1 -o out.txt -y > /dev/null
+          echo "TEST: taosBenchmark -c /etc/taos"
+          ./build/bin/taosBenchmark -c /etc/taos -t 1 -n 1 -y > /dev/null
           echo "TEST: kill taosBenchmark"
           ./build/bin/taosBenchmark -t 1 -n 100 -B 1 -i 1000 -r 1 -y &
           sleep 5
           pkill -INT taosBenchmark
 
+      - name: taosdump Command Line Basic Test
+        if:
+          (steps.changed-files-specific.outputs.any_changed == 'true'
+          && github.event_name == 'pull_request')
+          || github.event_name == 'push'
+        run: |
+          sudo pkill -9 taosadapter || :
+          sudo pkill -9 taosd || :
+          sleep 5
+
+          nohup taosadapter > /dev/null &
+          nohup sudo TAOS_SUPPORT_VNODES=256 taosd > /dev/null &
+          sleep 10
+
+          cd debug
           echo "TEST: taodump --help"
           ./build/bin/taosdump --help > /dev/null
           echo "TEST: taodump --version"

diff --git a/.github/workflows/3.0-macos-debug.yml b/.github/workflows/3.0-macos-debug.yml
@@ -64,7 +64,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if: |

diff --git a/.github/workflows/3.0-macos-release.yml b/.github/workflows/3.0-macos-release.yml
@@ -64,7 +64,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if: |

diff --git a/.github/workflows/3.0-non-x64.yml b/.github/workflows/3.0-non-x64.yml
@@ -134,7 +134,7 @@ jobs:
 
             echo "clone TDengine 3.0 on ${{ steps.setup.outputs.uname }}"
 
-            git clone --branch fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud  --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1
+            git clone --branch fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine --depth 1 https://github.com/taosdata/TDengine > /dev/null || exit 1
 
             echo "build TDengine 3.0 on ${{ steps.setup.outputs.uname }}"
             cd TDengine || exit 1

diff --git a/.github/workflows/3.0-taosBenchmark-debug.yml b/.github/workflows/3.0-taosBenchmark-debug.yml
@@ -61,7 +61,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if: |

diff --git a/.github/workflows/3.0-taosBenchmark-release.yml b/.github/workflows/3.0-taosBenchmark-release.yml
@@ -61,7 +61,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if: |

diff --git a/.github/workflows/3.0-taosdump-debug.yml b/.github/workflows/3.0-taosdump-debug.yml
@@ -65,7 +65,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if:

diff --git a/.github/workflows/3.0-taosdump-release-ws.yml b/.github/workflows/3.0-taosdump-release-ws.yml
@@ -65,7 +65,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if:

diff --git a/.github/workflows/3.0-taosdump-release.yml b/.github/workflows/3.0-taosdump-release.yml
@@ -65,7 +65,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: Change time zone
         if:

diff --git a/.github/workflows/3.0-windows-build.yml b/.github/workflows/3.0-windows-build.yml
@@ -62,7 +62,7 @@ jobs:
         with:
           repository: 'taosdata/TDengine'
           path: 'TDengine'
-          ref: 'fix/sangshuduo/TD-22022-ignore-drop-failure-for-cloud'
+          ref: 'fix/sangshuduo/TD-21932-taosbenchmark-schemaless-refine'
 
       - name: create debug directory
         if: |

diff --git a/example/insert.json b/example/insert.json
@@ -20,12 +20,7 @@
             "dbinfo": {
                 "name": "test",
                 "drop": "yes",
-                "replica": 1,
-                "precision": "ms",
-                "keep": 3650,
-                "minRows": 100,
-                "maxRows": 4096,
-                "comp": 2
+                "precision": "ms"
             },
             "super_tables": [
                 {

diff --git a/example/sml_telnet.json b/example/sml_telnet.json
@@ -0,0 +1,100 @@
+{
+  "filetype": "insert",
+  "cfgdir": "/etc/taos",
+  "host": "127.0.0.1",
+  "port": 6030,
+  "user": "root",
+  "password": "taosdata",
+  "thread_count": 4,
+  "connection_pool_size": 10,
+  "result_file": "./insert_res.txt",
+  "confirm_parameter_prompt": "no",
+  "prepared_rand": 10,
+  "chinese": "no",
+  "insert_interval": 0,
+  "num_of_records_per_req": 10,
+  "databases": [
+    {
+      "dbinfo": {
+        "name": "test",
+        "drop": "yes"
+      },
+      "super_tables": [
+        {
+          "name": "meters",
+          "child_table_exists": "no",
+          "childtable_count": 10,
+          "childtable_prefix": "d",
+          "escape_character": "no",
+          "auto_create_table": "no",
+          "batch_create_tbl_num": 10,
+          "data_source": "rand",
+          "insert_mode": "sml",
+          "line_protocol": "telnet",
+          "childtable_limit": 0,
+          "childtable_offset": 0,
+          "insert_rows": 10,
+          "insert_interval": 0,
+          "interlace_rows": 0,
+          "disorder_ratio": 0,
+          "disorder_range": 1000,
+          "timestamp_step": 1,
+          "start_timestamp": "now",
+          "sample_file": "",
+          "use_sample_ts": "no",
+          "tags_file": "",
+          "columns": [
+            {
+              "type": "BOOL"
+            }
+          ],
+          "tags": [
+            {
+              "type": "INT"
+            },
+            {
+              "type": "BIGINT"
+            },
+            {
+              "type": "FLOAT"
+            },
+            {
+              "type": "DOUBLE"
+            },
+            {
+              "type": "SMALLINT"
+            },
+            {
+              "type": "TINYINT"
+            },
+            {
+              "type": "BOOL"
+            },
+            {
+              "type": "NCHAR",
+              "len": 17,
+              "count": 1
+            },
+            {
+              "type": "UINT"
+            },
+            {
+              "type": "UBIGINT"
+            },
+            {
+              "type": "UTINYINT"
+            },
+            {
+              "type": "USMALLINT"
+            },
+            {
+              "type": "BINARY",
+              "len": 19,
+              "count": 1
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
diff --git a/inc/bench.h b/inc/bench.h
@@ -154,6 +154,8 @@
 #define BARRAY_MIN_SIZE 8
 #define SML_LINE_SQL_SYNTAX_OFFSET 7
 
+#define SML_JSON_TAOS_FORMAT    255
+
 
 #define BENCH_FILE  "(**IMPORTANT**) Set JSON configuration file(all options are going to read from this JSON file), which is mutually exclusive with other commandline options, examples are under /usr/local/taos/examples"
 #define BENCH_CFG_DIR "Configuration directory."

diff --git a/inc/benchData.h b/inc/benchData.h
@@ -31,4 +31,8 @@ void generateSmlJsonTags(tools_cJSON *tagsList, SSuperTable *stbInfo,
                             uint64_t start_table_from, int tbSeq);
 void generateSmlJsonCols(tools_cJSON *array, tools_cJSON *tag, SSuperTable *stbInfo,
                             uint32_t time_precision, int64_t timestamp);
+void generateSmlTaosJsonTags(tools_cJSON *tagsList, SSuperTable *stbInfo,
+                            uint64_t start_table_from, int tbSeq);
+void generateSmlTaosJsonCols(tools_cJSON *array, tools_cJSON *tag, SSuperTable *stbInfo,
+                            uint32_t time_precision, int64_t timestamp);
 #endif
diff --git a/src/benchCommandOpt.c b/src/benchCommandOpt.c
@@ -179,8 +179,19 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) {
                 if (false == g_arguments->port_inputted) {
                     g_arguments->port = DEFAULT_REST_PORT;
                 }
-            } else if (0 == strcasecmp(arg, "sml")) {
+            } else if (0 == strcasecmp(arg, "sml")
+                    || 0 == strcasecmp(arg, "sml-line")) {
                 stbInfo->iface = SML_IFACE;
+                stbInfo->lineProtocol = TSDB_SML_LINE_PROTOCOL;
+            } else if (0 == strcasecmp(arg, "sml-telnet")) {
+                stbInfo->iface = SML_IFACE;
+                stbInfo->lineProtocol = TSDB_SML_TELNET_PROTOCOL;
+            } else if (0 == strcasecmp(arg, "sml-json")) {
+                stbInfo->iface = SML_IFACE;
+                stbInfo->lineProtocol = TSDB_SML_JSON_PROTOCOL;
+            } else if (0 == strcasecmp(arg, "sml-taosjson")) {
+                stbInfo->iface = SML_IFACE;
+                stbInfo->lineProtocol = SML_JSON_TAOS_FORMAT;
             } else {
                 errorPrint(
                            "Invalid -I: %s, will auto set to default (taosc)\n",
@@ -396,13 +407,13 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) {
                         arg);
                 g_arguments->binwidth = DEFAULT_BINWIDTH;
             } else if (g_arguments->binwidth >
-			    (TSDB_MAX_BINARY_LEN - sizeof(int64_t) -2)) {
+                (TSDB_MAX_BINARY_LEN - sizeof(int64_t) -2)) {
                 errorPrint(
-                           "-w(%d) > (TSDB_MAX_BINARY_LEN(%u"
-                                   ")-(TIMESTAMP length(%zu) - extrabytes(2), "
-				   "will auto set to default(64)\n",
-                           g_arguments->binwidth,
-			   TSDB_MAX_BINARY_LEN, sizeof(int64_t));
+                        "-w(%d) > (TSDB_MAX_BINARY_LEN(%u"
+                        ")-(TIMESTAMP length(%zu) - extrabytes(2), "
+                        "will auto set to default(64)\n",
+                        g_arguments->binwidth,
+                        TSDB_MAX_BINARY_LEN, sizeof(int64_t));
                 g_arguments->binwidth = DEFAULT_BINWIDTH;
             }
             break;
@@ -515,7 +526,7 @@ int32_t benchParseSingleOpt(int32_t key, char* arg) {
             }
             g_arguments->nthreads_auto = false;
             g_arguments->inputted_vgroups = atoi(arg);
-	    break;
+            break;
 #endif
 
         case 'V':