Merge pull request #21902 from taosdata/fix/32m.0

Fix/32m.0

cmake/cmake.define

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0)
 
-set(CMAKE_VERBOSE_MAKEFILE OFF)
+set(CMAKE_VERBOSE_MAKEFILE ON)
 set(TD_BUILD_TAOSA_INTERNAL FALSE)
 
 #set output directory

cmake/cmake.platform

@@ -188,7 +188,7 @@ if (TD_LINUX)
     set(TD_DEPS_DIR "x86")
   ENDIF()
 endif()
-MESSAGE(STATUS "DEPS_DIR" ${TD_DEPS_DIR})
+MESSAGE(STATUS "DEPS_DIR: " ${TD_DEPS_DIR})
 
 MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
 MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")

cmake/cmake.version

@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.5.2.alpha")
+  SET(TD_VER_NUMBER "3.0.6.0.alpha")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

contrib/CMakeLists.txt

@@ -77,7 +77,6 @@ if(${BUILD_WITH_LEVELDB})
     cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_LEVELDB})
 
-
 if (${BUILD_CONTRIB})
     if(${BUILD_WITH_ROCKSDB})
         cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -96,7 +95,6 @@ else()
     endif()
 endif()
 
-
 # canonical-raft
 if(${BUILD_WITH_CRAFT})
     cat("${TD_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})

docs/en/07-develop/07-tmq.mdx

@@ -244,6 +244,8 @@ The following SQL statement creates a topic in TDengine:
 CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1;
 ```
 
+- There is an upper limit to the number of topics created, controlled by the parameter tmqMaxTopicNum, with a default of 20
+
 Multiple subscription types are supported.
 
 #### Subscribe to a Column
@@ -265,25 +267,28 @@ You can subscribe to a topic through a SELECT statement. Statements that specify
 Syntax:
 
 ```sql
-CREATE TOPIC topic_name AS STABLE stb_name
+CREATE TOPIC topic_name [with meta] AS STABLE stb_name [where_condition]
 ```
 
 Creating a topic in this manner differs from a `SELECT * from stbName` statement as follows:
 
 - The table schema can be modified.
 - Unstructured data is returned. The format of the data returned changes based on the supertable schema.
-- A different table schema may exist for every data block to be processed.
+- The 'with meta' parameter is optional. When selected, statements such as creating super tables and sub tables will be returned, mainly used for Taosx to perform super table migration
+- The 'where_condition' parameter is optional and will be used to filter and subscribe to sub tables that meet the criteria. Where conditions cannot have ordinary columns, only tags or tbnames. Functions can be used in where conditions to filter tags, but cannot be aggregate functions because sub table tag values cannot be aggregated. It can also be a constant expression, such as 2>1 (subscribing to all child tables), Or false (subscribe to 0 sub tables)
 - The data returned does not include tags.
 
 ### Subscribe to a Database
 
 Syntax:
 
 ```sql
-CREATE TOPIC topic_name [WITH META] AS DATABASE db_name;
+CREATE TOPIC topic_name [with meta] AS DATABASE db_name;
 ```
 
-This SQL statement creates a subscription to all tables in the database. You can add the `WITH META` parameter to include schema changes in the subscription, including creating and deleting supertables; adding, deleting, and modifying columns; and creating, deleting, and modifying the tags of subtables. Consumers can determine the message type from the API. Note that this differs from Kafka.
+This SQL statement creates a subscription to all tables in the database.
+
+- The 'with meta' parameter is optional. When selected, it will return statements for creating all super tables and sub tables in the database, mainly used for Taosx database migration
 
 ## Create a Consumer
 
@@ -295,7 +300,7 @@ You configure the following parameters when creating a consumer:
 |        `td.connect.user`         | string  | User Name                          |  |
 |        `td.connect.pass`         | string  | Password                        |   |
 |        `td.connect.port`         | string  | Port of the server side            |   |
-|           `group.id`           | string  | Consumer group ID; consumers with the same ID are in the same group                        | **Required**. Maximum length: 192.                 |
+|           `group.id`           | string  | Consumer group ID; consumers with the same ID are in the same group                        | **Required**. Maximum length: 192. Each topic can create up to 100 consumer groups.                 |
 |          `client.id`           | string  | Client ID                                                | Maximum length: 192.                             |
 |      `auto.offset.reset`       |  enum   | Initial offset for the consumer group                                     | Specify `earliest`, `latest`, or `none`(default) |
 |      `enable.auto.commit`      | boolean | Commit automatically; true: user application doesn't need to explicitly commit; false: user application need to handle commit by itself                                           | Default value is true                  |

docs/en/12-taos-sql/06-select.md

@@ -55,7 +55,7 @@ window_clause: {
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)]
 
 interp_clause:
-    RANGE(ts_val, ts_val) EVERY(every_val) FILL(fill_mod_and_val)
+    RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
 
 partition_by_clause:
     PARTITION BY expr [, expr] ...

docs/en/12-taos-sql/10-function.md

@@ -889,9 +889,10 @@ ignore_null_values: {
 - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter.
 - The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input.
 - `INTERP` must be used along with `RANGE`, `EVERY`, `FILL` keywords.
-- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1 <= timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified.
+- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1 <= timestamp2. timestamp1 is the starting point of the output time range. timestamp2 is the ending point of the output time range.
 - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY(time_unit)`. Starting from timestamp1, one interpolation is performed for every time interval specified `time_unit` parameter. The parameter `time_unit` must be an integer, with no quotes, with a time unit of: a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds.
 - Interpolation is performed based on `FILL` parameter. For more information about FILL clause, see [FILL Clause](../distinguished/#fill-clause).
+- When only one timestamp value is specified in `RANGE` clause, `INTERP` is used to generate interpolation at this point in time. In this case, `EVERY` clause can be omitted. For example, SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear).
 - `INTERP` can be applied to supertable by interpolating primary key sorted data of all its childtables. It can also be used with `partition by tbname` when applied to supertable to generate interpolation on each single timeline.
 - Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.2.0).
 - Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.3.0).
@@ -902,7 +903,7 @@ ignore_null_values: {
 - We want to downsample every 1 hour and use a linear fill for missing values. Note the order in which the "partition by" clause and the "range", "every" and "fill" parameters are used.
 
 ```sql
-SELECT _irowts,INTERP(current) FROM test.meters PARTITION BY TBNAME RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(LINEAR) 
+SELECT _irowts,INTERP(current) FROM test.meters PARTITION BY TBNAME RANGE('2017-07-22 00:00:00','2017-07-24 12:25:00') EVERY(1h) FILL(LINEAR)
 ```
 
 ### LAST

docs/en/12-taos-sql/24-show.md

@@ -36,7 +36,7 @@ Shows information about connections to the system.
 SHOW CONSUMERS;
 ```
 
-Shows information about all active consumers in the system.
+Shows information about all consumers in the system.
 
 ## SHOW CREATE DATABASE
 

docs/zh/07-develop/07-tmq.mdx

@@ -243,6 +243,7 @@ TDengine 使用 SQL 创建一个 topic：
 ```sql
 CREATE TOPIC topic_name AS SELECT ts, c1, c2, c3 FROM tmqdb.stb WHERE c1 > 1;
 ```
+- topic创建个数有上限，通过参数 tmqMaxTopicNum 控制，默认 20 个
 
 TMQ 支持多种订阅类型：
 
@@ -265,26 +266,29 @@ CREATE TOPIC topic_name as subquery
 语法：
 
 ```sql
-CREATE TOPIC topic_name AS STABLE stb_name
+CREATE TOPIC topic_name [with meta] AS STABLE stb_name [where_condition]
 ```
 
 与 `SELECT * from stbName` 订阅的区别是：
 
 - 不会限制用户的表结构变更。
 - 返回的是非结构化的数据：返回数据的结构会随之超级表的表结构变化而变化。
-- 用户对于要处理的每一个数据块都可能有不同的表结构。
+- with meta 参数可选，选择时将返回创建超级表，子表等语句，主要用于taosx做超级表迁移
+- where_condition 参数可选，选择时将用来过滤符合条件的子表，订阅这些子表。where 条件里不能有普通列，只能是tag或tbname，where条件里可以用函数，用来过滤tag，但是不能是聚合函数，因为子表tag值无法做聚合。也可以是常量表达式，比如 2 > 1（订阅全部子表），或者 false（订阅0个子表）
 - 返回数据不包含标签。
 
 ### 数据库订阅
 
 语法：
 
 ```sql
-CREATE TOPIC topic_name AS DATABASE db_name;
+CREATE TOPIC topic_name [with meta] AS DATABASE db_name;
 ```
 
 通过该语句可创建一个包含数据库所有表数据的订阅
 
+- with meta 参数可选，选择时将返回创建数据库里所有超级表，子表的语句，主要用于taosx做数据库迁移
+
 ## 创建消费者 *consumer*
 
 消费者需要通过一系列配置选项创建，基础配置项如下表所示：
@@ -295,7 +299,7 @@ CREATE TOPIC topic_name AS DATABASE db_name;
 |       `td.connect.user`        | string  | 用户名                         |    |
 |       `td.connect.pass`        | string  | 密码                          |  |
 |       `td.connect.port`        | integer | 服务端的端口号                         |  |
-|           `group.id`           | string  | 消费组 ID，同一消费组共享消费进度                        | **必填项**。最大长度：192。                 |
+|           `group.id`           | string  | 消费组 ID，同一消费组共享消费进度                        | <br />**必填项**。最大长度：192。<br />每个topic最多可建立100个 consumer group                 |
 |          `client.id`           | string  | 客户端 ID                                                | 最大长度：192。                             |
 |      `auto.offset.reset`       |  enum   | 消费组订阅的初始位置                                     | <br />`earliest`: default;从头开始订阅; <br/>`latest`: 仅从最新数据开始订阅; <br/>`none`: 没有提交的 offset 无法订阅 |
 |      `enable.auto.commit`      | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit     | 默认值为 true                   |

docs/zh/12-taos-sql/06-select.md

@@ -55,7 +55,7 @@ window_clause: {
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)]
 
 interp_clause:
-    RANGE(ts_val, ts_val) EVERY(every_val) FILL(fill_mod_and_val)
+    RANGE(ts_val [, ts_val]) EVERY(every_val) FILL(fill_mod_and_val)
 
 partition_by_clause:
     PARTITION BY expr [, expr] ...

docs/zh/12-taos-sql/10-function.md

@@ -890,9 +890,10 @@ ignore_null_values: {
 - INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。
 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。
 - INTERP 需要同时与 RANGE，EVERY 和 FILL 关键字一起使用。
-- INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1 <= timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。
+- INTERP 的输出时间范围根据 RANGE(timestamp1, timestamp2)字段来指定，需满足 timestamp1 <= timestamp2。其中 timestamp1 为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2 为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。
 - INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（time_unit 值）进行插值，time_unit 可取值时间单位：1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天)，1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值.
 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](../distinguished/#fill-子句)
+- INTERP 可以在 RANGE 字段中只指定唯一的时间戳对单个时间点进行插值，在这种情况下，EVERY 字段可以省略。例如：SELECT INTERP(col) FROM tb RANGE('2023-01-01 00:00:00') FILL(linear).
 - INTERP 作用于超级表时, 会将该超级表下的所有子表数据按照主键列排序后进行插值计算，也可以搭配 PARTITION BY tbname 使用，将结果强制规约到单个时间线。
 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.2.0版本以后支持)。
 - INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0版本以后支持)。

docs/zh/12-taos-sql/24-show.md

@@ -36,7 +36,7 @@ SHOW CONNECTIONS;
 SHOW CONSUMERS;
 ```
 
-显示当前数据库下所有活跃的消费者的信息。
+显示当前数据库下所有消费者的信息。
 
 ## SHOW CREATE DATABASE
 

include/common/tcommon.h

@@ -198,6 +198,7 @@ typedef struct SDataBlockInfo {
   SBlockID    id;
   int16_t     hasVarCol;
   int16_t     dataLoad;  // denote if the data is loaded or not
+  uint8_t     scanFlag;
 
   // TODO: optimize and remove following
   int64_t     version;    // used for stream, and need serialization

include/common/tdatablock.h

@@ -187,6 +187,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
 int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);
 
 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows);
+int32_t colDataGetRowLength(const SColumnInfoData* pColumnInfoData, int32_t rowIdx);
 void    colDataTrim(SColumnInfoData* pColumnInfoData);
 
 size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
@@ -248,6 +249,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData
                                     tb_uid_t suid);
 
 char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
+int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf);
 
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
   return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock);

include/common/tglobal.h

@@ -83,6 +83,7 @@ extern int64_t tsVndCommitMaxIntervalMs;
 extern int64_t tsMndSdbWriteDelta;
 extern int64_t tsMndLogRetention;
 extern int8_t  tsGrant;
+extern bool    tsMndSkipGrant;
 
 // monitor
 extern bool     tsEnableMonitor;
@@ -119,6 +120,7 @@ extern bool    tsQueryUseNodeAllocator;
 extern bool    tsKeepColumnName;
 extern bool    tsEnableQueryHb;
 extern bool    tsEnableScience;
+extern bool    tsTtlChangeOnWrite;
 extern int32_t tsRedirectPeriod;
 extern int32_t tsRedirectFactor;
 extern int32_t tsRedirectMaxPeriod;
@@ -184,6 +186,7 @@ extern int64_t tsStreamBufferSize;
 extern int64_t tsCheckpointInterval;
 extern bool    tsFilterScalarMode;
 extern int32_t tsMaxStreamBackendCache;
+extern int32_t tsPQSortMemThreshold;
 
 // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)