clickhouse-test: improve left queries after the test hardening

docs/en/operations/system-tables/processes.md

@@ -13,6 +13,8 @@ Columns:
 -   `memory_usage` (UInt64) – Amount of RAM the request uses. It might not include some types of dedicated memory. See the [max_memory_usage](../../operations/settings/query-complexity.md#settings_max_memory_usage) setting.
 -   `query` (String) – The query text. For `INSERT`, it does not include the data to insert.
 -   `query_id` (String) – Query ID, if defined.
+-   `is_cancelled` (Int8) – Was query cancelled.
+-   `is_all_data_sent` (Int8) – Was all data sent to the client (in other words query had been finished on the server).
 
 ```sql
 :) SELECT * FROM system.processes LIMIT 10 FORMAT Vertical;
@@ -43,6 +45,7 @@ http_user_agent:
 quota_key:
 elapsed:              0.000582537
 is_cancelled:         0
+is_all_data_sent:     0
 read_rows:            0
 read_bytes:           0
 total_rows_approx:    0

src/Interpreters/ProcessList.cpp

@@ -455,6 +455,7 @@ QueryStatusInfo QueryStatus::getInfo(bool get_thread_list, bool get_profile_even
     res.client_info       = client_info;
     res.elapsed_seconds   = watch.elapsedSeconds();
     res.is_cancelled      = is_killed.load(std::memory_order_relaxed);
+    res.is_all_data_sent  = is_all_data_sent.load(std::memory_order_relaxed);
     res.read_rows         = progress_in.read_rows;
     res.read_bytes        = progress_in.read_bytes;
     res.total_rows        = progress_in.total_rows_to_read;

src/Interpreters/ProcessList.h

@@ -61,6 +61,7 @@ struct QueryStatusInfo
     Int64 peak_memory_usage;
     ClientInfo client_info;
     bool is_cancelled;
+    bool is_all_data_sent;
 
     /// Optional fields, filled by query
     std::vector<UInt64> thread_ids;
@@ -101,6 +102,9 @@ class QueryStatus : public WithContext
 
     std::atomic<bool> is_killed { false };
 
+    /// All data to the client already had been sent. Including EndOfStream.
+    std::atomic<bool> is_all_data_sent { false };
+
     void setUserProcessList(ProcessListForUser * user_process_list_);
     /// Be careful using it. For example, queries field of ProcessListForUser could be modified concurrently.
     const ProcessListForUser * getUserProcessList() const { return user_process_list; }
@@ -194,6 +198,9 @@ class QueryStatus : public WithContext
 
     bool isKilled() const { return is_killed; }
 
+    bool isAllDataSent() const { return is_all_data_sent; }
+    void setAllDataSent() { is_all_data_sent = true; }
+
     /// Adds a pipeline to the QueryStatus
     void addPipelineExecutor(PipelineExecutor * e);
 

src/Server/TCPHandler.cpp

@@ -31,6 +31,7 @@
 #include <Interpreters/InternalTextLogsQueue.h>
 #include <Interpreters/OpenTelemetrySpanLog.h>
 #include <Interpreters/Session.h>
+#include <Interpreters/ProcessList.h>
 #include <Server/TCPServer.h>
 #include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/MergeTree/MergeTreeDataPartUUID.h>
@@ -1710,6 +1711,12 @@ void TCPHandler::sendException(const Exception & e, bool with_stack_trace)
 void TCPHandler::sendEndOfStream()
 {
     state.sent_all_data = true;
+    /// The following queries does not have process_list_entry:
+    /// - internal
+    /// - SHOW PROCESSLIST
+    if (state.io.process_list_entry)
+        (*state.io.process_list_entry)->setAllDataSent();
+
     writeVarUInt(Protocol::Server::EndOfStream, *out);
     out->next();
 }

src/Storages/System/StorageSystemProcesses.cpp

@@ -52,6 +52,7 @@ NamesAndTypesList StorageSystemProcesses::getNamesAndTypes()
 
         {"elapsed", std::make_shared<DataTypeFloat64>()},
         {"is_cancelled", std::make_shared<DataTypeUInt8>()},
+        {"is_all_data_sent", std::make_shared<DataTypeUInt8>()},
         {"read_rows", std::make_shared<DataTypeUInt64>()},
         {"read_bytes", std::make_shared<DataTypeUInt64>()},
         {"total_rows_approx", std::make_shared<DataTypeUInt64>()},
@@ -120,6 +121,7 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, ContextPtr c
 
         res_columns[i++]->insert(process.elapsed_seconds);
         res_columns[i++]->insert(process.is_cancelled);
+        res_columns[i++]->insert(process.is_all_data_sent);
         res_columns[i++]->insert(process.read_rows);
         res_columns[i++]->insert(process.read_bytes);
         res_columns[i++]->insert(process.total_rows);

tests/clickhouse-test

@@ -225,6 +225,7 @@ def get_processlist_after_test(args):
         FROM clusterAllReplicas('test_cluster_database_replicated', system.processes)
         WHERE
             query NOT LIKE '%system.processes%' AND
+            NOT is_all_data_sent AND
             Settings['log_comment'] = '{log_comment}' AND
             current_database = '{database}'
         """)
@@ -234,6 +235,7 @@ def get_processlist_after_test(args):
         FROM system.processes
         WHERE
             query NOT LIKE '%system.processes%' AND
+            NOT is_all_data_sent AND
             Settings['log_comment'] = '{log_comment}' AND
             current_database = '{database}'
         """)

tests/queries/0_stateless/02117_show_create_table_system.sql

@@ -16,69 +16,69 @@ tables_with_database_column=(
 
  */
 use system;
-show create table aggregate_function_combinators;
-show create table asynchronous_inserts;
-show create table asynchronous_metrics;
-show create table build_options;
-show create table clusters;
-show create table collations;
-show create table columns;
-show create table contributors;
-show create table current_roles;
-show create table data_skipping_indices;
-show create table data_type_families;
-show create table databases;
-show create table detached_parts;
-show create table dictionaries;
-show create table disks;
-show create table distributed_ddl_queue;
-show create table distribution_queue;
-show create table enabled_roles;
-show create table errors;
-show create table events;
-show create table formats;
-show create table functions;
-show create table grants;
-show create table graphite_retentions;
-show create table licenses;
-show create table macros;
-show create table merge_tree_settings;
-show create table merges;
-show create table metrics;
-show create table models;
-show create table mutations;
-show create table numbers;
-show create table numbers_mt;
-show create table one;
-show create table part_moves_between_shards;
-show create table parts;
-show create table parts_columns;
-show create table privileges;
-show create table processes;
-show create table projection_parts;
-show create table projection_parts_columns;
-show create table quota_limits;
-show create table quota_usage;
-show create table quotas;
-show create table quotas_usage;
-show create table replicas;
-show create table replicated_fetches;
-show create table replicated_merge_tree_settings;
-show create table replication_queue;
-show create table role_grants;
-show create table roles;
-show create table row_policies;
-show create table settings;
-show create table settings_profile_elements;
-show create table settings_profiles;
-show create table stack_trace;
-show create table storage_policies;
-show create table table_engines;
-show create table table_functions;
-show create table tables;
-show create table time_zones;
-show create table user_directories;
-show create table users;
-show create table warnings;
-show create table zeros;
-show create table zeros_mt;
+show create table aggregate_function_combinators format TSVRaw;
+show create table asynchronous_inserts format TSVRaw;
+show create table asynchronous_metrics format TSVRaw;
+show create table build_options format TSVRaw;
+show create table clusters format TSVRaw;
+show create table collations format TSVRaw;
+show create table columns format TSVRaw;
+show create table contributors format TSVRaw;
+show create table current_roles format TSVRaw;
+show create table data_skipping_indices format TSVRaw;
+show create table data_type_families format TSVRaw;
+show create table databases format TSVRaw;
+show create table detached_parts format TSVRaw;
+show create table dictionaries format TSVRaw;
+show create table disks format TSVRaw;
+show create table distributed_ddl_queue format TSVRaw;
+show create table distribution_queue format TSVRaw;
+show create table enabled_roles format TSVRaw;
+show create table errors format TSVRaw;
+show create table events format TSVRaw;
+show create table formats format TSVRaw;
+show create table functions format TSVRaw;
+show create table grants format TSVRaw;
+show create table graphite_retentions format TSVRaw;
+show create table licenses format TSVRaw;
+show create table macros format TSVRaw;
+show create table merge_tree_settings format TSVRaw;
+show create table merges format TSVRaw;
+show create table metrics format TSVRaw;
+show create table models format TSVRaw;
+show create table mutations format TSVRaw;
+show create table numbers format TSVRaw;
+show create table numbers_mt format TSVRaw;
+show create table one format TSVRaw;
+show create table part_moves_between_shards format TSVRaw;
+show create table parts format TSVRaw;
+show create table parts_columns format TSVRaw;
+show create table privileges format TSVRaw;
+show create table processes format TSVRaw;
+show create table projection_parts format TSVRaw;
+show create table projection_parts_columns format TSVRaw;
+show create table quota_limits format TSVRaw;
+show create table quota_usage format TSVRaw;
+show create table quotas format TSVRaw;
+show create table quotas_usage format TSVRaw;
+show create table replicas format TSVRaw;
+show create table replicated_fetches format TSVRaw;
+show create table replicated_merge_tree_settings format TSVRaw;
+show create table replication_queue format TSVRaw;
+show create table role_grants format TSVRaw;
+show create table roles format TSVRaw;
+show create table row_policies format TSVRaw;
+show create table settings format TSVRaw;
+show create table settings_profile_elements format TSVRaw;
+show create table settings_profiles format TSVRaw;
+show create table stack_trace format TSVRaw;
+show create table storage_policies format TSVRaw;
+show create table table_engines format TSVRaw;
+show create table table_functions format TSVRaw;
+show create table tables format TSVRaw;
+show create table time_zones format TSVRaw;
+show create table user_directories format TSVRaw;
+show create table users format TSVRaw;
+show create table warnings format TSVRaw;
+show create table zeros format TSVRaw;
+show create table zeros_mt format TSVRaw;