Add new profile events for queries with subqueries

Differs from SelectQuery/InsertQuery that it could be > 1 (in case of
MATERIALIZED VIEW attached to the table), since each local SELECT/INSERT
query is accounted, not only initial.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>

src/Common/ProfileEvents.cpp

@@ -8,6 +8,9 @@
     M(Query, "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.") \
     M(SelectQuery, "Same as Query, but only for SELECT queries.") \
     M(InsertQuery, "Same as Query, but only for INSERT queries.") \
+    M(QueriesWithSubqueries, "Count queries with all subqueries") \
+    M(SelectQueriesWithSubqueries, "Count SELECT queries with all subqueries") \
+    M(InsertQueriesWithSubqueries, "Count INSERT queries with all subqueries") \
     M(AsyncInsertQuery, "Same as InsertQuery, but only for asynchronous INSERT queries.") \
     M(AsyncInsertBytes, "Data size in bytes of asynchronous INSERT queries.") \
     M(AsyncInsertRows, "Number of rows inserted by asynchronous INSERT queries.") \

src/Interpreters/InterpreterFactory.cpp

@@ -114,6 +114,7 @@
 namespace ProfileEvents
 {
     extern const Event Query;
+    extern const Event QueriesWithSubqueries;
     extern const Event SelectQuery;
     extern const Event InsertQuery;
 }
@@ -131,6 +132,15 @@ std::unique_ptr<IInterpreter> InterpreterFactory::get(ASTPtr & query, ContextMut
 {
     ProfileEvents::increment(ProfileEvents::Query);
 
+    /// SELECT and INSERT query will handle QueriesWithSubqueries on their own.
+    if (!(query->as<ASTSelectQuery>() ||
+        query->as<ASTSelectWithUnionQuery>() ||
+        query->as<ASTSelectIntersectExceptQuery>() ||
+        query->as<ASTInsertQuery>()))
+    {
+        ProfileEvents::increment(ProfileEvents::QueriesWithSubqueries);
+    }
+
     if (query->as<ASTSelectQuery>())
     {
         if (context->getSettingsRef().allow_experimental_analyzer)

src/Interpreters/InterpreterInsertQuery.cpp

@@ -34,8 +34,15 @@
 #include <TableFunctions/TableFunctionFactory.h>
 #include <Common/ThreadStatus.h>
 #include <Common/checkStackSize.h>
+#include <Common/ProfileEvents.h>
 
 
+namespace ProfileEvents
+{
+    extern const Event InsertQueriesWithSubqueries;
+    extern const Event QueriesWithSubqueries;
+}
+
 namespace DB
 {
 
@@ -234,6 +241,9 @@ Chain InterpreterInsertQuery::buildChain(
     ThreadStatusesHolderPtr thread_status_holder,
     std::atomic_uint64_t * elapsed_counter_ms)
 {
+    ProfileEvents::increment(ProfileEvents::InsertQueriesWithSubqueries);
+    ProfileEvents::increment(ProfileEvents::QueriesWithSubqueries);
+
     ThreadGroupPtr running_group;
     if (current_thread)
         running_group = current_thread->getThreadGroup();

src/Interpreters/InterpreterSelectQuery.cpp

@@ -13,6 +13,7 @@
 #include <Parsers/ASTTablesInSelectQuery.h>
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/parseQuery.h>
+#include <Parsers/FunctionParameterValuesVisitor.h>
 
 #include <Access/Common/AccessFlags.h>
 #include <Access/ContextAccess.h>
@@ -93,11 +94,17 @@
 #include <Common/FieldVisitorsAccurateComparison.h>
 #include <Common/checkStackSize.h>
 #include <Common/scope_guard_safe.h>
-#include <Parsers/FunctionParameterValuesVisitor.h>
 #include <Common/typeid_cast.h>
+#include <Common/ProfileEvents.h>
 
 #include "config_version.h"
 
+namespace ProfileEvents
+{
+    extern const Event SelectQueriesWithSubqueries;
+    extern const Event QueriesWithSubqueries;
+}
+
 namespace DB
 {
 
@@ -1329,6 +1336,9 @@ static bool hasWithTotalsInAnySubqueryInFromClause(const ASTSelectQuery & query)
 
 void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<Pipe> prepared_pipe)
 {
+    ProfileEvents::increment(ProfileEvents::SelectQueriesWithSubqueries);
+    ProfileEvents::increment(ProfileEvents::QueriesWithSubqueries);
+
     /** Streams of data. When the query is executed in parallel, we have several data streams.
      *  If there is no GROUP BY, then perform all operations before ORDER BY and LIMIT in parallel, then
      *  if there is an ORDER BY, then glue the streams using ResizeProcessor, and then MergeSorting transforms,

src/Planner/Planner.cpp

@@ -2,6 +2,7 @@
 
 #include <Core/ProtocolDefines.h>
 #include <Common/logger_useful.h>
+#include <Common/ProfileEvents.h>
 
 #include <DataTypes/DataTypeString.h>
 
@@ -73,6 +74,12 @@
 #include <Planner/CollectColumnIdentifiers.h>
 #include <Planner/PlannerQueryProcessingInfo.h>
 
+namespace ProfileEvents
+{
+    extern const Event SelectQueriesWithSubqueries;
+    extern const Event QueriesWithSubqueries;
+}
+
 namespace DB
 {
 
@@ -1155,6 +1162,9 @@ void Planner::buildPlanForUnionNode()
 
 void Planner::buildPlanForQueryNode()
 {
+    ProfileEvents::increment(ProfileEvents::SelectQueriesWithSubqueries);
+    ProfileEvents::increment(ProfileEvents::QueriesWithSubqueries);
+
     auto & query_node = query_tree->as<QueryNode &>();
     const auto & query_context = planner_context->getQueryContext();
 

tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.reference

@@ -0,0 +1,24 @@
+view	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1		1	0	1	2	3
+subquery	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1		0	1	0	2	2
+CSE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1		0	1	0	2	2
+CSE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1		0	1	0	2	2
+CTE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1		0	1	0	2	2
+CTE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1		0	1	0	4	4
+view	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1	1	1	0	1	3	4
+subquery	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1	1	0	1	0	2	2
+CSE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1	1	0	1	0	2	2
+CSE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1	1	0	1	0	2	2
+CTE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1	1	0	1	0	2	2
+CTE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+1	1	0	1	0	4	4

tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -n -q "
+    DROP TABLE IF EXISTS mv;
+    DROP TABLE IF EXISTS output;
+    DROP TABLE IF EXISTS input;
+
+    CREATE TABLE input (key Int) Engine=Null;
+    CREATE TABLE output AS input Engine=Null;
+    CREATE MATERIALIZED VIEW mv TO output AS SELECT * FROM input;
+"
+
+for allow_experimental_analyzer in 0 1; do
+    query_id="$(random_str 10)"
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "INSERT INTO input SELECT * FROM numbers(1)"
+    $CLICKHOUSE_CLIENT -mn -q "
+        SYSTEM FLUSH LOGS;
+        SELECT
+            1 view,
+            Settings['allow_experimental_analyzer'] allow_experimental_analyzer,
+            ProfileEvents['InsertQuery'] InsertQuery,
+            ProfileEvents['SelectQuery'] SelectQuery,
+            ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
+            -- FIXME: for analyzer it will have one more for sample block
+            ProfileEvents['SelectQueriesWithSubqueries'] SelectQueriesWithSubqueries,
+            ProfileEvents['QueriesWithSubqueries'] QueriesWithSubqueries
+        FROM system.query_log
+        WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query_id = '$query_id'
+        FORMAT TSVWithNames;
+    "
+
+    query_id="$(random_str 10)"
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "SELECT * FROM system.one WHERE dummy IN (SELECT * FROM system.one) FORMAT Null"
+    $CLICKHOUSE_CLIENT -mn -q "
+        SYSTEM FLUSH LOGS;
+        SELECT
+            1 subquery,
+            Settings['allow_experimental_analyzer'] allow_experimental_analyzer,
+            ProfileEvents['InsertQuery'] InsertQuery,
+            ProfileEvents['SelectQuery'] SelectQuery,
+            ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
+            ProfileEvents['SelectQueriesWithSubqueries'] SelectQueriesWithSubqueries,
+            ProfileEvents['QueriesWithSubqueries'] QueriesWithSubqueries
+        FROM system.query_log
+        WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query_id = '$query_id'
+        FORMAT TSVWithNames;
+    "
+
+    query_id="$(random_str 10)"
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH (SELECT * FROM system.one) AS x SELECT x FORMAT Null"
+    $CLICKHOUSE_CLIENT -mn -q "
+        SYSTEM FLUSH LOGS;
+        SELECT
+            1 CSE,
+            Settings['allow_experimental_analyzer'] allow_experimental_analyzer,
+            ProfileEvents['InsertQuery'] InsertQuery,
+            ProfileEvents['SelectQuery'] SelectQuery,
+            ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
+            ProfileEvents['SelectQueriesWithSubqueries'] SelectQueriesWithSubqueries,
+            ProfileEvents['QueriesWithSubqueries'] QueriesWithSubqueries
+        FROM system.query_log
+        WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query_id = '$query_id'
+        FORMAT TSVWithNames;
+    "
+
+    query_id="$(random_str 10)"
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH (SELECT * FROM system.one) AS x SELECT x, x FORMAT Null"
+    $CLICKHOUSE_CLIENT -mn -q "
+        SYSTEM FLUSH LOGS;
+        SELECT
+            1 CSE_Multi,
+            Settings['allow_experimental_analyzer'] allow_experimental_analyzer,
+            ProfileEvents['InsertQuery'] InsertQuery,
+            ProfileEvents['SelectQuery'] SelectQuery,
+            ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
+            ProfileEvents['SelectQueriesWithSubqueries'] SelectQueriesWithSubqueries,
+            ProfileEvents['QueriesWithSubqueries'] QueriesWithSubqueries
+        FROM system.query_log
+        WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query_id = '$query_id'
+        FORMAT TSVWithNames;
+    "
+
+    query_id="$(random_str 10)"
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH x AS (SELECT * FROM system.one) SELECT * FROM x FORMAT Null"
+    $CLICKHOUSE_CLIENT -mn -q "
+        SYSTEM FLUSH LOGS;
+        SELECT
+            1 CTE,
+            Settings['allow_experimental_analyzer'] allow_experimental_analyzer,
+            ProfileEvents['InsertQuery'] InsertQuery,
+            ProfileEvents['SelectQuery'] SelectQuery,
+            ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
+            ProfileEvents['SelectQueriesWithSubqueries'] SelectQueriesWithSubqueries,
+            ProfileEvents['QueriesWithSubqueries'] QueriesWithSubqueries
+        FROM system.query_log
+        WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query_id = '$query_id'
+        FORMAT TSVWithNames;
+    "
+
+    query_id="$(random_str 10)"
+    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH x AS (SELECT * FROM system.one) SELECT * FROM x UNION ALL SELECT * FROM x FORMAT Null"
+    $CLICKHOUSE_CLIENT -mn -q "
+        SYSTEM FLUSH LOGS;
+        SELECT
+            1 CTE_Multi,
+            Settings['allow_experimental_analyzer'] allow_experimental_analyzer,
+            ProfileEvents['InsertQuery'] InsertQuery,
+            ProfileEvents['SelectQuery'] SelectQuery,
+            ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
+            ProfileEvents['SelectQueriesWithSubqueries'] SelectQueriesWithSubqueries,
+            ProfileEvents['QueriesWithSubqueries'] QueriesWithSubqueries
+        FROM system.query_log
+        WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query_id = '$query_id'
+        FORMAT TSVWithNames;
+    "
+done