Merge #118456

118456: sql: introduce `crdb_internal.execute_internally` builtin r=yuzefovich a=yuzefovich

This commit introduces a new undocumented generator builtin `crdb_internal.execute_internally` which executes the provided string argument (which can be almost any single query) via the internal executor and then converts each resulting row into a string that is returned. This seems like a useful thing to have in general, but in particular when trying to understand the plans used by the internal queries.

Additionally, this commit provides several overloads of this new builtin:
- an optional boolean argument 'session_bound' indicates whether the query execution should happen via the IE that is bound to the session invoking it or not (if not, then it'll be "jobs-like" execution that creates a fresh session data - the only exception is that the user of this fresh session data is overridden to the session's user)
- an optional string argument 'overrides' which specifies comma-separated list of session overrides, e.g. `'Database=foo,OptimizerUseHistograms=true'`.  These overrides apply as the very last step when initializing the session data for the internal executor and could allow us reproduce queries from different contexts. Note that these overrides are applied on a best-effort basis and mostly don't support custom types in the protobuf messages.
- an optional boolean argument 'use_session_txn' which specifies that the session's txn must be used when 'session_bound' is false. Usage of this parameter requires that 'overrides' parameter is also specified (in order to distinguish this boolean from the other one). Note that an error is returned whenever using the session-bound executor because there we always use the session's txn, so this parameter would be redundant and probably confusing (like can it be run as a nested txn?).

The builtin is undocumented, so there is no release note. I envision that only CRDB people should use this builtin. The only requirement imposed on it is that the user is the admin. The query gets executed under the session's user, so this should be ok from the security's point of view. Also some statements that modify txn state are prohibited.

Fixes: #118426.

Release note: None

Co-authored-by: Yahor Yuzefovich <yahor@cockroachlabs.com>

pkg/sql/exec_util.go

@@ -91,6 +91,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging"
 	"github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/asof"
+	"github.com/cockroachdb/cockroach/pkg/sql/sem/builtins"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/catconstants"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/eval"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
@@ -126,6 +127,21 @@ import (
 	"github.com/cockroachdb/redact"
 )
 
+func init() {
+	builtins.ExecuteQueryViaJobExecContext = func(
+		evalCtx *eval.Context,
+		ctx context.Context,
+		opName string,
+		txn *kv.Txn,
+		override sessiondata.InternalExecutorOverride,
+		stmt string,
+		qargs ...interface{},
+	) (eval.InternalRows, error) {
+		ie := evalCtx.JobExecContext.(JobExecContext).ExecCfg().InternalDB.Executor()
+		return ie.QueryIteratorEx(ctx, opName, txn, override, stmt, qargs...)
+	}
+}
+
 // ClusterOrganization is the organization name.
 var ClusterOrganization = settings.RegisterStringSetting(
 	settings.SystemVisible,

pkg/sql/internal.go

@@ -862,6 +862,17 @@ func applyOverrides(o sessiondata.InternalExecutorOverride, sd *sessiondata.Sess
 	if o.OptimizerUseHistograms {
 		sd.OptimizerUseHistograms = true
 	}
+
+	if o.MultiOverride != "" {
+		overrides := strings.Split(o.MultiOverride, ",")
+		for _, override := range overrides {
+			parts := strings.Split(override, "=")
+			if len(parts) == 2 {
+				sd.Update(parts[0], parts[1])
+			}
+		}
+	}
+	// Add any new overrides above the MultiOverride.
 }
 
 func (ie *InternalExecutor) maybeNodeSessionDataOverride(

pkg/sql/opt/exec/execbuilder/testdata/execute_internally_builtin

@@ -0,0 +1,170 @@
+# LogicTest: local
+
+# Note that even though this file contains some queries that actually get
+# executed, we choose to keep it in the execbuilder tests because it also has
+# EXPLAIN output.
+
+statement ok
+CREATE TABLE t (k PRIMARY KEY) AS VALUES (1), (3), (5);
+
+# When the internal executor is not session bound, it doesn't have Database
+# session variable set, so we must specify the fully-qualified table name.
+query T rowsort
+SELECT crdb_internal.execute_internally('SELECT k FROM test.public.t;');
+----
+1
+3
+5
+
+# When the internal executor is session bound that is not needed.
+query T rowsort
+SELECT crdb_internal.execute_internally('SELECT k FROM t;', true);
+----
+1
+3
+5
+
+# Set the database via the override.
+query T rowsort
+SELECT crdb_internal.execute_internally('EXPLAIN SELECT k FROM t;', 'Database=test');
+----
+distribution: local
+vectorized: true
+·
+• scan
+  missing stats
+  table: t@t_pkey
+  spans: FULL SCAN
+
+# Same query as above, but with vectorized engine disabled.
+query T rowsort
+SELECT crdb_internal.execute_internally('EXPLAIN SELECT k FROM t;', 'Database=test,VectorizeMode=off');
+----
+distribution: local
+vectorized: false
+·
+• scan
+  missing stats
+  table: t@t_pkey
+  spans: FULL SCAN
+
+# optimizer_use_histograms is the only variable that differs from the default
+# right now (except when the internal executor is session-bound, #102954).
+query T
+SELECT crdb_internal.execute_internally('SHOW optimizer_use_histograms;', false);
+----
+off
+
+query T
+SELECT crdb_internal.execute_internally('SHOW optimizer_use_histograms;', true);
+----
+on
+
+# Ensure that we can override it.
+query T
+SELECT crdb_internal.execute_internally('SHOW optimizer_use_histograms;', false, 'OptimizerUseHistograms=true');
+----
+on
+
+query T
+SELECT crdb_internal.execute_internally('SHOW optimizer_use_histograms;', true, 'OptimizerUseHistograms=false');
+----
+off
+
+# Some sanity checks around error handling.
+statement error unknown signature
+SELECT crdb_internal.execute_internally(1);
+
+statement error unknown signature
+SELECT crdb_internal.execute_internally('SELECT 1;', false, true);
+
+statement error when session bound internal executor is used, it always uses the session txn - omit the last argument
+SELECT crdb_internal.execute_internally('SELECT 1;', true, '', true);
+
+query error internally-executed-query-builtin: relation "foo" does not exist
+SELECT crdb_internal.execute_internally('SELECT col FROM foo;');
+
+statement error syntax error
+SELECT crdb_internal.execute_internally('SELECT col FROM;');
+
+statement error only one statement is supported, 2 were given
+SELECT crdb_internal.execute_internally('SELECT col FROM foo; SELECT 1;');
+
+statement error only one statement is supported, 0 were given
+SELECT crdb_internal.execute_internally('');
+
+statement error this statement is disallowed
+SELECT crdb_internal.execute_internally('BEGIN;');
+
+statement error this statement is disallowed
+SELECT crdb_internal.execute_internally('COMMIT;', 'Database=test', true);
+
+# Check transactionality.
+statement ok
+BEGIN;
+
+statement ok
+CREATE TABLE t2 (k INT PRIMARY KEY);
+
+# When running in a separate txn, we cannot see the newly created table.
+query error internally-executed-query-builtin: relation "t2" does not exist
+SELECT crdb_internal.execute_internally('SELECT k FROM t2;', 'Database=test', false);
+
+statement ok
+ROLLBACK;
+
+statement ok
+BEGIN;
+
+statement ok
+CREATE TABLE t2 (k INT PRIMARY KEY);
+
+# But using the session's txn with non-session-bound executor should work.
+statement ok
+SELECT crdb_internal.execute_internally('SELECT k FROM t2;', 'Database=test', true);
+
+statement ok
+SELECT crdb_internal.execute_internally('INSERT INTO t2 VALUES (1);', 'Database=test', true);
+
+query T
+SELECT crdb_internal.execute_internally('SELECT * FROM t2;', 'Database=test', true);
+----
+1
+
+statement ok
+COMMIT;
+
+user testuser
+
+statement error crdb_internal.execute_internally\(\) requires admin privilege
+SELECT crdb_internal.execute_internally('SELECT session_user;');
+
+user root
+
+statement ok
+GRANT admin TO testuser
+
+user testuser
+
+# Ensure that changing the user via the override isn't possible.
+query T
+SELECT crdb_internal.execute_internally('SELECT session_user;');
+----
+testuser
+
+query T
+SELECT crdb_internal.execute_internally('SELECT session_user;', 'User=root');
+----
+testuser
+
+user root
+
+query T
+SELECT crdb_internal.execute_internally('SELECT session_user;');
+----
+root
+
+query T
+SELECT crdb_internal.execute_internally('SELECT session_user;', 'User=testuser');
+----
+root

pkg/sql/sem/builtins/fixed_oids.go

@@ -2564,6 +2564,12 @@ var builtinOidsArray = []string{
 	2596: `array_agg(arg1: refcursor[]) -> refcursor[][]`,
 	2597: `array_agg(arg1: tuple[]) -> tuple[][]`,
 	2598: `setseed(seed: float) -> void`,
+	2599: `crdb_internal.execute_internally(query: string) -> string`,
+	2600: `crdb_internal.execute_internally(query: string, session_bound: bool) -> string`,
+	2601: `crdb_internal.execute_internally(query: string, overrides: string) -> string`,
+	2602: `crdb_internal.execute_internally(query: string, session_bound: bool, overrides: string) -> string`,
+	2603: `crdb_internal.execute_internally(query: string, overrides: string, use_session_txn: bool) -> string`,
+	2604: `crdb_internal.execute_internally(query: string, session_bound: bool, overrides: string, use_session_txn: bool) -> string`,
 }
 
 var builtinOidsBySignature map[string]oid.Oid