release-23.2: sql: add server.max_open_transactions_per_gateway cluster setting

docs/generated/settings/settings-for-tenants.txt

@@ -80,6 +80,7 @@ server.identity_map.configuration	string		system-identity to database-username m
 server.log_gc.max_deletions_per_cycle	integer	1000	the maximum number of entries to delete on each purge of log-like system tables	application
 server.log_gc.period	duration	1h0m0s	the period at which log-like system tables are checked for old entries	application
 server.max_connections_per_gateway	integer	-1	the maximum number of SQL connections per gateway allowed at a given time (note: this will only limit future connection attempts and will not affect already established connections). Negative values result in unlimited number of connections. Superusers are not affected by this limit.	application
+server.max_open_transactions_per_gateway	integer	-1	the maximum number of open SQL transactions per gateway allowed at a given time. Negative values result in unlimited number of connections. Superusers are not affected by this limit.	application
 server.oidc_authentication.autologin.enabled	boolean	false	if true, logged-out visitors to the DB Console will be automatically redirected to the OIDC login endpoint	application
 server.oidc_authentication.button_text	string	Log in with your OIDC provider	text to show on button on DB Console login page to login with your OIDC provider (only shown if OIDC is enabled)	application
 server.oidc_authentication.claim_json_key	string		sets JSON key of principal to extract from payload after OIDC authentication completes (usually email or sid)	application

docs/generated/settings/settings.html

@@ -106,6 +106,7 @@
 <tr><td><div id="setting-server-log-gc-max-deletions-per-cycle" class="anchored"><code>server.log_gc.max_deletions_per_cycle</code></div></td><td>integer</td><td><code>1000</code></td><td>the maximum number of entries to delete on each purge of log-like system tables</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-log-gc-period" class="anchored"><code>server.log_gc.period</code></div></td><td>duration</td><td><code>1h0m0s</code></td><td>the period at which log-like system tables are checked for old entries</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-max-connections-per-gateway" class="anchored"><code>server.max_connections_per_gateway</code></div></td><td>integer</td><td><code>-1</code></td><td>the maximum number of SQL connections per gateway allowed at a given time (note: this will only limit future connection attempts and will not affect already established connections). Negative values result in unlimited number of connections. Superusers are not affected by this limit.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
+<tr><td><div id="setting-server-max-open-transactions-per-gateway" class="anchored"><code>server.max_open_transactions_per_gateway</code></div></td><td>integer</td><td><code>-1</code></td><td>the maximum number of open SQL transactions per gateway allowed at a given time. Negative values result in unlimited number of connections. Superusers are not affected by this limit.</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-autologin" class="anchored"><code>server.oidc_authentication.autologin.enabled</code></div></td><td>boolean</td><td><code>false</code></td><td>if true, logged-out visitors to the DB Console will be automatically redirected to the OIDC login endpoint</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-button-text" class="anchored"><code>server.oidc_authentication.button_text</code></div></td><td>string</td><td><code>Log in with your OIDC provider</code></td><td>text to show on button on DB Console login page to login with your OIDC provider (only shown if OIDC is enabled)</td><td>Serverless/Dedicated/Self-Hosted</td></tr>
 <tr><td><div id="setting-server-oidc-authentication-claim-json-key" class="anchored"><code>server.oidc_authentication.claim_json_key</code></div></td><td>string</td><td><code></code></td><td>sets JSON key of principal to extract from payload after OIDC authentication completes (usually email or sid)</td><td>Serverless/Dedicated/Self-Hosted</td></tr>

pkg/sql/conn_executor.go

@@ -117,6 +117,14 @@ var maxNumNonRootConnections = settings.RegisterIntSetting(
 	-1,
 )
 
+var maxOpenTransactions = settings.RegisterIntSetting(
+	settings.ApplicationLevel,
+	"server.max_open_transactions_per_gateway",
+	"the maximum number of open SQL transactions per gateway allowed at a given time. "+
+		"Negative values result in unlimited number of connections. Superusers are not affected by this limit.",
+	-1,
+	settings.WithPublic)
+
 // maxNumNonRootConnectionsReason is used to supplement the error message for connections that denied due to
 // server.cockroach_cloud.max_client_connections_per_gateway.
 // Note(alyshan): This setting is not public. It is intended to be used by Cockroach Cloud when limiting

pkg/sql/conn_executor_exec.go

@@ -547,6 +547,25 @@ func (ex *connExecutor) execStmtInOpenState(
 	p.noticeSender = res
 	ih := &p.instrumentation
 
+	if maxOpen := maxOpenTransactions.Get(&ex.server.cfg.Settings.SV); maxOpen > 0 {
+		// NB: SQLTxnsOpen does not include internal executor transactions.
+		if ex.metrics.EngineMetrics.SQLTxnsOpen.Value() > maxOpen {
+			hasAdmin, err := ex.planner.HasAdminRole(ctx)
+			if err != nil {
+				return makeErrEvent(err)
+			}
+			if !hasAdmin {
+				return makeErrEvent(errors.WithHintf(
+					pgerror.Newf(
+						pgcode.ConfigurationLimitExceeded,
+						"cannot execute operation due to server.max_open_transactions_per_gateway cluster setting",
+					),
+					"the maximum number of open transactions is %d", maxOpen,
+				))
+			}
+		}
+	}
+
 	// Special top-level handling for EXPLAIN ANALYZE.
 	if e, ok := ast.(*tree.ExplainAnalyze); ok {
 		switch e.Mode {

pkg/sql/tests/BUILD.bazel

@@ -47,6 +47,7 @@ go_test(
         "inverted_index_test.go",
         "kv_test.go",
         "main_test.go",
+        "max_open_txns_test.go",
         "monotonic_insert_test.go",
         "random_schema_test.go",
         "read_committed_test.go",

pkg/sql/tests/max_open_txns_test.go

@@ -0,0 +1,96 @@
+// Copyright 2024 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
+	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/stretchr/testify/require"
+)
+
+// Test the server.max_open_transactions_per_gateway cluster setting. Only
+// non-admins are subject to the limit.
+func TestMaxOpenTxns(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	ctx := context.Background()
+	srv, db, _ := serverutils.StartServer(t, base.TestServerArgs{})
+	defer srv.Stopper().Stop(ctx)
+	s := srv.ApplicationLayer()
+
+	rootSQLRunner := sqlutils.MakeSQLRunner(db)
+	rootSQLRunner.Exec(t, "SET CLUSTER SETTING server.max_open_transactions_per_gateway = 4")
+	rootSQLRunner.Exec(t, "CREATE USER testuser")
+
+	testUserDB := s.SQLConn(t, serverutils.User("testuser"))
+
+	tx1 := rootSQLRunner.Begin(t)
+	_, err := testUserDB.Exec("SELECT 1")
+	require.NoError(t, err)
+
+	tx2 := rootSQLRunner.Begin(t)
+	_, err = testUserDB.Exec("SELECT 1")
+	require.NoError(t, err)
+
+	tx3 := rootSQLRunner.Begin(t)
+	_, err = testUserDB.Exec("SELECT 1")
+	require.NoError(t, err)
+
+	// After four transactions have been opened, testuser cannot run a query.
+	tx4 := rootSQLRunner.Begin(t)
+	_, err = testUserDB.Exec("SELECT 1")
+	require.ErrorContains(t, err, "cannot execute operation due to server.max_open_transactions_per_gateway cluster setting")
+
+	// testuser also cannot run anything in an explicit transaction. Starting the
+	// transaction is allowed though.
+	testuserTx, err := testUserDB.Begin()
+	require.NoError(t, err)
+	_, err = testuserTx.Exec("SELECT 1")
+	require.ErrorContains(t, err, "cannot execute operation due to server.max_open_transactions_per_gateway cluster setting")
+	require.NoError(t, testuserTx.Rollback())
+
+	// Increase the limit, allowing testuser to run queries in a transaction.
+	rootSQLRunner.Exec(t, "SET CLUSTER SETTING server.max_open_transactions_per_gateway = 5")
+	testuserTx, err = testUserDB.Begin()
+	require.NoError(t, err)
+	_, err = testuserTx.Exec("SELECT 1")
+	require.NoError(t, err)
+
+	// Lower the limit again, and verify that the setting applies to transactions
+	// that were already open at the time the setting changed.
+	rootSQLRunner.Exec(t, "SET CLUSTER SETTING server.max_open_transactions_per_gateway = 4")
+	_, err = testuserTx.Exec("SELECT 2")
+	require.ErrorContains(t, err, "cannot execute operation due to server.max_open_transactions_per_gateway cluster setting")
+	require.NoError(t, testuserTx.Rollback())
+
+	// Making testuser admin should allow it to run queries.
+	rootSQLRunner.Exec(t, "GRANT admin TO testuser")
+	_, err = testUserDB.Exec("SELECT 1")
+	require.NoError(t, err)
+
+	testuserTx, err = testUserDB.Begin()
+	require.NoError(t, err)
+	_, err = testuserTx.Exec("SELECT 1")
+	require.NoError(t, err)
+	require.NoError(t, testuserTx.Rollback())
+
+	require.NoError(t, tx1.Commit())
+	require.NoError(t, tx2.Commit())
+	require.NoError(t, tx3.Commit())
+	require.NoError(t, tx4.Commit())
+}