sql: make writes to system.eventlog conditional

This patch is meant to help recovering partial availability in
clusters where the `system.eventlog` table / range are unsavailable.

Prior to this patch, when any SQL action was causing a notable event,
that event would be written transactionally (in the same transaction)
to the table `system.eventlog`. If that table happened to be
unavailable, the action would not complete. This was true of even
basic operations like changing a cluster setting, changing privileges
on unrelated tables, etc.

This patch changes that by introducing a new cluster setting
`server.eventlog.enabled` to make these writes conditional.

Release note (general change): The new cluster setting
`server.eventlog.enabled` controls whether notable events are also
written to the table `system.eventlog`. Its default value is
`true`. Changing this cluster setting can help recovering partial
cluster availability when the `system.eventlog` table becomes
unavailable. Note that even when `false`, notable events are still
propagated to the logging system, where they can be e.g. redirected to
files.

docs/generated/settings/settings.html

@@ -43,7 +43,8 @@
 <tr><td><code>server.clock.forward_jump_check_enabled</code></td><td>boolean</td><td><code>false</code></td><td>if enabled, forward clock jumps > max_offset/2 will cause a panic</td></tr>
 <tr><td><code>server.clock.persist_upper_bound_interval</code></td><td>duration</td><td><code>0s</code></td><td>the interval between persisting the wall time upper bound of the clock. The clock does not generate a wall time greater than the persisted timestamp and will panic if it sees a wall time greater than this value. When cockroach starts, it waits for the wall time to catch-up till this persisted timestamp. This guarantees monotonic wall time across server restarts. Not setting this or setting a value of 0 disables this feature.</td></tr>
 <tr><td><code>server.consistency_check.max_rate</code></td><td>byte size</td><td><code>8.0 MiB</code></td><td>the rate limit (bytes/sec) to use for consistency checks; used in conjunction with server.consistency_check.interval to control the frequency of consistency checks. Note that setting this too high can negatively impact performance.</td></tr>
-<tr><td><code>server.eventlog.ttl</code></td><td>duration</td><td><code>2160h0m0s</code></td><td>if nonzero, event log entries older than this duration are deleted every 10m0s. Should not be lowered below 24 hours.</td></tr>
+<tr><td><code>server.eventlog.enabled</code></td><td>boolean</td><td><code>true</code></td><td>if set, logged notable events are also stored in the table system.eventlog</td></tr>
+<tr><td><code>server.eventlog.ttl</code></td><td>duration</td><td><code>2160h0m0s</code></td><td>if nonzero, entries in system.eventlog older than this duration are deleted every 10m0s. Should not be lowered below 24 hours.</td></tr>
 <tr><td><code>server.host_based_authentication.configuration</code></td><td>string</td><td><code></code></td><td>host-based authentication configuration to use during connection authentication</td></tr>
 <tr><td><code>server.oidc_authentication.autologin</code></td><td>boolean</td><td><code>false</code></td><td>if true, logged-out visitors to the DB Console will be automatically redirected to the OIDC login endpoint (this feature is experimental)</td></tr>
 <tr><td><code>server.oidc_authentication.button_text</code></td><td>string</td><td><code>Login with your OIDC provider</code></td><td>text to show on button on DB Console login page to login with your OIDC provider (only shown if OIDC is enabled) (this feature is experimental)</td></tr>

pkg/server/server_systemlog_gc.go

@@ -50,7 +50,7 @@ var (
 	eventLogTTL = settings.RegisterPublicDurationSetting(
 		"server.eventlog.ttl",
 		fmt.Sprintf(
-			"if nonzero, event log entries older than this duration are deleted every %s. "+
+			"if nonzero, entries in system.eventlog older than this duration are deleted every %s. "+
 				"Should not be lowered below 24 hours.",
 			systemLogGCPeriod,
 		),

pkg/sql/event_log.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/kv"
 	"github.com/cockroachdb/cockroach/pkg/security"
+	"github.com/cockroachdb/cockroach/pkg/settings"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
@@ -105,6 +106,12 @@ func logEventInternalForSQLStatements(
 		event)
 }
 
+var eventLogEnabled = settings.RegisterPublicBoolSetting(
+	"server.eventlog.enabled",
+	"if set, logged notable events are also stored in the table system.eventlog",
+	true,
+)
+
 // InsertEventRecord inserts a single event into the event log as part
 // of the provided transaction, using the provided internal executor.
 //
@@ -142,6 +149,11 @@ func InsertEventRecord(
 		log.StructuredEvent(ctx, info)
 	})
 
+	// If writes to the event log table are disabled, take a shortcut.
+	if !eventLogEnabled.Get(&ex.s.cfg.Settings.SV) {
+		return nil
+	}
+
 	const insertEventTableStmt = `
 INSERT INTO system.eventlog (
   timestamp, "eventType", "targetID", "reportingID", info

pkg/sql/logictest/testdata/logic_test/event_log

@@ -713,3 +713,25 @@ ORDER BY "timestamp", info
 1  {"EventType": "drop_schema", "InstanceID": 1, "SchemaName": "sc", "Statement": "DROP SCHEMA \"\".sc", "User": "root"}
 1  {"EventType": "drop_schema", "InstanceID": 1, "SchemaName": "s", "Statement": "DROP SCHEMA \"\".s, \"\".t", "User": "root"}
 1  {"EventType": "drop_schema", "InstanceID": 1, "SchemaName": "t", "Statement": "DROP SCHEMA \"\".s, \"\".t", "User": "root"}
+
+
+subtest eventlog_setting_disable
+
+statement ok
+SET CLUSTER SETTING server.eventlog.enabled = false
+
+statement ok
+CREATE ROLE rinvisible
+
+statement ok
+DROP ROLE rinvisible
+
+query ITT
+SELECT "reportingID", "eventType", info::JSONB - 'Timestamp' - 'DescriptorID'
+FROM system.eventlog
+WHERE "eventType" LIKE '%_role' AND info LIKE '%invisible%'
+----
+
+
+statement ok
+SET CLUSTER SETTING server.eventlog.enabled = false