Merge #111401 #113498 #113626 #113835

111401: sql: populate information_schema.user_defined_types and attributes r=annrpom a=annrpom

commit 1-4:
### sql: add necessary builtins for information_schema.user_defined_types 

These patches add the following builtins:
- information_schema._pg_char_octet_length: returns the octet length of a given char/varchar/string
- pg_encoding_max_length: returns the maximum length of a given encoding. Since we only support UTF8, this will return length 4.
- information_schema._pg_datetime_precision: returns the precision of the given datetime-related type
- information_schema._pg_interval_type: displays the interval type duration if it is set

Needed for: #109603
Epic: none

Release note (sql change): The information_schema._pg_char_octet_length, information_schema._pg_datetime_precision, information_schema._pg_interval_type, and pg_encoding_max_length builtin functions are now supported, which improves compatibility with PostgreSQL.

---
last commit:
### sql: populate information_schema.user_defined_types and attributes

This patch populates information_schema.user_defined_types with
information about user defined types and information_schema.attributes
with information about the attributes of composite data types.

Epic: none
Fixes: #109603

Release note (sql change): This patch populates
information_schema.user_defined_types with information about user
defined types and information_schema.attributes with information about
the attributes of composite data types.

113498: obsservice: export StatementInsightsStatistics r=maryliag a=maryliag

_commit 1:_

**obsservice: add stmt insights proto and conversion**

Add proto definition for Statement Insights to be
used on Obs Service.
Creates function to copy a `insights.Insight` to the new
`obspb.StatementInsightsStatistics` format.

For this version, there are some parameters not being set,
such as contention. Those will be populated on a following
iteration.

Epic: none

Release note: None

----

_commit 2:_

**obsservice: export StatementInsightsStatistics**

This patch hooks into the Flush functionality used
by PersistedSQLStats, in preparation for Insights
to be sent to external o11y systems.
A following PR will do the actual export during the flush.

Lastly, it's acknowledged that the transformation
required here is likely going to be heavy on
allocations. During the prototyping phase however,
we leave the optimization and/or restructuring of
the exported type for the future. For now, we use
a sync.Pool in an effort to reduce allocations/GC.

Release note: none


113626: debug: fix help text for `--include-range-info` debug zip flag r=dhartunian a=abarganier

Fixes: #112053

This patch fixes the help text for the `--include-range-info` command available in `cockroach debug zip`. This is necessary, because `debug zip` now only creates a single file per node. The help text was not updated to reflect this.

Release note: none

113835: team: embed TEAMS.yaml r=RaduBerinde a=RaduBerinde

The `internal/team` code looks for `TEAMS.yaml` in the repo. This
means that `roachtest` must be run in the tree (and if the branches
don't match, obscure errors could happen in principle).

This commit moves to embedding the data using `go:embed`. Because
`go:embed` doesn't allow embedding of files outside the package, we
also have to add a generation rule to copy the file.

Fixes: #111661
Release note: None

Co-authored-by: Annie Pompa <annie@cockroachlabs.com>
Co-authored-by: maryliag <marylia@cockroachlabs.com>
Co-authored-by: Alex Barganier <abarganier@cockroachlabs.com>
Co-authored-by: Radu Berinde <radu@cockroachlabs.com>

build/bazelutil/check.sh

@@ -44,6 +44,7 @@ pkg/util/log/channels.go://go:generate go run gen/main.go logpb/log.proto loggin
 pkg/util/log/channels.go://go:generate go run gen/main.go logpb/log.proto severity.go severity/severity_generated.go
 pkg/util/log/sinks.go://go:generate mockgen -package=log -destination=mocks_generated_test.go --mock_names=TestingLogSink=MockLogSink . TestingLogSink
 pkg/util/timeutil/zoneinfo.go://go:generate go run gen/main.go
+pkg/internal/team/team.go://go:generate cp ../../../TEAMS.yaml TEAMS.yaml
 "
 
 EXISTING_CRDB_TEST_BUILD_CONSTRAINTS="

pkg/cli/cliflags/flags.go

@@ -1673,8 +1673,15 @@ necessary to support CockroachDB.
 	ZipIncludeRangeInfo = FlagInfo{
 		Name: "include-range-info",
 		Description: `
-Include information about each individual range in nodes/*/ranges/*.json files.
-For large clusters, this can dramatically increase debug zip size/file count.
+Include one file per node with information about the KV ranges stored on that node, 
+in nodes/{node ID}/ranges.json. This information can be vital when debugging issues 
+that involve the KV storage layer, such as data placement, load balancing, performance 
+or other behaviors. In certain situations, on large clusters with large numbers of ranges, 
+these files can be omitted if and only if the issue being investigated is already known to
+be in another layer of the system (for example, an error message about an unsupported 
+feature or incompatible value in a SQL schema change or statement). Note however many 
+higher-level issues are ultimately related to the underlying KV storage layer described 
+by these files so only set this to false if directed to do so by Cockroach Labs support.
 `,
 	}
 

pkg/gen/misc.bzl

@@ -6,6 +6,7 @@ MISC_SRCS = [
     "//pkg/ccl/backupccl:restore_memory_monitoring_generated_test.go",
     "//pkg/ccl/backupccl:restore_mid_schema_change_generated_test.go",
     "//pkg/ccl/kvccl/kvtenantccl/upgradeinterlockccl:generated_test.go",
+    "//pkg/internal/team:TEAMS.yaml",
     "//pkg/kv/kvpb:batch_generated.go",
     "//pkg/kv/kvserver/concurrency:keylocks_interval_btree.go",
     "//pkg/kv/kvserver/concurrency:keylocks_interval_btree_test.go",

pkg/internal/team/.gitignore

@@ -0,0 +1 @@
+TEAMS.yaml

pkg/internal/team/BUILD.bazel

@@ -3,24 +3,34 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "team",
     srcs = ["team.go"],
+    data = [
+        ":gen-teams-yaml",  # keep
+    ],
+    embedsrcs = ["TEAMS.yaml"],
     importpath = "github.com/cockroachdb/cockroach/pkg/internal/team",
     visibility = ["//pkg:__subpackages__"],
     deps = [
-        "//pkg/build/bazel",
-        "//pkg/internal/reporoot",
         "@com_github_cockroachdb_errors//:errors",
         "@in_gopkg_yaml_v2//:yaml_v2",
     ],
 )
 
+genrule(
+    name = "gen-teams-yaml",
+    srcs = ["//:TEAMS.yaml"],
+    outs = ["TEAMS.yaml"],
+    cmd = "cat $(SRCS) > $@",
+    visibility = [
+        ":__pkg__",
+        "//pkg/gen:__pkg__",
+    ],
+)
+
 go_test(
     name = "team_test",
     size = "small",
     srcs = ["team_test.go"],
     args = ["-test.timeout=55s"],
-    data = [
-        "//:TEAMS.yaml",
-    ],
     embed = [":team"],
     deps = ["@com_github_stretchr_testify//require"],
 )

pkg/internal/team/team.go

@@ -13,12 +13,10 @@
 package team
 
 import (
+	_ "embed"
 	"io"
-	"os"
-	"path/filepath"
+	"strings"
 
-	"github.com/cockroachdb/cockroach/pkg/build/bazel"
-	"github.com/cockroachdb/cockroach/pkg/internal/reporoot"
 	"github.com/cockroachdb/errors"
 	"gopkg.in/yaml.v2"
 )
@@ -83,28 +81,14 @@ func (m Map) GetAliasesForPurpose(alias Alias, purpose Purpose) ([]Alias, bool)
 	return sl, true
 }
 
+//go:generate cp ../../../TEAMS.yaml TEAMS.yaml
+
+//go:embed TEAMS.yaml
+var teamsYaml string
+
 // DefaultLoadTeams loads teams from the repo root's TEAMS.yaml.
 func DefaultLoadTeams() (Map, error) {
-	var path string
-	if os.Getenv("BAZEL_TEST") != "" {
-		runfiles, err := bazel.RunfilesPath()
-		if err != nil {
-			return nil, err
-		}
-		path = filepath.Join(runfiles, "TEAMS.yaml")
-	} else {
-		root := reporoot.GetFor(".", "TEAMS.yaml")
-		if root == "" {
-			return nil, errors.New("TEAMS.yaml not found")
-		}
-		path = filepath.Join(root, "TEAMS.yaml")
-	}
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer func() { _ = f.Close() }()
-	return LoadTeams(f)
+	return LoadTeams(strings.NewReader(teamsYaml))
 }
 
 // Purpose determines which alias to return for a given team via

pkg/obs/event_exporter.go

@@ -220,6 +220,12 @@ func NewEventsExporter(
 				Version: "1.0",
 			},
 		},
+		obspb.StatementInsightsStatsEvent: {
+			instrumentationScope: otel_pb.InstrumentationScope{
+				Name:    string(obspb.StatementInsightsStatsEvent),
+				Version: "1.0",
+			},
+		},
 	}
 	s.buf.mu.memAccount = memMonitor.MakeBoundAccount()
 	return s

pkg/obsservice/cmd/obsservice/main.go

@@ -85,7 +85,8 @@ from one or more CockroachDB clusters.`,
 
 		// Run the event ingestion in the background.
 		eventRouter := router.NewEventRouter(map[obspb.EventType]obslib.EventConsumer{
-			obspb.EventlogEvent: &obsutil.StdOutConsumer{},
+			obspb.EventlogEvent:               &obsutil.StdOutConsumer{},
+			obspb.StatementInsightsStatsEvent: &obsutil.StdOutConsumer{},
 		})
 		ingester := ingest.MakeEventIngester(ctx, eventRouter, nil)
 

pkg/obsservice/obspb/BUILD.bazel

@@ -23,6 +23,8 @@ proto_library(
         "//pkg/obsservice/obspb/opentelemetry-proto/logs/v1:v1_proto",
         "//pkg/obsservice/obspb/opentelemetry-proto/resource/v1:v1_proto",
         "@com_github_gogo_protobuf//gogoproto:gogo_proto",
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:timestamp_proto",
     ],
 )
 

pkg/obsservice/obspb/event_types.go

@@ -14,7 +14,8 @@ type EventType string
 const (
 	// EventlogEvent represents general events about the cluster that historically
 	// have been persisted inside CRDB in the system.eventlog table.
-	EventlogEvent EventType = "eventlog"
+	EventlogEvent               EventType = "eventlog"
+	StatementInsightsStatsEvent EventType = "stmtinsightsstats"
 )
 
 // EventlogEventTypeAttribute represents the key of the attribute containing

pkg/obsservice/obspb/obsservice.proto

@@ -11,7 +11,8 @@ syntax = "proto3";
 package cockroach.obspb;
 
 import "gogoproto/gogo.proto";
-
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
 import "obsservice/obspb/opentelemetry-proto/logs/v1/logs.proto";
 import "obsservice/obspb/opentelemetry-proto/common/v1/common.proto";
 import "obsservice/obspb/opentelemetry-proto/resource/v1/resource.proto";
@@ -36,3 +37,107 @@ message Event {
   // The LogRecord containing the specific event information.
   opentelemetry.proto.logs.v1.LogRecord log_record = 3;
 }
+
+message StatementInsightsStatistics {
+  enum Status {
+    Completed = 0;
+    Failed = 1;
+  }
+
+  enum Problem {
+    // This statement execution was not itself problematic, but it happened in
+    // the same transaction as another problematic statement.
+    None = 0;
+
+    // This statement execution was slow, either above a fixed latency threshold
+    // or detected as an anomaly.
+    SlowExecution = 1;
+
+    // This statement execution failed completely, due to contention, resource
+    // saturation, or syntax errors.
+    FailedExecution = 2;
+  }
+
+  enum Cause {
+    // We have not identified specific reasons why this statement was slow.
+    // We will not explicitly use this zero value; it is here defensively.
+    Unset = 0;
+
+    // This statement was slow because we picked the wrong plan, possibly due to
+    // outdated statistics, the statement using different literals or search
+    // conditions, or a change in the database schema.
+    PlanRegression = 1;
+
+    // This statement was slow because a good plan was not available, whether
+    // due to outdated statistics or missing indexes.
+    SuboptimalPlan = 2;
+
+    // This statement was slow because of contention.
+    HighContention = 3;
+
+    // This statement was slow because of being retried multiple times, again due
+    // to contention. The "high" threshold may be configured by the
+    // `sql.insights.high_retry_count.threshold` cluster setting.
+    HighRetryCount = 4;
+  }
+
+  bytes id = 1 [(gogoproto.customname) = "ID"];
+  uint64 fingerprint_id = 2 [(gogoproto.customname) = "FingerprintID"];
+  bytes transaction_id = 3 [(gogoproto.customname) = "TransactionID"];
+  uint64 txn_fingerprint_id = 4 [(gogoproto.customname) = "TxnFingerprintID"];
+  bytes session_id = 5 [(gogoproto.customname) = "SessionID"];
+  string query = 6;
+  Status status = 7;
+  google.protobuf.Timestamp start_time = 8 [(gogoproto.stdtime) = true];
+  google.protobuf.Timestamp end_time = 9 [(gogoproto.stdtime) = true];
+  bool full_scan = 10;
+  bool implicit_txn = 11;
+  string user = 12;
+  string user_priority = 13;
+  string application_name = 14;
+  string database = 15;
+  string plan_gist = 16;
+  int64 rows_read = 17;
+  int64 rows_written = 18;
+  int64 retries = 19;
+  string auto_retry_reason = 20;
+  // Nodes is the ordered list of nodes ids on which the statement was executed.
+  repeated int64 nodes = 21;
+  google.protobuf.Duration contention = 22 [(gogoproto.stdduration) = true];
+  repeated string index_recommendations = 23;
+  Problem problem = 24;
+  repeated Cause causes = 25;
+  int64 cpu_sql_nanos = 26 [(gogoproto.customname) = "CPUSQLNanos"];
+  string error_code = 27;
+  // IdleLatSeconds is the time spent in seconds in an open transaction waiting
+  // for the client to send the statement.
+  double idle_lat_seconds = 28;
+  // ParseLat is the time in seconds to transform the SQL string into an AST.
+  double parse_lat_seconds = 29;
+  // PlanLatSeconds is the time spent in seconds to transform the AST into a logical query plan.
+  double plan_lat_seconds = 30;
+  // RunLatSeconds is the time in seconds to run the query and fetch/compute the result rows.
+  double run_lat_seconds = 31;
+  // ServiceLatSeconds is the time in seconds to service the query, from start of parse to end of execute.
+  double service_lat_seconds = 32;
+  repeated ContentionEvent contention_events = 33 [(gogoproto.nullable) = true];
+}
+
+message ContentionEvent {
+  // Key is the key that this and the other transaction conflicted on.
+  bytes key = 1;
+  string pretty_key = 2;
+  bytes id = 3 [(gogoproto.customname) = "BlockingTxnID"];
+  uint64 blocking_txn_fingerprint_id = 4 [(gogoproto.customname) = "BlockingTxnFingerprintID"];
+  // Duration spent contending against the other transaction.
+  google.protobuf.Duration duration = 5 [(gogoproto.stdduration) = true];
+  bytes waiting_txn_id = 6 [(gogoproto.customname) = "WaitingTxnID"];
+  uint64 waiting_txn_fingerprint_id = 7 [(gogoproto.customname) = "WaitingTxnFingerprintID"];
+  google.protobuf.Timestamp collection_ts = 8 [(gogoproto.stdtime) = true];
+  uint64 waiting_stmt_fingerprint_id = 9 [(gogoproto.customname) = "WaitingStmtFingerprintID"];
+  bytes waiting_stmt_id = 10 [(gogoproto.customname) = "WaitingStmtID"];
+  string database_name = 11;
+  string schema_name = 12;
+  string index_name = 13;
+  string table_name = 14;
+}