Skip to content

Commit

Permalink
sql: create and fill crdb_internal.node_transaction_statistics table
Browse files Browse the repository at this point in the history 
Previously there was no way to query the transaction level metrics
collected by individual nodes. This patch adds this capability by
exposing a new internal table called
`crdb_internal.node_transaction_statistics` which is analagous to
`crdb_internal.node_statement_statistics` albeit for transactions.

Closes #53504

Release justification: low risk update to new functionality
Release note (sql change): A new crdb_internal table called
`node_transaction_statistics` is exposed as part of this change,
which allows users to query transaction metrics collected on a
particular node.
  • Loading branch information
@arulajmani
arulajmani committed Sep 1, 2020
1 parent 3fd6e3b commit ee2287c
Show file tree
Hide file tree
Showing 4 changed files with 196 additions and 44 deletions.
10 changes: 6 additions & 4 deletions pkg/sql/app_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,13 +253,15 @@ func (a *appStats) getStatsForStmtWithKey(
return s
}

func (a *appStats) getStatsForTxnWithKey(key txnKey, stmtIDs []roachpb.StmtID) *txnStats {
func (a *appStats) getStatsForTxnWithKey(
key txnKey, stmtIDs []roachpb.StmtID, createIfNonexistent bool,
) *txnStats {
a.Lock()
defer a.Unlock()
// Retrieve the per-transaction statistic object, and create it if it doesn't
// exist yet.
s, ok := a.txns[key]
if !ok {
if !ok && createIfNonexistent {
s = &txnStats{}
s.statementIDs = stmtIDs
a.txns[key] = s
Expand Down Expand Up @@ -317,7 +319,7 @@ func (a *appStats) Add(other *appStats) {

// Merge the txn stats
for k, v := range txnMap {
t := a.getStatsForTxnWithKey(k, v.statementIDs)
t := a.getStatsForTxnWithKey(k, v.statementIDs, true /* createIfNonExistent */)
t.mu.Lock()
t.mu.data.Add(&v.mu.data)
t.mu.Unlock()
Expand Down Expand Up @@ -398,7 +400,7 @@ func (a *appStats) recordTransaction(
}

// Get the statistics object.
s := a.getStatsForTxnWithKey(key, statementIDs)
s := a.getStatsForTxnWithKey(key, statementIDs, true /* createIfNonexistent */)

// Collect the per-transaction statistics.
s.mu.Lock()
Expand Down
1 change: 1 addition & 0 deletions pkg/sql/catalog/catconstants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ const (
CrdbInternalTablesTableLastStatsID
CrdbInternalTxnStatsTableID
CrdbInternalZonesTableID
CrdbInternalTransactionStatsTableID
InformationSchemaID
InformationSchemaAdministrableRoleAuthorizationsID
InformationSchemaApplicableRolesID
Expand Down
112 changes: 112 additions & 0 deletions pkg/sql/crdb_internal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ var crdbInternal = virtualSchema{
catconstants.CrdbInternalTablesTableID: crdbInternalTablesTable,
catconstants.CrdbInternalTxnStatsTableID: crdbInternalTxnStatsTable,
catconstants.CrdbInternalZonesTableID: crdbInternalZonesTable,
catconstants.CrdbInternalTransactionStatsTableID: crdbInternalTransactionStatisticsTable,
},
validWithNoDatabaseContext: true,
}
Expand Down Expand Up @@ -715,6 +716,20 @@ func (s stmtList) Less(i, j int) bool {
return s[i].anonymizedStmt < s[j].anonymizedStmt
}

type txnList []txnKey

func (t txnList) Len() int {
return len(t)
}

func (t txnList) Swap(i, j int) {
t[i], t[j] = t[j], t[i]
}

func (t txnList) Less(i, j int) bool {
return t[i] < t[j]
}

var crdbInternalStmtStatsTable = virtualSchemaTable{
comment: `statement statistics (in-memory, not durable; local node only). ` +
`This table is wiped periodically (by default, at least every two hours)`,
Expand Down Expand Up @@ -845,6 +860,103 @@ CREATE TABLE crdb_internal.node_statement_statistics (
},
}

var crdbInternalTransactionStatisticsTable = virtualSchemaTable{
comment: `finer grain transaction statistics (in-memory, not durable; local node only). ` +
`This table is wiped periodically (by default, at least every two hours)`,
schema: `
CREATE TABLE crdb_internal.node_transaction_statistics (
node_id INT NOT NULL,
application_name STRING NOT NULL,
statement_ids STRING[],
count INT,
max_retries INT,
service_lat_avg FLOAT NOT NULL,
service_lat_var FLOAT NOT NULL,
retry_lat_avg FLOAT NOT NULL,
retry_lat_var FLOAT NOT NULL,
commit_lat_avg FLOAT NOT NULL,
commit_lat_var FLOAT NOT NULL,
rows_read_avg FLOAT NOT NULL,
rows_read_var FLOAT NOT NULL
)
`,
populate: func(ctx context.Context, p *planner, _ *dbdesc.Immutable, addRow func(...tree.Datum) error) error {
if err := p.RequireAdminRole(ctx, "access application statistics"); err != nil {
return err
}
sqlStats := p.extendedEvalCtx.sqlStatsCollector.sqlStats
if sqlStats == nil {
return errors.AssertionFailedf(
"cannot access sql statistics from this context")
}

nodeID, _ := p.execCfg.NodeID.OptionalNodeID() // zero if not available

// Retrieve the application names and sort them to ensure the
// output is deterministic.
var appNames []string
sqlStats.Lock()

for n := range sqlStats.apps {
appNames = append(appNames, n)
}
sqlStats.Unlock()
sort.Strings(appNames)

for _, appName := range appNames {
appStats := sqlStats.getStatsForApplication(appName)

// Retrieve the statement keys and sort them to ensure the
// output is deterministic.
var txnKeys txnList
appStats.Lock()
for k := range appStats.txns {
txnKeys = append(txnKeys, k)
}
appStats.Unlock()
sort.Sort(txnKeys)

// Now retrieve the per-txn stats proper.
for _, txnKey := range txnKeys {
// The key should exist, so it's okay to pass nil for the statementIDs
// as that is only set the first time a key is constructed.
s := appStats.getStatsForTxnWithKey(txnKey, nil, false /* createIfNonexistent */)
stmtIDsDatum := tree.NewDArray(types.String)
for _, stmtID := range s.statementIDs {
if err := stmtIDsDatum.Append(tree.NewDString(string(stmtID))); err != nil {
return err
}
}

s.mu.Lock()

err := addRow(
tree.NewDInt(tree.DInt(nodeID)),
tree.NewDString(appName),
stmtIDsDatum,
tree.NewDInt(tree.DInt(s.mu.data.Count)),
tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)),
tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)),
tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))),
tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.Mean)),
tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.GetVariance(s.mu.data.Count))),
tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.Mean)),
tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.GetVariance(s.mu.data.Count))),
tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)),
tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))),
)

s.mu.Unlock()
if err != nil {
return err
}
}

}
return nil
},
}

var crdbInternalTxnStatsTable = virtualSchemaTable{
comment: `per-application transaction statistics (in-memory, not durable; local node only). ` +
`This table is wiped periodically (by default, at least every two hours)`,
Expand Down
117 changes: 77 additions & 40 deletions pkg/sql/logictest/testdata/logic_test/crdb_internal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,46 +11,47 @@ DROP DATABASE crdb_internal
query TTTI
SHOW TABLES FROM crdb_internal
----
crdb_internal backward_dependencies table NULL
crdb_internal builtin_functions table NULL
crdb_internal cluster_queries table NULL
crdb_internal cluster_sessions table NULL
crdb_internal cluster_settings table NULL
crdb_internal cluster_transactions table NULL
crdb_internal create_statements table NULL
crdb_internal create_type_statements table NULL
crdb_internal databases table NULL
crdb_internal feature_usage table NULL
crdb_internal forward_dependencies table NULL
crdb_internal gossip_alerts table NULL
crdb_internal gossip_liveness table NULL
crdb_internal gossip_network table NULL
crdb_internal gossip_nodes table NULL
crdb_internal index_columns table NULL
crdb_internal jobs table NULL
crdb_internal kv_node_status table NULL
crdb_internal kv_store_status table NULL
crdb_internal leases table NULL
crdb_internal node_build_info table NULL
crdb_internal node_metrics table NULL
crdb_internal node_queries table NULL
crdb_internal node_runtime_info table NULL
crdb_internal node_sessions table NULL
crdb_internal node_statement_statistics table NULL
crdb_internal node_transactions table NULL
crdb_internal node_txn_stats table NULL
crdb_internal partitions table NULL
crdb_internal predefined_comments table NULL
crdb_internal ranges view NULL
crdb_internal ranges_no_leases table NULL
crdb_internal schema_changes table NULL
crdb_internal session_trace table NULL
crdb_internal session_variables table NULL
crdb_internal table_columns table NULL
crdb_internal table_indexes table NULL
crdb_internal table_row_statistics table NULL
crdb_internal tables table NULL
crdb_internal zones table NULL
crdb_internal backward_dependencies table NULL
crdb_internal builtin_functions table NULL
crdb_internal cluster_queries table NULL
crdb_internal cluster_sessions table NULL
crdb_internal cluster_settings table NULL
crdb_internal cluster_transactions table NULL
crdb_internal create_statements table NULL
crdb_internal create_type_statements table NULL
crdb_internal databases table NULL
crdb_internal feature_usage table NULL
crdb_internal forward_dependencies table NULL
crdb_internal gossip_alerts table NULL
crdb_internal gossip_liveness table NULL
crdb_internal gossip_network table NULL
crdb_internal gossip_nodes table NULL
crdb_internal index_columns table NULL
crdb_internal jobs table NULL
crdb_internal kv_node_status table NULL
crdb_internal kv_store_status table NULL
crdb_internal leases table NULL
crdb_internal node_build_info table NULL
crdb_internal node_metrics table NULL
crdb_internal node_queries table NULL
crdb_internal node_runtime_info table NULL
crdb_internal node_sessions table NULL
crdb_internal node_statement_statistics table NULL
crdb_internal node_transaction_statistics table NULL
crdb_internal node_transactions table NULL
crdb_internal node_txn_stats table NULL
crdb_internal partitions table NULL
crdb_internal predefined_comments table NULL
crdb_internal ranges view NULL
crdb_internal ranges_no_leases table NULL
crdb_internal schema_changes table NULL
crdb_internal session_trace table NULL
crdb_internal session_variables table NULL
crdb_internal table_columns table NULL
crdb_internal table_indexes table NULL
crdb_internal table_row_statistics table NULL
crdb_internal tables table NULL
crdb_internal zones table NULL

statement ok
CREATE DATABASE testdb; CREATE TABLE testdb.foo(x INT)
Expand Down Expand Up @@ -133,6 +134,11 @@ SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
----
node_id application_name flags key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var implicit_txn

query ITTIIRRRRRRRR colnames
SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
----
node_id application_name statement_ids count max_retries service_lat_avg service_lat_var retry_lat_avg retry_lat_var commit_lat_avg commit_lat_var rows_read_avg rows_read_var

query IITTTTTTT colnames
SELECT * FROM crdb_internal.session_trace WHERE span_idx < 0
----
Expand Down Expand Up @@ -637,3 +643,34 @@ SELECT * FROM crdb_internal.create_type_statements
52 test public 62 enum2 CREATE TYPE public.enum2 AS ENUM () {}

# Test the virtual index as well.

statement ok
SET application_name = "test_txn_statistics"

statement ok
CREATE TABLE t_53504()

statement ok
BEGIN; SELECT * FROM t_53504; SELECT * FROM t_53504; SELECT * FROM t_53504; COMMIT;

statement ok
BEGIN; SELECT * FROM t_53504; SELECT * FROM t_53504; COMMIT;

statement ok
BEGIN; SELECT * FROM t_53504; SELECT * FROM t_53504; COMMIT;

statement ok
BEGIN; SELECT * FROM t_53504; COMMIT;

statement ok
SELECT * FROM t_53504

query ITTI colnames
SELECT node_id, application_name, statement_ids, count FROM crdb_internal.node_transaction_statistics where application_name = 'test_txn_statistics'
----
node_id application_name statement_ids count
1 test_txn_statistics {7d5470c38539309ff3b933fec35fefad,7d5470c38539309ff3b933fec35fefad,7d5470c38539309ff3b933fec35fefad} 1
1 test_txn_statistics {94a482186de515de23eca3d1b443e033} 1
1 test_txn_statistics {7d5470c38539309ff3b933fec35fefad} 1
1 test_txn_statistics {7d5470c38539309ff3b933fec35fefad,7d5470c38539309ff3b933fec35fefad} 2
1 test_txn_statistics {ca672b3b015c5f7ca3b4d8488eb2f528} 1

0 comments on commit ee2287c

Please sign in to comment.