Skip to content

Commit

Permalink
cluster-ui: speed up insights page requests
Browse files Browse the repository at this point in the history 
Previously, we were querying from `crdb_internal.statement_statistics` and
`crdb_internal.transaction_statistics` on the schema insights and txn insights
pages. Querying from this table is slow because it triggers a cluster-wide
fanout to collect unflushsed sql stats. We can use the persisted table instead,
which will lag behind a little in live data, but will be much faster to query from.

Some tests for the txn contention details api are also added.

Epic: none
Fixes: #107291

Release note (bug fix): Schema insights page should hit request timeouts less
frequently, if at all.
  • Loading branch information
@xinhaoz
xinhaoz committed Aug 2, 2023
1 parent 326a559 commit ca37116
Show file tree
Hide file tree
Showing 6 changed files with 281 additions and 41 deletions.
4 changes: 2 additions & 2 deletions pkg/ui/workspaces/cluster-ui/src/api/contentionApi.ts
View file
Expand Up @@ -61,7 +61,7 @@ export async function getContentionDetailsApi(
const result = await executeInternalSql<ContentionResponseColumns>(request);

if (sqlResultsAreEmpty(result)) {
if (result.error) {
if (result?.error) {
// We don't return an error if it failed to retrieve the contention information.
getLogger().error(
"Insights encounter an error while retrieving contention information.",
Expand All @@ -76,7 +76,7 @@ export async function getContentionDetailsApi(
}

const contentionDetails: ContentionDetails[] = [];
result.execution.txn_results.forEach(x => {
result.execution?.txn_results.forEach(x => {
x.rows.forEach(row => {
contentionDetails.push({
blockingExecutionID: row.blocking_txn_id,
Expand Down
2 changes: 1 addition & 1 deletion pkg/ui/workspaces/cluster-ui/src/api/schemaInsightsApi.ts
View file
Expand Up @@ -196,7 +196,7 @@ FROM
statistics -> 'statistics' ->> 'lastExecAt' DESC
) AS rank
FROM
crdb_internal.statement_statistics
crdb_internal.statement_statistics_persisted
WHERE
aggregated_ts >= now() - INTERVAL '1 week'
)
Expand Down
4 changes: 2 additions & 2 deletions pkg/ui/workspaces/cluster-ui/src/api/sqlApi.ts
View file
Expand Up @@ -126,8 +126,8 @@ export function sqlResultsAreEmpty(
result: SqlExecutionResponse<unknown>,
): boolean {
return (
!result.execution?.txn_results?.length ||
result.execution.txn_results.every(txn => txnResultIsEmpty(txn))
!result?.execution?.txn_results?.length ||
result?.execution.txn_results.every(txn => txnResultIsEmpty(txn))
);
}

Expand Down
234 changes: 234 additions & 0 deletions pkg/ui/workspaces/cluster-ui/src/api/txnInsightsApi.spec.ts
View file
@@ -0,0 +1,234 @@
// Copyright 2023 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.

import {
getTxnInsightsContentionDetailsApi,
TxnStmtFingerprintsResponseColumns,
FingerprintStmtsResponseColumns,
} from "./txnInsightsApi";
import * as sqlApi from "./sqlApi";
import { SqlExecutionResponse } from "./sqlApi";
import {
InsightExecEnum,
InsightNameEnum,
TxnContentionInsightDetails,
} from "../insights";
import { ContentionResponseColumns } from "./contentionApi";
import moment from "moment-timezone";

function mockSqlResponse<T>(rows: T[]): SqlExecutionResponse<T> {
return {
execution: {
retries: 0,
txn_results: [
{
tag: "",
start: "",
end: "",
rows_affected: 0,
statement: 1,
rows: [...rows],
},
],
},
};
}

type TxnContentionDetailsTests = {
name: string;
contentionResp: SqlExecutionResponse<ContentionResponseColumns>;
txnFingerprintsResp: SqlExecutionResponse<TxnStmtFingerprintsResponseColumns>;
stmtsFingerprintsResp: SqlExecutionResponse<FingerprintStmtsResponseColumns>;
expected: TxnContentionInsightDetails;
};

describe("test txn insights api functions", () => {
const waitingTxnID = "1a2a4828-5fc6-42d1-ab93-fadd4a514b69";
const contentionDetailsMock: ContentionResponseColumns = {
contention_duration: "00:00:00.00866",
waiting_stmt_id: "17761e953a52c0300000000000000001",
waiting_stmt_fingerprint_id: "b75264458f6e2ef3",
schema_name: "public",
database_name: "system",
table_name: "namespace",
index_name: "primary",
key: `/NamespaceTable/30/1/0/0/"movr"/4/1`,
collection_ts: "2023-07-28 19:25:36.434081+00",
blocking_txn_id: "a13773b3-9bca-4019-9cfb-a376d6a4f412",
blocking_txn_fingerprint_id: "4329ab5f4493f82d",
waiting_txn_id: waitingTxnID,
waiting_txn_fingerprint_id: "1831d909096f992c",
};

afterEach(() => {
jest.resetAllMocks();
jest.clearAllMocks();
});

test.each([
{
name: "all api responses empty",
contentionResp: mockSqlResponse([]),
txnFingerprintsResp: mockSqlResponse([]),
stmtsFingerprintsResp: mockSqlResponse([]),
expected: null,
},
{
name: "no fingerprints available",
contentionResp: mockSqlResponse([contentionDetailsMock]),
txnFingerprintsResp: mockSqlResponse([]),
stmtsFingerprintsResp: mockSqlResponse([]),
expected: {
transactionExecutionID: contentionDetailsMock.waiting_txn_id,
application: undefined,
transactionFingerprintID:
contentionDetailsMock.waiting_txn_fingerprint_id,
blockingContentionDetails: [
{
blockingExecutionID: contentionDetailsMock.blocking_txn_id,
blockingTxnFingerprintID:
contentionDetailsMock.blocking_txn_fingerprint_id,
blockingTxnQuery: null,
collectionTimeStamp: moment("2023-07-28 19:25:36.434081+00").utc(),
contendedKey: '/NamespaceTable/30/1/0/0/"movr"/4/1',
contentionTimeMs: 9,
databaseName: contentionDetailsMock.database_name,
indexName: contentionDetailsMock.index_name,
schemaName: contentionDetailsMock.schema_name,
tableName: contentionDetailsMock.table_name,
waitingStmtFingerprintID:
contentionDetailsMock.waiting_stmt_fingerprint_id,
waitingStmtID: contentionDetailsMock.waiting_stmt_id,
waitingTxnFingerprintID:
contentionDetailsMock.waiting_txn_fingerprint_id,
waitingTxnID: contentionDetailsMock.waiting_txn_id,
},
],
execType: InsightExecEnum.TRANSACTION,
insightName: InsightNameEnum.highContention,
},
},
{
name: "no stmt fingerprints available",
contentionResp: mockSqlResponse([contentionDetailsMock]),
txnFingerprintsResp: mockSqlResponse<TxnStmtFingerprintsResponseColumns>([
{
transaction_fingerprint_id:
contentionDetailsMock.blocking_txn_fingerprint_id,
query_ids: ["a", "b", "c"],
app_name: undefined,
},
]),
stmtsFingerprintsResp: mockSqlResponse([]),
expected: {
transactionExecutionID: contentionDetailsMock.waiting_txn_id,
application: undefined,
transactionFingerprintID:
contentionDetailsMock.waiting_txn_fingerprint_id,
blockingContentionDetails: [
{
blockingExecutionID: contentionDetailsMock.blocking_txn_id,
blockingTxnFingerprintID:
contentionDetailsMock.blocking_txn_fingerprint_id,
blockingTxnQuery: [
"Query unavailable for stmt fingerprint 000000000000000a",
"Query unavailable for stmt fingerprint 000000000000000b",
"Query unavailable for stmt fingerprint 000000000000000c",
],
collectionTimeStamp: moment("2023-07-28 19:25:36.434081+00").utc(),
contendedKey: '/NamespaceTable/30/1/0/0/"movr"/4/1',
contentionTimeMs: 9,
databaseName: contentionDetailsMock.database_name,
indexName: contentionDetailsMock.index_name,
schemaName: contentionDetailsMock.schema_name,
tableName: contentionDetailsMock.table_name,
waitingStmtFingerprintID:
contentionDetailsMock.waiting_stmt_fingerprint_id,
waitingStmtID: contentionDetailsMock.waiting_stmt_id,
waitingTxnFingerprintID:
contentionDetailsMock.waiting_txn_fingerprint_id,
waitingTxnID: contentionDetailsMock.waiting_txn_id,
},
],
execType: InsightExecEnum.TRANSACTION,
insightName: InsightNameEnum.highContention,
},
},
{
name: "all info available",
contentionResp: mockSqlResponse([contentionDetailsMock]),
txnFingerprintsResp: mockSqlResponse<TxnStmtFingerprintsResponseColumns>([
{
transaction_fingerprint_id:
contentionDetailsMock.blocking_txn_fingerprint_id,
query_ids: ["a", "b", "c"],
app_name: undefined,
},
]),
stmtsFingerprintsResp: mockSqlResponse<FingerprintStmtsResponseColumns>([
{
statement_fingerprint_id: "a",
query: "select 1",
},
{
statement_fingerprint_id: "b",
query: "select 2",
},
{
statement_fingerprint_id: "c",
query: "select 3",
},
]),
expected: {
transactionExecutionID: contentionDetailsMock.waiting_txn_id,
application: undefined,
transactionFingerprintID:
contentionDetailsMock.waiting_txn_fingerprint_id,
blockingContentionDetails: [
{
blockingExecutionID: contentionDetailsMock.blocking_txn_id,
blockingTxnFingerprintID:
contentionDetailsMock.blocking_txn_fingerprint_id,
blockingTxnQuery: ["select 1", "select 2", "select 3"],
collectionTimeStamp: moment("2023-07-28 19:25:36.434081+00").utc(),
contendedKey: '/NamespaceTable/30/1/0/0/"movr"/4/1',
contentionTimeMs: 9,
databaseName: contentionDetailsMock.database_name,
indexName: contentionDetailsMock.index_name,
schemaName: contentionDetailsMock.schema_name,
tableName: contentionDetailsMock.table_name,
waitingStmtFingerprintID:
contentionDetailsMock.waiting_stmt_fingerprint_id,
waitingStmtID: contentionDetailsMock.waiting_stmt_id,
waitingTxnFingerprintID:
contentionDetailsMock.waiting_txn_fingerprint_id,
waitingTxnID: contentionDetailsMock.waiting_txn_id,
},
],
execType: InsightExecEnum.TRANSACTION,
insightName: InsightNameEnum.highContention,
},
},
] as TxnContentionDetailsTests[])(
"$name",
async (tc: TxnContentionDetailsTests) => {
await jest
.spyOn(sqlApi, "executeInternalSql")
.mockReturnValueOnce(Promise.resolve(tc.contentionResp))
.mockReturnValueOnce(Promise.resolve(tc.txnFingerprintsResp))
.mockReturnValueOnce(Promise.resolve(tc.stmtsFingerprintsResp));

const res = await getTxnInsightsContentionDetailsApi({
txnExecutionID: waitingTxnID,
});
expect(res).toEqual(tc.expected);
},
);
});

0 comments on commit ca37116

Please sign in to comment.