diff --git a/docs/generated/logsinks.md b/docs/generated/logsinks.md index 33e5ecf1ee29..c86762f3f96b 100644 --- a/docs/generated/logsinks.md +++ b/docs/generated/logsinks.md @@ -223,6 +223,7 @@ Type-specific configuration options: | `timeout` | the HTTP timeout. Defaults to 0 for no timeout. Inherited from `http-defaults.timeout` if not specified. | | `disable-keep-alives` | causes the logging sink to re-establish a new connection for every outgoing log message. This option is intended for testing only and can cause excessive network overhead in production systems. Inherited from `http-defaults.disable-keep-alives` if not specified. | | `headers` | a list of headers to attach to each HTTP request Inherited from `http-defaults.headers` if not specified. | +| `file-based-headers` | a list of headers with filepaths whose contents are attached to each HTTP request Inherited from `http-defaults.file-based-headers` if not specified. | | `compression` | can be "none" or "gzip" to enable gzip compression. Set to "gzip" by default. Inherited from `http-defaults.compression` if not specified. | diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/components/linegraph/index.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/components/linegraph/index.tsx index e25e1129f3fb..3288667a2661 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/components/linegraph/index.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/components/linegraph/index.tsx @@ -67,6 +67,7 @@ export interface OwnProps extends MetricsDataComponentProps { hoverState?: HoverState; preCalcGraphSize?: boolean; legendAsTooltip?: boolean; + showMetricsInTooltip?: boolean; } export type LineGraphProps = OwnProps & WithTimezoneProps; @@ -381,7 +382,41 @@ export class InternalLineGraph extends React.Component { tenantSource, preCalcGraphSize, legendAsTooltip, + showMetricsInTooltip, } = this.props; + let tt = tooltip; + const addLines: React.ReactNode = tooltip ? ( + <> +
+
+ + ) : null; + // Extend tooltip to include metrics names + if (showMetricsInTooltip) { + if (data?.results?.length === 1) { + tt = ( + <> + {tt} + {addLines} + Metric: {data.results[0].query.name} + + ); + } else if (data?.results?.length > 1) { + tt = ( + <> + {tt} + {addLines} + Metrics: + + + ); + } + } + if (!this.hasDataPoints(data) && isSecondaryTenant(tenantSource)) { return (
@@ -404,7 +439,7 @@ export class InternalLineGraph extends React.Component { diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/changefeeds.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/changefeeds.tsx index 95876a87a9fb..bdd4993d83d2 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/changefeeds.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/changefeeds.tsx @@ -37,6 +37,7 @@ export default function (props: GraphDashboardProps) { isKvGraph={false} sources={storeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > , - + @@ -125,7 +139,10 @@ export default function (props: GraphDashboardProps) { @@ -157,7 +176,10 @@ export default function (props: GraphDashboardProps) { @@ -171,13 +193,16 @@ export default function (props: GraphDashboardProps) { @@ -188,7 +213,9 @@ export default function (props: GraphDashboardProps) { title="Ranges in catchup mode" isKvGraph={false} sources={storeSources} - tooltip="Total number of ranges with an active rangefeed that are performing catchup scan" + tooltip={`Total number of ranges with an active rangefeed that are performing + catchup scan`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/distributed.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/distributed.tsx index 51db6fe7d6c6..778243f621aa 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/distributed.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/distributed.tsx @@ -25,6 +25,7 @@ export default function (props: GraphDashboardProps) { title="Batches" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > , - + @@ -99,7 +107,8 @@ export default function (props: GraphDashboardProps) { title="KV Transaction Durations: 99th percentile" tenantSource={tenantSource} tooltip={`The 99th percentile of transaction durations over a 1 minute period. - Values are displayed individually for each node.`} + Values are displayed individually for each node.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -118,7 +127,8 @@ export default function (props: GraphDashboardProps) { title="KV Transaction Durations: 90th percentile" tenantSource={tenantSource} tooltip={`The 90th percentile of transaction durations over a 1 minute period. - Values are displayed individually for each node.`} + Values are displayed individually for each node.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -136,8 +146,10 @@ export default function (props: GraphDashboardProps) { {_.map(nodeIDs, node => ( @@ -155,8 +167,10 @@ export default function (props: GraphDashboardProps) { {_.map(nodeIDs, node => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/graphTooltips.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/graphTooltips.tsx index cb9ff3898c4b..29109f69e341 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/graphTooltips.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/graphTooltips.tsx @@ -152,25 +152,25 @@ export const TransactionRestartsToolTip: React.FC<{
); -export const CircuitBreakerTrippedReplicasTooltip: React.FC = () => ( +export const CircuitBreakerTrippedReplicasTooltip: React.ReactNode = (
Number of Replicas for which the per-Replica circuit breaker is currently tripped.
); -export const CircuitBreakerTrippedEventsTooltip: React.FC = () => ( +export const CircuitBreakerTrippedEventsTooltip: React.ReactNode = (
The number of circuit breaker events occurred per aggregated interval of time across all nodes since the process started.
); -export const PausedFollowersTooltip: React.FC = () => ( +export const PausedFollowersTooltip: React.ReactNode = (
The number of nonessential followers that have replication paused.
); -export const ReceiverSnapshotsQueuedTooltip: React.FC = () => ( +export const ReceiverSnapshotsQueuedTooltip: React.ReactNode = (
The number of snapshots queued to be applied on a receiver which can only{" "} accept 1 at a time per store. diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/hardware.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/hardware.tsx index 92f005e2e0da..20838eef4454 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/hardware.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/hardware.tsx @@ -40,6 +40,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={
CPU usage for the CRDB nodes {tooltipSelection}
} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -57,6 +58,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={
Machine-wide CPU usage {tooltipSelection}
} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -74,6 +76,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={
Memory in use {tooltipSelection}
} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -90,6 +93,7 @@ export default function (props: GraphDashboardProps) { title="Disk Read MiB/s" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -107,6 +111,7 @@ export default function (props: GraphDashboardProps) { title="Disk Write MiB/s" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -124,6 +129,7 @@ export default function (props: GraphDashboardProps) { title="Disk Read IOPS" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -141,6 +147,7 @@ export default function (props: GraphDashboardProps) { title="Disk Write IOPS" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -158,6 +165,7 @@ export default function (props: GraphDashboardProps) { title="Disk Ops In Progress" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -175,6 +183,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/networking.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/networking.tsx index 5577fbb65aa6..38f09d66ad9f 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/networking.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/networking.tsx @@ -25,7 +25,7 @@ export default function (props: GraphDashboardProps) { props; return [ - + {nodeIDs.map(nid => ( , - + {nodeIDs.map(nid => ( {nodeIDs.map(nid => ( @@ -75,6 +76,7 @@ export default function (props: GraphDashboardProps) { title="RPC Heartbeat Latency: 99th percentile" isKvGraph={false} tooltip={`Round-trip latency for recent successful outgoing heartbeats.`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -91,7 +93,9 @@ export default function (props: GraphDashboardProps) { {nodeIDs.map(nid => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overload.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overload.tsx index a71456253d0e..d3b57855c0e7 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overload.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overload.tsx @@ -35,6 +35,7 @@ export default function (props: GraphDashboardProps) { title="CPU Percent" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -52,6 +53,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={`P99 scheduling latency for goroutines`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -73,6 +75,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={`The number of Goroutines waiting per CPU.`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -90,6 +93,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={`The number of sublevels/files in L0 normalized by admission thresholds.`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -109,6 +113,7 @@ export default function (props: GraphDashboardProps) { title="KV Admission Slots" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -134,6 +139,7 @@ export default function (props: GraphDashboardProps) { title="KV Admission IO Tokens Exhausted Duration Per Second" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -152,6 +158,7 @@ export default function (props: GraphDashboardProps) { title="Flow Tokens Wait Time: 75th percentile" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -185,6 +192,7 @@ export default function (props: GraphDashboardProps) { title="Requests Waiting For Flow Tokens" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -216,6 +224,7 @@ export default function (props: GraphDashboardProps) { title="Blocked Replication Streams" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -247,6 +256,7 @@ export default function (props: GraphDashboardProps) { title="Admission Work Rate" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -299,6 +309,7 @@ export default function (props: GraphDashboardProps) { title="Admission Delay Rate" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -344,6 +355,7 @@ export default function (props: GraphDashboardProps) { title="Admission Delay: 75th percentile" sources={nodeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overview.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overview.tsx index 982a221743ea..51c78319e6de 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overview.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overview.tsx @@ -39,8 +39,9 @@ export default function (props: GraphDashboardProps) { isKvGraph={false} sources={nodeSources} tenantSource={tenantSource} - tooltip={`A moving average of the number of SELECT, INSERT, UPDATE, and DELETE statements - successfully executed per second ${tooltipSelection}.`} + tooltip={`A moving average of the number of SELECT, INSERT, UPDATE, and DELETE + statements successfully executed per second ${tooltipSelection}.`} + showMetricsInTooltip={true} preCalcGraphSize={true} > @@ -81,6 +82,7 @@ export default function (props: GraphDashboardProps) {
} + showMetricsInTooltip={true} preCalcGraphSize={true} > @@ -100,7 +102,9 @@ export default function (props: GraphDashboardProps) { isKvGraph={false} sources={nodeSources} tenantSource={tenantSource} - tooltip={`A moving average of the number of SQL statements executed per second that experienced contention ${tooltipSelection}.`} + tooltip={`A moving average of the number of SQL statements executed per second + that experienced contention ${tooltipSelection}.`} + showMetricsInTooltip={true} preCalcGraphSize={true} > @@ -124,6 +128,7 @@ export default function (props: GraphDashboardProps) { } + showMetricsInTooltip={true} preCalcGraphSize={true} > @@ -144,6 +149,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={} + showMetricsInTooltip={true} preCalcGraphSize={true} > diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/queues.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/queues.tsx index 775305a616ed..011219f42654 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/queues.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/queues.tsx @@ -30,6 +30,7 @@ export default function (props: GraphDashboardProps) { title="Queue Processing Failures" sources={storeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/replication.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/replication.tsx index a462ab89badd..1f7836ad485e 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/replication.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/replication.tsx @@ -42,9 +42,10 @@ export default function (props: GraphDashboardProps) { title="Ranges" sources={storeSources} tenantSource={tenantSource} - tooltip={`Various details about the status of ranges. In the node view, - shows details about ranges the node is responsible for. In the cluster - view, shows details about ranges all across the cluster.`} + tooltip={`Various details about the status of ranges. In the node view, shows + details about ranges the node is responsible for. In the cluster view, + shows details about ranges all across the cluster.`} + showMetricsInTooltip={true} > @@ -67,6 +68,7 @@ export default function (props: GraphDashboardProps) { title="Replicas per Node" tenantSource={tenantSource} tooltip={`The number of replicas on each node.`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -83,8 +85,10 @@ export default function (props: GraphDashboardProps) { {nodeIDs.map(nid => ( @@ -102,8 +106,9 @@ export default function (props: GraphDashboardProps) { title="Average Replica Queries per Node" tenantSource={tenantSource} tooltip={`Moving average of the number of KV batch requests processed by - leaseholder replicas on each node per second. Tracks roughly the last - 30 minutes of requests. Used for load-based rebalancing decisions.`} + leaseholder replicas on each node per second. Tracks roughly the last + 30 minutes of requests. Used for load-based rebalancing decisions.`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -121,8 +126,9 @@ export default function (props: GraphDashboardProps) { title="Average Replica CPU per Node" tenantSource={tenantSource} tooltip={`Moving average of all replica CPU usage on each node per second. - Tracks roughly the last 30 minutes of usage. Used for load-based - rebalancing decisions.`} + Tracks roughly the last 30 minutes of usage. Used for load-based + rebalancing decisions.`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -140,6 +146,7 @@ export default function (props: GraphDashboardProps) { title="Logical Bytes per Node" tenantSource={tenantSource} tooltip={} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -157,6 +164,7 @@ export default function (props: GraphDashboardProps) { title="Replica Quiescence" sources={storeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > @@ -168,6 +176,7 @@ export default function (props: GraphDashboardProps) { title="Range Operations" sources={storeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > @@ -196,6 +205,7 @@ export default function (props: GraphDashboardProps) { title="Snapshots" sources={storeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -257,6 +268,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={ReceiverSnapshotsQueuedTooltip} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -274,6 +286,7 @@ export default function (props: GraphDashboardProps) { title="Circuit Breaker Tripped Replicas" tenantSource={tenantSource} tooltip={CircuitBreakerTrippedReplicasTooltip} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -292,6 +305,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={PausedFollowersTooltip} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -309,6 +323,7 @@ export default function (props: GraphDashboardProps) { title="Replicate Queue Actions: Successes" sources={storeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/requests.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/requests.tsx index ad1f52397aaf..34d69682bda5 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/requests.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/requests.tsx @@ -23,6 +23,7 @@ export default function (props: GraphDashboardProps) { title="Slow Raft Proposals" sources={storeSources} tenantSource={tenantSource} + showMetricsInTooltip={true} > } + showMetricsInTooltip={true} > @@ -76,8 +78,9 @@ export default function (props: GraphDashboardProps) { title="Goroutine Count" sources={nodeSources} tenantSource={tenantSource} - tooltip={`The number of Goroutines ${tooltipSelection}. - This count should rise and fall based on load.`} + tooltip={`The number of Goroutines ${tooltipSelection}. This count should rise + and fall based on load.`} + showMetricsInTooltip={true} > @@ -88,8 +91,9 @@ export default function (props: GraphDashboardProps) { title="Runnable Goroutines per CPU" sources={nodeSources} tenantSource={tenantSource} - tooltip={`The number of Goroutines waiting for CPU ${tooltipSelection}. - This count should rise and fall based on load.`} + tooltip={`The number of Goroutines waiting for CPU ${tooltipSelection}. This + count should rise and fall based on load.`} + showMetricsInTooltip={true} > {nodeIDs.map(nid => ( @@ -109,6 +113,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={`The number of times that Go’s garbage collector was invoked per second ${tooltipSelection}.`} + showMetricsInTooltip={true} > @@ -119,9 +124,10 @@ export default function (props: GraphDashboardProps) { title="GC Pause Time" sources={nodeSources} tenantSource={tenantSource} - tooltip={`The amount of processor time used by Go’s garbage collector - per second ${tooltipSelection}. - During garbage collection, application code execution is paused.`} + tooltip={`The amount of processor time used by Go’s garbage collector per second + ${tooltipSelection}. During garbage collection, application code + execution is paused.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/sql.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/sql.tsx index abf5d33e7543..3bc464c89a91 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/sql.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/sql.tsx @@ -39,6 +39,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={`The total number of open SQL Sessions ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -58,6 +59,7 @@ export default function (props: GraphDashboardProps) { isKvGraph={false} sources={nodeSources} tooltip={`Rate of SQL connection attempts ${tooltipSelection}`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -78,7 +80,8 @@ export default function (props: GraphDashboardProps) { isKvGraph={false} sources={nodeSources} tenantSource={tenantSource} - tooltip={`The total number of open SQL transactions ${tooltipSelection}.`} + tooltip={`The total number of open SQL transactions ${tooltipSelection}.`} + showMetricsInTooltip={true} > @@ -123,8 +128,9 @@ export default function (props: GraphDashboardProps) { isKvGraph={false} sources={nodeSources} tenantSource={tenantSource} - tooltip={`A ten-second moving average of the # of SELECT, INSERT, UPDATE, and DELETE statements - successfully executed per second ${tooltipSelection}.`} + tooltip={`A ten-second moving average of the # of SELECT, INSERT, UPDATE, and + DELETE statements successfully executed per second ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -208,7 +216,9 @@ export default function (props: GraphDashboardProps) { title="Active Flows for Distributed SQL Statements" isKvGraph={false} tenantSource={tenantSource} - tooltip="The number of flows on each node contributing to currently running distributed SQL statements." + tooltip={`The number of flows on each node contributing to currently running + distributed SQL statements.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -226,7 +236,9 @@ export default function (props: GraphDashboardProps) { title="Connection Latency: 99th percentile" isKvGraph={false} tenantSource={tenantSource} - tooltip={`Over the last minute, this node established and authenticated 99% of connections within this time.`} + tooltip={`Over the last minute, this node established and authenticated 99% of + connections within this time.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -245,7 +257,9 @@ export default function (props: GraphDashboardProps) { title="Connection Latency: 90th percentile" isKvGraph={false} tenantSource={tenantSource} - tooltip={`Over the last minute, this node established and authenticated 90% of connections within this time.`} + tooltip={`Over the last minute, this node established and authenticated 90% of + connections within this time.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -274,6 +288,7 @@ export default function (props: GraphDashboardProps) { } + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -302,6 +317,7 @@ export default function (props: GraphDashboardProps) { } + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -330,6 +346,7 @@ export default function (props: GraphDashboardProps) { } + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -358,6 +375,7 @@ export default function (props: GraphDashboardProps) { } + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -376,8 +394,10 @@ export default function (props: GraphDashboardProps) { title="KV Execution Latency: 99th percentile" isKvGraph={true} tenantSource={tenantSource} - tooltip={`The 99th percentile of latency between query requests and responses over a - 1 minute period. Values are displayed individually for each node.`} + tooltip={`The 99th percentile of latency between query requests and responses + over a 1 minute period. Values are displayed individually for each + node.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -396,8 +416,10 @@ export default function (props: GraphDashboardProps) { title="KV Execution Latency: 90th percentile" isKvGraph={true} tenantSource={tenantSource} - tooltip={`The 90th percentile of latency between query requests and responses over a - 1 minute period. Values are displayed individually for each node.`} + tooltip={`The 90th percentile of latency between query requests and responses + over a 1 minute period. Values are displayed individually for each + node.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -417,8 +439,9 @@ export default function (props: GraphDashboardProps) { isKvGraph={false} sources={nodeSources} tenantSource={tenantSource} - tooltip={`The total number of transactions initiated, committed, rolled back, - or aborted per second ${tooltipSelection}.`} + tooltip={`The total number of transactions initiated, committed, rolled back, or + aborted per second ${tooltipSelection}.`} + showMetricsInTooltip={true} > } + showMetricsInTooltip={true} > } + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -539,6 +564,7 @@ export default function (props: GraphDashboardProps) { } + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -557,8 +583,9 @@ export default function (props: GraphDashboardProps) { title="SQL Memory" isKvGraph={false} tenantSource={tenantSource} - tooltip={`The current amount of allocated SQL memory. This amount is - compared against the node's --max-sql-memory flag.`} + tooltip={`The current amount of allocated SQL memory. This amount is compared + against the node's --max-sql-memory flag.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, node => ( @@ -579,6 +606,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={`The total number of DDL statements per second ${tooltipSelection}.`} + showMetricsInTooltip={true} > } + showMetricsInTooltip={true} > } + showMetricsInTooltip={true} > @@ -59,6 +60,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={} + showMetricsInTooltip={true} > @@ -70,8 +72,9 @@ export default function (props: GraphDashboardProps) { title="Log Commit Latency: 99th Percentile" sources={storeSources} tenantSource={tenantSource} - tooltip={`The 99th %ile latency for commits to the Raft Log. - This measures essentially an fdatasync to the storage engine's write-ahead log.`} + tooltip={`The 99th %ile latency for commits to the Raft Log. This measures + essentially an fdatasync to the storage engine's write-ahead log.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -89,8 +92,9 @@ export default function (props: GraphDashboardProps) { title="Log Commit Latency: 50th Percentile" sources={storeSources} tenantSource={tenantSource} - tooltip={`The 50th %ile latency for commits to the Raft Log. - This measures essentially an fdatasync to the storage engine's write-ahead log.`} + tooltip={`The 50th %ile latency for commits to the Raft Log. This measures + essentially an fdatasync to the storage engine's write-ahead log.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -108,9 +112,10 @@ export default function (props: GraphDashboardProps) { title="Command Commit Latency: 99th Percentile" sources={storeSources} tenantSource={tenantSource} - tooltip={`The 99th %ile latency for commits of Raft commands. - This measures applying a batch to the storage engine - (including writes to the write-ahead log), but no fsync.`} + tooltip={`The 99th %ile latency for commits of Raft commands. This measures + applying a batch to the storage engine (including writes to the + write-ahead log), but no fsync.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -128,9 +133,10 @@ export default function (props: GraphDashboardProps) { title="Command Commit Latency: 50th Percentile" sources={storeSources} tenantSource={tenantSource} - tooltip={`The 50th %ile latency for commits of Raft commands. - This measures applying a batch to the storage engine - (including writes to the write-ahead log), but no fsync.`} + tooltip={`The 50th %ile latency for commits of Raft commands. This measures + applying a batch to the storage engine (including writes to the + write-ahead log), but no fsync.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -148,7 +154,9 @@ export default function (props: GraphDashboardProps) { title="Read Amplification" sources={storeSources} tenantSource={tenantSource} - tooltip={`The average number of real read operations executed per logical read operation ${tooltipSelection}.`} + tooltip={`The average number of real read operations executed per logical read + operation ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -167,6 +175,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={`The number of SSTables in use ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -184,8 +193,9 @@ export default function (props: GraphDashboardProps) { title="File Descriptors" sources={nodeSources} tenantSource={tenantSource} - tooltip={`The number of open file descriptors ${tooltipSelection}, compared with the - file descriptor limit.`} + tooltip={`The number of open file descriptors ${tooltipSelection}, compared with + the file descriptor limit.`} + showMetricsInTooltip={true} > @@ -198,6 +208,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={`Bytes written by memtable flushes ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -217,6 +228,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={`Bytes written to WAL files ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -236,6 +248,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={`Bytes written by compactions ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -255,6 +268,7 @@ export default function (props: GraphDashboardProps) { sources={storeSources} tenantSource={tenantSource} tooltip={`Bytes written by sstable ingestions ${tooltipSelection}.`} + showMetricsInTooltip={true} > {_.map(nodeIDs, nid => ( @@ -273,8 +287,10 @@ export default function (props: GraphDashboardProps) { title="Write Stalls" sources={storeSources} tenantSource={tenantSource} - tooltip={`The number of intentional write stalls per second ${tooltipSelection}. Write stalls - are used to backpressure incoming writes during periods of heavy write traffic.`} + tooltip={`The number of intentional write stalls per second ${tooltipSelection}. + Write stalls are used to backpressure incoming writes during periods + of heavy write traffic.`} + showMetricsInTooltip={true} > } + showMetricsInTooltip={true} > {_.map(percentiles, p => ( @@ -91,6 +94,7 @@ export default function (props: GraphDashboardProps) { sources={nodeSources} tenantSource={tenantSource} tooltip={`Number of active spans being processed by TTL.`} + showMetricsInTooltip={true} > 0 { + hs.dynamicHeaders = &dynamicHeaders{ + headerToFilepath: dhFilepaths, + } + err := hs.RefreshDynamicHeaders() + if err != nil { + return nil, err + } + } return hs, nil } @@ -71,6 +91,19 @@ type httpSink struct { contentType string doRequest func(sink *httpSink, logEntry []byte) (*http.Response, error) config *logconfig.HTTPSinkConfig + // staticHeaders holds all the config headers defined by direct values. + staticHeaders map[string]string + // dynamicHeaders holds all the config headers defined by values from files. + // It will be nil if there are no filepaths provided. + dynamicHeaders *dynamicHeaders +} + +type dynamicHeaders struct { + headerToFilepath map[string]string + mu struct { + syncutil.Mutex + headerToValue map[string]string + } } // output emits some formatted bytes to this sink. @@ -123,9 +156,20 @@ func doPost(hs *httpSink, b []byte) (*http.Response, error) { req.Header.Add(httputil.ContentEncodingHeader, httputil.GzipEncoding) } - for k, v := range hs.config.Headers { + // Add both the staticHeaders and dynamicHeaders to the request. + for k, v := range hs.staticHeaders { req.Header.Add(k, v) } + // If the filepathMap was populated we know to check the values. + if hs.dynamicHeaders != nil { + func() { + hs.dynamicHeaders.mu.Lock() + defer hs.dynamicHeaders.mu.Unlock() + for k, v := range hs.dynamicHeaders.mu.headerToValue { + req.Header.Add(k, v) + } + }() + } req.Header.Add(httputil.ContentTypeHeader, hs.contentType) resp, err := hs.client.Do(req) if err != nil { @@ -172,3 +216,24 @@ func (e HTTPLogError) Error() string { "received %v response attempting to log to [%v]", e.StatusCode, e.Address) } + +// RefreshDynamicHeaders loads and sets the new dynamic headers for a given sink. +// It iterates over dynamicHeaders.filepath reading each file for contents and then +// updating dynamicHeaders.mu.value. +func (hs *httpSink) RefreshDynamicHeaders() error { + if hs.dynamicHeaders == nil { + return nil + } + dhVals := make(map[string]string, len(hs.dynamicHeaders.headerToFilepath)) + for key, filepath := range hs.dynamicHeaders.headerToFilepath { + data, err := os.ReadFile(filepath) + if err != nil { + return err + } + dhVals[key] = string(data) + } + hs.dynamicHeaders.mu.Lock() + defer hs.dynamicHeaders.mu.Unlock() + hs.dynamicHeaders.mu.headerToValue = dhVals + return nil +} diff --git a/pkg/util/log/http_sink_test.go b/pkg/util/log/http_sink_test.go index 8debeac07128..234d043fbe09 100644 --- a/pkg/util/log/http_sink_test.go +++ b/pkg/util/log/http_sink_test.go @@ -16,6 +16,8 @@ import ( "io" "net" "net/http" + "os" + "path/filepath" "strings" "testing" "time" @@ -28,6 +30,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" ) var ( @@ -53,6 +56,7 @@ func testBase( fn func(header http.Header, body string) error, hangServer bool, deadline time.Duration, + recall time.Duration, ) { sc := ScopeWithoutShowLogs(t) defer sc.Close(t) @@ -164,6 +168,13 @@ func testBase( return } + // Issue a second log event if recall is specified so that the test can be + // run again. + if recall > 0 { + time.Sleep(recall) + Ops.Infof(context.Background(), "hello world") + } + // If the test was not requiring a timeout, it was requiring some // logging message to match the predicate. If we don't see the // predicate match, it is a test failure. @@ -200,7 +211,7 @@ func TestMessageReceived(t *testing.T) { return nil } - testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0)) + testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0), time.Duration(0)) } // TestHTTPSinkTimeout verifies that a log call to a hanging server doesn't last @@ -223,7 +234,7 @@ func TestHTTPSinkTimeout(t *testing.T) { }, } - testBase(t, defaults, nil /* testFn */, true /* hangServer */, 500*time.Millisecond) + testBase(t, defaults, nil /* testFn */, true /* hangServer */, 500*time.Millisecond, time.Duration(0)) } // TestHTTPSinkContentTypeJSON verifies that the HTTP sink content type @@ -258,7 +269,7 @@ func TestHTTPSinkContentTypeJSON(t *testing.T) { return nil } - testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0)) + testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0), time.Duration(0)) } // TestHTTPSinkContentTypePlainText verifies that the HTTP sink content type @@ -293,7 +304,7 @@ func TestHTTPSinkContentTypePlainText(t *testing.T) { return nil } - testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0)) + testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0), time.Duration(0)) } func TestHTTPSinkHeadersAndCompression(t *testing.T) { @@ -305,6 +316,13 @@ func TestHTTPSinkHeadersAndCompression(t *testing.T) { format := "json" expectedContentType := "application/json" expectedContentEncoding := logconfig.GzipCompression + val := "secret-value" + filepathVal := "another-secret-value" + filepathReplaceVal := "third-secret-value" + // Test filepath method of providing header values. + tempDir := t.TempDir() + filename := filepath.Join(tempDir, "filepath_test.txt") + require.NoError(t, os.WriteFile(filename, []byte(filepathVal), 0777)) defaults := logconfig.HTTPDefaults{ Address: &address, Timeout: &timeout, @@ -318,9 +336,12 @@ func TestHTTPSinkHeadersAndCompression(t *testing.T) { }, Compression: &logconfig.GzipCompression, - Headers: map[string]string{"X-CRDB-TEST": "secret-value"}, + // Provide both the old format and new format in order to test backwards compatability. + Headers: map[string]string{"X-CRDB-TEST": val}, + FileBasedHeaders: map[string]string{"X-CRDB-TEST-2": filename}, } + var callCt int testFn := func(header http.Header, body string) error { t.Log(body) contentType := header.Get("Content-Type") @@ -340,17 +361,36 @@ func TestHTTPSinkHeadersAndCompression(t *testing.T) { if !isGzipped([]byte(body)) { return errors.New("expected gzipped body") } + var matchCount int + filepathExpectedVal := filepathVal + if callCt > 0 { + filepathExpectedVal = filepathReplaceVal + } for k, v := range header { - if k == "X-Crdb-Test" { + if k == "X-Crdb-Test" || k == "X-Crdb-Test-2" { for _, vv := range v { - if vv == "secret-value" { - return nil + if vv == "secret-value" || vv == filepathExpectedVal { + matchCount++ } } } } - return errors.New("expected to find special header in request") + if matchCount != 2 { + return errors.New("expected to find special header in request") + } + // If this is the first time the testFn has been called, update file contents and send SIGHUP. + if callCt == 0 { + callCt++ + if err := os.WriteFile(filename, []byte(filepathReplaceVal), 0777); err != nil { + return err + } + t.Log("issuing SIGHUP") + if err := unix.Kill(unix.Getpid(), unix.SIGHUP); err != nil { + t.Fatal(err) + } + } + return nil } - testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0)) + testBase(t, defaults, testFn, false /* hangServer */, time.Duration(0), 1*time.Second) } diff --git a/pkg/util/log/log_flush.go b/pkg/util/log/log_flush.go index 5bf42fc18714..b87969c96f89 100644 --- a/pkg/util/log/log_flush.go +++ b/pkg/util/log/log_flush.go @@ -133,12 +133,17 @@ func flushDaemon() { } } -// signalFlusher flushes the log(s) every time SIGHUP is received. +// signalFlusher updates any header values from files in the http sinks +// and also flushes the log(s) every time SIGHUP is received. // This handles both the primary and secondary loggers. func signalFlusher() { ch := sysutil.RefreshSignaledChan() for sig := range ch { Ops.Infof(context.Background(), "%s received, flushing logs", sig) + err := RefreshHttpSinkHeaders() + if err != nil { + Ops.Infof(context.Background(), "error while refreshing http sink headers: %s", err) + } FlushFiles() } } @@ -153,3 +158,11 @@ func StartAlwaysFlush() { // There may be something in the buffers already; flush it. FlushFiles() } + +// RefreshHttpSinkHeaders will iterate over all http sinks and replace the sink's +// dynamicHeaders with newly generated dynamicHeaders. +func RefreshHttpSinkHeaders() error { + return logging.allSinkInfos.iterHttpSinks(func(hs *httpSink) error { + return hs.RefreshDynamicHeaders() + }) +} diff --git a/pkg/util/log/logconfig/config.go b/pkg/util/log/logconfig/config.go index 1bb961912336..0a2a6fbd242d 100644 --- a/pkg/util/log/logconfig/config.go +++ b/pkg/util/log/logconfig/config.go @@ -513,6 +513,10 @@ type HTTPDefaults struct { // Headers is a list of headers to attach to each HTTP request Headers map[string]string `yaml:",omitempty,flow"` + // FileBasedHeaders is a list of headers with filepaths whose contents are + // attached to each HTTP request + FileBasedHeaders map[string]string `yaml:"file-based-headers,omitempty,flow"` + // Compression can be "none" or "gzip" to enable gzip compression. // Set to "gzip" by default. Compression *string `yaml:",omitempty"` diff --git a/pkg/util/log/logconfig/testdata/validate b/pkg/util/log/logconfig/testdata/validate index 69b710e37db6..b03266962ed5 100644 --- a/pkg/util/log/logconfig/testdata/validate +++ b/pkg/util/log/logconfig/testdata/validate @@ -598,12 +598,14 @@ sinks: address: a channels: STORAGE headers: {X-CRDB-HEADER: header-value-a} + file-based-headers: {X-CRDB-FILE-HEADER: /a/path/to/file} buffering: max-staleness: 10s b: address: b channels: OPS headers: {X-CRDB-HEADER: header-value-b, X-ANOTHER-HEADER: zz-yy-bb} + file-based-headers: {X-ANOTHER-FILE-HEADER: /other/path/to/file, X-CRDB-FILE-HEADER: /some/path/to/file} buffering: flush-trigger-size: 5.0KiB c: @@ -630,6 +632,7 @@ sinks: timeout: 2s disable-keep-alives: false headers: {X-CRDB-HEADER: header-value-a} + file-based-headers: {X-CRDB-FILE-HEADER: /a/path/to/file} compression: gzip filter: INFO format: json-compact @@ -650,6 +653,7 @@ sinks: timeout: 2s disable-keep-alives: false headers: {X-ANOTHER-HEADER: zz-yy-bb, X-CRDB-HEADER: header-value-b} + file-based-headers: {X-ANOTHER-FILE-HEADER: /other/path/to/file, X-CRDB-FILE-HEADER: /some/path/to/file} compression: gzip filter: INFO format: json-compact diff --git a/pkg/util/log/logconfig/validate.go b/pkg/util/log/logconfig/validate.go index e3e0c6ee3640..636907478a7e 100644 --- a/pkg/util/log/logconfig/validate.go +++ b/pkg/util/log/logconfig/validate.go @@ -461,6 +461,14 @@ func (c *Config) validateHTTPSinkConfig(hsc *HTTPSinkConfig) error { if *hsc.Compression != GzipCompression && *hsc.Compression != NoneCompression { return errors.New("compression must be 'gzip' or 'none'") } + // If both header types are populated, make sure theres no duplicate keys + if hsc.Headers != nil && hsc.FileBasedHeaders != nil { + for key := range hsc.Headers { + if _, exists := hsc.FileBasedHeaders[key]; exists { + return errors.Newf("headers and file-based-headers have the same key %s", key) + } + } + } return c.ValidateCommonSinkConfig(hsc.CommonSinkConfig) } diff --git a/pkg/util/log/registry.go b/pkg/util/log/registry.go index dce3e20bac17..9f43e0c57aad 100644 --- a/pkg/util/log/registry.go +++ b/pkg/util/log/registry.go @@ -102,6 +102,19 @@ func (r *sinkInfoRegistry) iterBufferedSinks(fn func(bs *bufferedSink) error) er }) } +// iterHttpSink iterates over all the http sinks and stops at the first error +// encountered. +func (r *sinkInfoRegistry) iterHttpSinks(fn func(hs *httpSink) error) error { + return r.iter(func(si *sinkInfo) error { + if hs, ok := si.sink.(*httpSink); ok { + if err := fn(hs); err != nil { + return err + } + } + return nil + }) +} + // put adds a sinkInfo into the registry. func (r *sinkInfoRegistry) put(l *sinkInfo) { r.mu.Lock()