backupccl: add exportrequest.delay.total metric

This commit adds a metric to track how long export requests are blocked
by the concurrency limiter.

Release note (ops change): Introduce a new metric
(exportrequest.delay.total) to track how long ExportRequests (issued by
BACKUP) are delayed by throttling mechansisms.

pkg/kv/kvserver/metrics.go

@@ -1051,6 +1051,14 @@ var (
 		Unit:        metric.Unit_NANOSECONDS,
 	}
 
+	// Export request counter.
+	metaExportEvalTotalDelay = metric.Metadata{
+		Name:        "exportrequest.delay.total",
+		Help:        "Amount by which evaluation of Export requests was delayed",
+		Measurement: "Nanoseconds",
+		Unit:        metric.Unit_NANOSECONDS,
+	}
+
 	// Encryption-at-rest metrics.
 	// TODO(mberhault): metrics for key age, per-key file/bytes counts.
 	metaEncryptionAlgorithm = metric.Metadata{
@@ -1282,6 +1290,9 @@ type StoreMetrics struct {
 	AddSSTableProposalTotalDelay  *metric.Counter
 	AddSSTableProposalEngineDelay *metric.Counter
 
+	// Export request stats.
+	ExportRequestProposalTotalDelay *metric.Counter
+
 	// Encryption-at-rest stats.
 	// EncryptionAlgorithm is an enum representing the cipher in use, so we use a gauge.
 	EncryptionAlgorithm *metric.Gauge
@@ -1672,6 +1683,9 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		AddSSTableProposalTotalDelay:  metric.NewCounter(metaAddSSTableEvalTotalDelay),
 		AddSSTableProposalEngineDelay: metric.NewCounter(metaAddSSTableEvalEngineDelay),
 
+		// ExportRequest proposal.
+		ExportRequestProposalTotalDelay: metric.NewCounter(metaExportEvalTotalDelay),
+
 		// Encryption-at-rest.
 		EncryptionAlgorithm: metric.NewGauge(metaEncryptionAlgorithm),
 

pkg/kv/kvserver/store_send.go

@@ -299,6 +299,7 @@ func (s *Store) maybeThrottleBatch(
 		}
 
 		waited := timeutil.Since(before)
+		s.metrics.ExportRequestProposalTotalDelay.Inc(waited.Nanoseconds())
 		if waited > time.Second {
 			log.Infof(ctx, "Export request was delayed by %v", waited)
 		}

pkg/ts/catalog/chart_catalog.go

@@ -2305,6 +2305,17 @@ var charts = []sectionDescription{
 			},
 		},
 	},
+	{
+		Organization: [][]string{{DistributionLayer, "Bulk", "Egress"}},
+		Charts: []chartDescription{
+			{
+				Title: "Export Delays",
+				Metrics: []string{
+					"exportrequest.delay.total",
+				},
+			},
+		},
+	},
 	{
 		Organization: [][]string{{StorageLayer, "Storage", "KV"}},
 		Charts: []chartDescription{