db: add metrics for problem spans, failed compactions

Author: RaduBerinde

compaction.go

@@ -1589,7 +1589,7 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) {
 
 	d.clearCompactingState(c, err != nil)
 	delete(d.mu.compact.inProgress, c)
-	d.mu.versions.incrementCompactions(c.kind, c.extraLevels, c.pickerMetrics)
+	d.mu.versions.incrementCompactions(c.kind, c.extraLevels, c.pickerMetrics, c.bytesWritten, err)
 
 	var flushed flushableList
 	if err == nil {
@@ -2492,9 +2492,6 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) {
 			// the manifest lock, we don't expect this bool to change its value
 			// as only the holder of the manifest lock will ever write to it.
 			if c.cancel.Load() {
-				d.mu.versions.metrics.Compact.CancelledCount++
-				d.mu.versions.metrics.Compact.CancelledBytes += c.bytesWritten
-
 				err = firstError(err, ErrCancelledCompaction)
 				// This is the first time we've seen a cancellation during the
 				// life of this compaction (or the original condition on err == nil
@@ -2530,11 +2527,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) {
 	// NB: clearing compacting state must occur before updating the read state;
 	// L0Sublevels initialization depends on it.
 	d.clearCompactingState(c, err != nil)
-	if err != nil && errors.Is(err, ErrCancelledCompaction) {
-		d.mu.versions.metrics.Compact.CancelledCount++
-		d.mu.versions.metrics.Compact.CancelledBytes += c.bytesWritten
-	}
-	d.mu.versions.incrementCompactions(c.kind, c.extraLevels, c.pickerMetrics)
+	d.mu.versions.incrementCompactions(c.kind, c.extraLevels, c.pickerMetrics, c.bytesWritten, err)
 	d.mu.versions.incrementCompactionBytes(-c.bytesWritten)
 
 	info.TotalDuration = d.timeNow().Sub(c.beganAt)

db.go

@@ -2016,6 +2016,7 @@ func (d *DB) Metrics() *Metrics {
 			metrics.Compact.Duration += d.timeNow().Sub(c.beganAt)
 		}
 	}
+	metrics.Compact.NumProblemSpans = d.problemSpans.Len()
 
 	for _, m := range d.mu.mem.queue {
 		metrics.MemTable.Size += m.totalBytes()

internal/problemspans/by_level.go

@@ -96,6 +96,22 @@ func (bl *ByLevel) Excise(bounds base.UserKeyBounds) {
 	}
 }
 
+// Len returns the number of non-overlapping spans that have not expired. Two
+// spans that touch are both counted if they have different expiration times.
+func (bl *ByLevel) Len() int {
+	if bl.empty.Load() {
+		// Fast path.
+		return 0
+	}
+	bl.mu.Lock()
+	defer bl.mu.Unlock()
+	n := 0
+	for i := range bl.levels {
+		n += bl.levels[i].Len()
+	}
+	return n
+}
+
 // String prints all active (non-expired) span fragments.
 func (bl *ByLevel) String() string {
 	bl.mu.Lock()

internal/problemspans/set.go

@@ -101,6 +101,18 @@ func (s *Set) IsEmpty() bool {
 	return s.rt.IsEmpty()
 }
 
+// Len returns the number of non-overlapping spans that have not expired. Two
+// spans that touch are both counted if they have different expiration times.
+func (s *Set) Len() int {
+	s.now = s.nowFn()
+	n := 0
+	s.rt.EnumerateAll(func(start, end axisds.Endpoint[[]byte], prop expirationTime) bool {
+		n++
+		return true
+	})
+	return n
+}
+
 // String prints all active (non-expired) span fragments.
 func (s *Set) String() string {
 	var buf strings.Builder

metrics.go

@@ -197,6 +197,11 @@ type Metrics struct {
 		// CancelledBytes the number of bytes written by compactions that were
 		// cancelled.
 		CancelledBytes int64
+		// Total number of compactions that hit an error.
+		FailedCount int64
+		// NumProblemSpans is the current (instantaneous) count of "problem spans"
+		// which temporarily block compactions.
+		NumProblemSpans int
 		// MarkedFiles is a count of files that are marked for
 		// compaction. Such files are compacted in a rewrite compaction
 		// when no other compactions are picked.
@@ -687,11 +692,16 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 
 	w.Printf("Flushes: %d\n", redact.Safe(m.Flush.Count))
 
-	w.Printf("Compactions: %d  estimated debt: %s  in progress: %d (%s)\n",
+	w.Printf("Compactions: %d  estimated debt: %s  in progress: %d (%s)  canceled: %d (%s)  failed: %d  problem spans: %d\n",
 		redact.Safe(m.Compact.Count),
 		humanize.Bytes.Uint64(m.Compact.EstimatedDebt),
 		redact.Safe(m.Compact.NumInProgress),
-		humanize.Bytes.Int64(m.Compact.InProgressBytes))
+		humanize.Bytes.Int64(m.Compact.InProgressBytes),
+		redact.Safe(m.Compact.CancelledCount),
+		humanize.Bytes.Int64(m.Compact.CancelledBytes),
+		redact.Safe(m.Compact.FailedCount),
+		redact.Safe(m.Compact.NumProblemSpans),
+	)
 
 	w.Printf("             default: %d  delete: %d  elision: %d  move: %d  read: %d  tombstone-density: %d  rewrite: %d  copy: %d  multi-level: %d\n",
 		redact.Safe(m.Compact.DefaultCount),
@@ -702,7 +712,8 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 		redact.Safe(m.Compact.TombstoneDensityCount),
 		redact.Safe(m.Compact.RewriteCount),
 		redact.Safe(m.Compact.CopyCount),
-		redact.Safe(m.Compact.MultiLevelCount))
+		redact.Safe(m.Compact.MultiLevelCount),
+	)
 
 	w.Printf("MemTables: %d (%s)  zombie: %d (%s)\n",
 		redact.Safe(m.MemTable.Count),

metrics_test.go

@@ -15,9 +15,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cockroachdb/crlib/crstrings"
 	"github.com/cockroachdb/datadriven"
+	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/cache"
 	"github.com/cockroachdb/pebble/internal/humanize"
+	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/internal/manual"
 	"github.com/cockroachdb/pebble/internal/testkeys"
 	"github.com/cockroachdb/pebble/objstorage/remote"
@@ -47,6 +50,11 @@ func exampleMetrics() Metrics {
 	m.Compact.EstimatedDebt = 6
 	m.Compact.InProgressBytes = 7
 	m.Compact.NumInProgress = 2
+	m.Compact.CounterLevelCount = 10
+	m.Compact.CancelledCount = 3
+	m.Compact.CancelledBytes = 3 * 1024
+	m.Compact.FailedCount = 5
+	m.Compact.NumProblemSpans = 2
 	m.Flush.Count = 8
 	m.Flush.AsIngestBytes = 34
 	m.Flush.AsIngestTableCount = 35
@@ -440,6 +448,22 @@ func TestMetrics(t *testing.T) {
 			}
 			return b.String()
 
+		case "problem-spans":
+			d.mu.Lock()
+			defer d.mu.Unlock()
+			d.problemSpans.Init(manifest.NumLevels, d.cmp)
+			for _, line := range crstrings.Lines(td.Input) {
+				var level int
+				var span1, span2 string
+				n, err := fmt.Sscanf(line, "L%d %s %s", &level, &span1, &span2)
+				if err != nil || n != 3 {
+					td.Fatalf(t, "malformed problem span %q", line)
+				}
+				bounds := base.ParseUserKeyBounds(span1 + " " + span2)
+				d.problemSpans.Add(level, bounds, time.Hour*10)
+			}
+			return ""
+
 		default:
 			return fmt.Sprintf("unknown command: %s", td.Cmd)
 		}

testdata/event_listener

@@ -239,7 +239,7 @@ total |     3  2.2KB     0B       0 |    -    -    - |  843B |     1   746B |
 ----------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 48B  written: 97B (102% overhead)
 Flushes: 3
-Compactions: 1  estimated debt: 2.2KB  in progress: 0 (0B)
+Compactions: 1  estimated debt: 2.2KB  in progress: 0 (0B)  canceled: 0 (0B)  failed: 0  problem spans: 0
              default: 1  delete: 0  elision: 0  move: 0  read: 0  tombstone-density: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B, local: 0B)
@@ -342,7 +342,7 @@ total |     6  4.4KB     0B       0 |    -    -    - | 2.3KB |     3  2.2KB |
 ----------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 82B  written: 132B (61% overhead)
 Flushes: 6
-Compactions: 1  estimated debt: 4.4KB  in progress: 0 (0B)
+Compactions: 1  estimated debt: 4.4KB  in progress: 0 (0B)  canceled: 0 (0B)  failed: 0  problem spans: 0
              default: 1  delete: 0  elision: 0  move: 0  read: 0  tombstone-density: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (512KB)  zombie: 1 (512KB)
 Zombie tables: 0 (0B, local: 0B)

testdata/ingest

@@ -45,7 +45,7 @@ total |     1   569B     0B       0 |    -    -    - |  569B |     1   569B |
 ----------------------------------------------------------------------------------------------------------------------------
 WAL: 1 files (0B)  in: 0B  written: 0B (0% overhead)
 Flushes: 0
-Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
+Compactions: 0  estimated debt: 0B  in progress: 0 (0B)  canceled: 0 (0B)  failed: 0  problem spans: 0
              default: 0  delete: 0  elision: 0  move: 0  read: 0  tombstone-density: 0  rewrite: 0  copy: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B, local: 0B)