Skip to content

Commit 85743ac

Browse files
RaduBerindeRaduBerinde
committed
metrics: improve tables for compaction kinds and files
The compaction kinds table looks funny and is too long. We no longer use the horizontal orientation feature (which has some issues, and unnecessarily makes all columns have the same width); instead, we transpose the table. We also improve the files table to fit all information.
1 parent 12f4568 commit 85743ac

File tree

8 files changed

+270
-244
lines changed

8 files changed

+270
-244
lines changed

‎metrics.go‎

Lines changed: 74 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ package pebble
66

77
import (
88
"fmt"
9+
"iter"
910
"math"
10-
"slices"
1111
"strings"
1212
"time"
1313
"unsafe"
@@ -228,7 +228,6 @@ type Metrics struct {
228228
RewriteCount int64
229229
MultiLevelCount int64
230230
BlobFileRewriteCount int64
231-
CounterLevelCount int64
232231
// An estimate of the number of bytes that need to be compacted for the LSM
233232
// to reach a stable state.
234233
EstimatedDebt uint64
@@ -669,9 +668,19 @@ var (
669668
table.Literal[*LevelMetrics](" "),
670669
table.Bytes("blobsz", 6, table.AlignRight, func(m *LevelMetrics) uint64 { return m.BlobBytesFlushed + m.BlobBytesCompacted }),
671670
)
672-
compactionKindTable = table.Define[pair[string, int64]](
673-
table.String("kind", 9, table.AlignRight, func(p pair[string, int64]) string { return p.k }),
674-
table.Count("count", 9, table.AlignRight, func(p pair[string, int64]) int64 { return p.v }),
671+
compactionKindTable = table.Define[compactionKindsInfo](
672+
table.String("kind", 6, table.AlignRight, func(i compactionKindsInfo) string { return "count" }),
673+
table.Div(),
674+
table.String("default", 8, table.AlignRight, func(i compactionKindsInfo) string { return i.def }),
675+
table.String("delete", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.delete }),
676+
table.String("elision", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.elision }),
677+
table.String("copy", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.copy }),
678+
table.String("move", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.move }),
679+
table.String("read", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.read }),
680+
table.String("tomb", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.tombstone }),
681+
table.String("rewrite", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.rewrite }),
682+
table.String("multi", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.multilevel }),
683+
table.String("blob", 9, table.AlignRight, func(i compactionKindsInfo) string { return i.blob }),
675684
)
676685
commitPipelineInfoTableTopHeader = `COMMIT PIPELINE`
677686
commitPipelineInfoTableSubHeader = ` wals | memtables | ingestions`
@@ -709,25 +718,21 @@ var (
709718
table.Div(),
710719
table.String("open", 11, table.AlignRight, func(i iteratorInfo) string { return i.snapshotsOpen }),
711720
)
712-
fileInfoTableHeader = `FILES tables | blob files | blob values`
721+
fileInfoTableHeader = `FILES tables | blob files | blob values`
713722
fileInfoTable = table.Define[tableAndBlobInfo](
714723
table.String("stats prog", 13, table.AlignRight, func(i tableAndBlobInfo) string { return i.tableInfo.stats }),
715724
table.Div(),
716-
table.String("backing", 9, table.AlignRight, func(i tableAndBlobInfo) string { return i.tableInfo.backing }),
717-
table.Div(),
718-
table.String("zombie", 9, table.AlignRight, func(i tableAndBlobInfo) string { return i.tableInfo.zombie }),
725+
table.String("backing", 10, table.AlignRight, func(i tableAndBlobInfo) string { return i.tableInfo.backing }),
719726
table.Div(),
720-
table.String("loc zomb", 9, table.AlignRight, func(i tableAndBlobInfo) string { return i.tableInfo.localZombie }),
727+
table.String("zombie", 21, table.AlignRight, func(i tableAndBlobInfo) string { return i.tableInfo.zombie }),
721728
table.Div(),
722-
table.String("live", 7, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.live }),
729+
table.String("live", 10, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.live }),
723730
table.Div(),
724-
table.String("zombie", 8, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.zombie }),
731+
table.String("zombie", 10, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.zombie }),
725732
table.Div(),
726733
table.String("total", 6, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.total }),
727734
table.Div(),
728-
table.String("refed", 6, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.referenced }),
729-
table.Div(),
730-
table.String("refed %", 7, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.referencedPercent }),
735+
table.String("refed", 10, table.AlignRight, func(i tableAndBlobInfo) string { return i.blobInfo.referenced }),
731736
)
732737
cgoMemInfoTableHeader = `CGO MEMORY | block cache | memtables`
733738
cgoMemInfoTable = table.Define[cgoMemInfo](
@@ -769,6 +774,19 @@ var (
769774
)
770775
)
771776

777+
type compactionKindsInfo struct {
778+
def string
779+
delete string
780+
elision string
781+
copy string
782+
move string
783+
read string
784+
tombstone string
785+
rewrite string
786+
multilevel string
787+
blob string
788+
}
789+
772790
type commitPipelineInfo struct {
773791
files string
774792
written string
@@ -790,18 +808,16 @@ type iteratorInfo struct {
790808
snapshotsOpen string
791809
}
792810
type tableInfo struct {
793-
stats string
794-
backing string
795-
zombie string
796-
localZombie string
811+
stats string
812+
backing string
813+
zombie string
797814
}
798815

799816
type blobInfo struct {
800-
live string
801-
zombie string
802-
total string
803-
referenced string
804-
referencedPercent string
817+
live string
818+
zombie string
819+
total string
820+
referenced string
805821
}
806822

807823
type tableAndBlobInfo struct {
@@ -834,11 +850,6 @@ type keysInfo struct {
834850
rangeDels string
835851
}
836852

837-
type pair[k, v any] struct {
838-
k k
839-
v v
840-
}
841-
842853
// String pretty-prints the metrics.
843854
//
844855
// See testdata/metrics for an example.
@@ -893,21 +904,23 @@ func (m *Metrics) String() string {
893904
cur = cur.NewlineReturn()
894905
return cur
895906
}
896-
compactionKindContents := []pair[string, int64]{
897-
{k: "default", v: m.Compact.DefaultCount},
898-
{k: "delete", v: m.Compact.DeleteOnlyCount},
899-
{k: "elision", v: m.Compact.ElisionOnlyCount},
900-
{k: "move", v: m.Compact.MoveCount},
901-
{k: "read", v: m.Compact.ReadCount},
902-
{k: "tomb", v: m.Compact.TombstoneDensityCount},
903-
{k: "rewrite", v: m.Compact.RewriteCount},
904-
{k: "copy", v: m.Compact.CopyCount},
905-
{k: "multi", v: m.Compact.MultiLevelCount},
906-
{k: "blob", v: m.Compact.BlobFileRewriteCount},
907+
908+
compactionKindsContents := compactionKindsInfo{
909+
def: humanizeCount(m.Compact.DefaultCount).String(),
910+
delete: humanizeCount(m.Compact.DeleteOnlyCount).String(),
911+
elision: humanizeCount(m.Compact.ElisionOnlyCount).String(),
912+
copy: humanizeCount(m.Compact.CopyCount).String(),
913+
move: humanizeCount(m.Compact.MoveCount).String(),
914+
read: humanizeCount(m.Compact.ReadCount).String(),
915+
tombstone: humanizeCount(m.Compact.TombstoneDensityCount).String(),
916+
rewrite: humanizeCount(m.Compact.RewriteCount).String(),
917+
multilevel: humanizeCount(m.Compact.MultiLevelCount).String(),
918+
blob: humanizeCount(m.Compact.BlobFileRewriteCount).String(),
907919
}
920+
908921
cur = renderTableWithDivider(cur, func(cur ascii.Cursor) ascii.Cursor {
909-
return compactionKindTable.Render(cur, table.RenderOptions{Orientation: table.Horizontally}, slices.Values(compactionKindContents))
910-
}, max(compactionLevelMetricsTable.CumulativeFieldWidth, cur.Column()), -1)
922+
return compactionKindTable.Render(cur, table.RenderOptions{}, oneItemIter(compactionKindsContents))
923+
}, compactionKindTable.CumulativeFieldWidth, -1)
911924

912925
commitPipelineInfoContents := commitPipelineInfo{
913926
// wals.
@@ -922,13 +935,10 @@ func (m *Metrics) String() string {
922935
total: crhumanize.Count(m.WAL.BytesIn + m.WAL.BytesWritten).String(),
923936
flushable: fmt.Sprintf("%s (%s)", humanizeCount(m.Flush.AsIngestCount), humanizeBytes(m.Flush.AsIngestBytes)),
924937
}
925-
commitPipelineInfoIter := func(yield func(commitPipelineInfo) bool) {
926-
yield(commitPipelineInfoContents)
927-
}
928938
cur = cur.WriteString(commitPipelineInfoTableTopHeader).NewlineReturn()
929939
cur = cur.WriteString(commitPipelineInfoTableSubHeader)
930940
cur = renderTableWithDivider(cur, func(cur ascii.Cursor) ascii.Cursor {
931-
return commitPipelineInfoTable.Render(cur, table.RenderOptions{}, commitPipelineInfoIter)
941+
return commitPipelineInfoTable.Render(cur, table.RenderOptions{}, oneItemIter(commitPipelineInfoContents))
932942
}, max(commitPipelineInfoTable.CumulativeFieldWidth, cur.Column()), -3)
933943

934944
iteratorInfoContents := iteratorInfo{
@@ -940,13 +950,10 @@ func (m *Metrics) String() string {
940950
sstableItersOpen: humanizeCount(m.TableIters).String(),
941951
snapshotsOpen: humanizeCount(m.Snapshots.Count).String(),
942952
}
943-
iteratorInfoIter := func(yield func(iteratorInfo) bool) {
944-
yield(iteratorInfoContents)
945-
}
946953
cur = cur.WriteString(iteratorInfoTableTopHeader).NewlineReturn()
947954
cur = cur.WriteString(iteratorInfoTableSubHeader)
948955
cur = renderTableWithDivider(cur, func(cur ascii.Cursor) ascii.Cursor {
949-
return iteratorInfoTable.Render(cur, table.RenderOptions{}, iteratorInfoIter)
956+
return iteratorInfoTable.Render(cur, table.RenderOptions{}, oneItemIter(iteratorInfoContents))
950957
}, max(iteratorInfoTable.CumulativeFieldWidth, cur.Column()), -3)
951958

952959
status := fmt.Sprintf("%s pending", humanizeCount(m.Table.PendingStatsCollectionCount))
@@ -956,28 +963,23 @@ func (m *Metrics) String() string {
956963
status = "all loaded"
957964
}
958965
tableInfoContents := tableInfo{
959-
stats: status,
960-
backing: fmt.Sprintf("%s (%d)", humanizeBytes(m.Table.BackingTableSize), m.Table.BackingTableCount),
961-
zombie: fmt.Sprintf("%s (%d)", humanizeBytes(m.Table.ZombieSize), m.Table.ZombieCount),
962-
localZombie: humanizeBytes(m.Table.Local.ZombieSize).String(),
966+
stats: status,
967+
backing: fmt.Sprintf("%s (%s)", humanizeCount(m.Table.BackingTableCount), humanizeBytes(m.Table.BackingTableSize)),
968+
zombie: fmt.Sprintf("%s (%s local:%s)", humanizeCount(m.Table.ZombieCount), humanizeBytes(m.Table.ZombieSize), humanizeBytes(m.Table.Local.ZombieSize)),
963969
}
964970
blobInfoContents := blobInfo{
965-
live: fmt.Sprintf("%s (%s)", humanizeCount(m.BlobFiles.LiveCount), humanizeBytes(m.BlobFiles.LiveSize)),
966-
zombie: fmt.Sprintf("%s (%s)", humanizeCount(m.BlobFiles.ZombieCount), humanizeBytes(m.BlobFiles.ZombieSize)),
967-
total: humanizeBytes(m.BlobFiles.ValueSize).String(),
968-
referenced: humanizeBytes(m.BlobFiles.ReferencedValueSize).String(),
969-
referencedPercent: fmt.Sprintf("%.0f%%", percent(m.BlobFiles.ReferencedValueSize, m.BlobFiles.ValueSize)),
971+
live: fmt.Sprintf("%s (%s)", humanizeCount(m.BlobFiles.LiveCount), humanizeBytes(m.BlobFiles.LiveSize)),
972+
zombie: fmt.Sprintf("%s (%s)", humanizeCount(m.BlobFiles.ZombieCount), humanizeBytes(m.BlobFiles.ZombieSize)),
973+
total: humanizeBytes(m.BlobFiles.ValueSize).String(),
974+
referenced: fmt.Sprintf("%.0f%% (%s)", percent(m.BlobFiles.ReferencedValueSize, m.BlobFiles.ValueSize), humanizeBytes(m.BlobFiles.ReferencedValueSize)),
970975
}
971976
fileInfoContents := tableAndBlobInfo{
972977
tableInfo: tableInfoContents,
973978
blobInfo: blobInfoContents,
974979
}
975-
fileInfoIter := func(yield func(tableAndBlobInfo) bool) {
976-
yield(fileInfoContents)
977-
}
978980
cur = cur.WriteString(fileInfoTableHeader)
979981
cur = renderTableWithDivider(cur, func(cur ascii.Cursor) ascii.Cursor {
980-
return fileInfoTable.Render(cur, table.RenderOptions{}, fileInfoIter)
982+
return fileInfoTable.Render(cur, table.RenderOptions{}, oneItemIter(fileInfoContents))
981983
}, max(fileInfoTable.CumulativeFieldWidth, cur.Column()), -2)
982984

983985
var inUseTotal uint64
@@ -996,12 +998,9 @@ func (m *Metrics) String() string {
996998
bcEnts: humanizeBytes(inUse(manual.BlockCacheEntry)).String(),
997999
memtablesTot: humanizeBytes(inUse(manual.MemTable)).String(),
9981000
}
999-
cgoMemInfoIter := func(yield func(cgoMemInfo) bool) {
1000-
yield(cgoMemInfoContents)
1001-
}
10021001
cur = cur.WriteString(cgoMemInfoTableHeader)
10031002
cur = renderTableWithDivider(cur, func(cur ascii.Cursor) ascii.Cursor {
1004-
return cgoMemInfoTable.Render(cur, table.RenderOptions{}, cgoMemInfoIter)
1003+
return cgoMemInfoTable.Render(cur, table.RenderOptions{}, oneItemIter(cgoMemInfoContents))
10051004
}, max(cgoMemInfoTable.CumulativeFieldWidth, cur.Column()), -2)
10061005

10071006
compactionMetricsInfoContents := compactionMetricsInfo{
@@ -1013,12 +1012,9 @@ func (m *Metrics) String() string {
10131012
failed: m.Compact.FailedCount,
10141013
problemSpans: fmt.Sprintf("%d%s", m.Compact.NumProblemSpans, ifNonZero(m.Compact.NumProblemSpans, "!!")),
10151014
}
1016-
compactionMetricsInfoIter := func(yield func(compactionMetricsInfo) bool) {
1017-
yield(compactionMetricsInfoContents)
1018-
}
10191015
cur = cur.WriteString(compactionInfoTableTopHeader)
10201016
cur = renderTableWithDivider(cur, func(cur ascii.Cursor) ascii.Cursor {
1021-
return compactionInfoTable.Render(cur, table.RenderOptions{}, compactionMetricsInfoIter)
1017+
return compactionInfoTable.Render(cur, table.RenderOptions{}, oneItemIter(compactionMetricsInfoContents))
10221018
}, max(compactionInfoTable.CumulativeFieldWidth, cur.Column()), -2)
10231019

10241020
keysInfoContents := keysInfo{
@@ -1028,12 +1024,9 @@ func (m *Metrics) String() string {
10281024
pointDels: humanizeBytes(m.Table.Garbage.PointDeletionsBytesEstimate).String(),
10291025
rangeDels: humanizeBytes(m.Table.Garbage.RangeDeletionsBytesEstimate).String(),
10301026
}
1031-
keysInfoIter := func(yield func(keysInfo) bool) {
1032-
yield(keysInfoContents)
1033-
}
10341027
cur = cur.WriteString(keysInfoTableTopHeader)
10351028
cur = renderTableWithDivider(cur, func(cur ascii.Cursor) ascii.Cursor {
1036-
return keysInfoTable.Render(cur, table.RenderOptions{}, keysInfoIter)
1029+
return keysInfoTable.Render(cur, table.RenderOptions{}, oneItemIter(keysInfoContents))
10371030
}, max(keysInfoTable.CumulativeFieldWidth, cur.Column()), -2)
10381031
cur.Offset(-1, 0).RepeatByte(max(keysInfoTable.CumulativeFieldWidth, cur.Column()), '-')
10391032

@@ -1125,3 +1118,9 @@ func humanizeCount[T crhumanize.Integer](value T) crhumanize.SafeString {
11251118
func humanizeBytes[T crhumanize.Integer](value T) crhumanize.SafeString {
11261119
return crhumanize.Bytes(value, crhumanize.Compact, crhumanize.OmitI)
11271120
}
1121+
1122+
func oneItemIter[T any](v T) iter.Seq[T] {
1123+
return func(yield func(T) bool) {
1124+
yield(v)
1125+
}
1126+
}

‎metrics_test.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ func exampleMetrics() Metrics {
5353
m.Compact.RewriteCount = 17
5454
m.Compact.MultiLevelCount = 18
5555
m.Compact.BlobFileRewriteCount = 19
56-
m.Compact.CounterLevelCount = 20
5756
m.Compact.EstimatedDebt = 6 * GB
5857
m.Compact.InProgressBytes = 1 * MB
5958
m.Compact.NumInProgress = 2

‎testdata/compaction/l0_to_lbase_compaction‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,9 @@ level | score ff cff | tables size | top in read | tables blob | ta
5454
5 | 0 0 0 | 0 0B | 0B 0B 0B | 0B 0B | 0 0B 0B
5555
total | 0 0.09 0.09 | 3 6MB | 0B 0B 0B | 0B 0B | 0 0B 0B
5656
--------------------------------------------------------------------------------------------------
57-
kind | default delete elision move read tomb rewrite copy multi blob
58-
count | 0 0 0 3 0 0 0 0 0 0
57+
kind | default delete elision copy move read tomb rewrite multi blob
58+
-------+------------------------------------------------------------------------------------------
59+
count | 0 0 0 0 3 0 0 0 0 0
5960
--------------------------------------------------------------------------------------------------
6061
COMMIT PIPELINE
6162
wals | memtables | ingestions
@@ -69,10 +70,10 @@ ITERATORS
6970
-------------+-------------+--------------+-------------+--------------+-------------+------------
7071
1.5K (6.4MB) | 60.0% | 3 (840B) | 90.0% | 0.0% | 0 | 0
7172
--------------------------------------------------------------------------------------------------
72-
FILES tables | blob files | blob values
73-
stats prog | backing | zombie | loc zomb | live | zombie | total | refed | refed %
74-
--------------+-----------+-----------+-----------+---------+----------+--------+--------+--------
75-
all loaded | 0B (0) | 0B (0) | 0B | 0 (0B) | 0 (0B) | 0B | 0B | 0%
73+
FILES tables | blob files | blob values
74+
stats prog | backing | zombie | live | zombie | total | refed
75+
--------------+------------+-----------------------+------------+------------+--------+-----------
76+
all loaded | 0 (0B) | 0 (0B local:0B) | 0 (0B) | 0 (0B) | 0B | 0% (0B)
7677
--------------------------------------------------------------------------------------------------
7778
CGO MEMORY | block cache | memtables
7879
tot | tot | data | maps | ents | tot

0 commit comments

Comments
 (0)