db: add some tests for compression stats

Author: RaduBerinde

data_test.go

@@ -1253,6 +1253,10 @@ func runTableStatsCmd(td *datadriven.TestData, d *DB) string {
 				fmt.Fprintf(&b, "tombstone-dense-blocks-ratio: %0.1f\n", f.Stats.TombstoneDenseBlocksRatio)
 			}
 
+			if !f.Stats.CompressionStats.IsEmpty() {
+				fmt.Fprintf(&b, "compression: %s\n", f.Stats.CompressionStats.String())
+			}
+
 			return b.String()
 		}
 	}
@@ -1801,30 +1805,34 @@ func parseDBOptionsArgs(opts *Options, args []datadriven.CmdArg) error {
 				opts.TargetFileSizes[i] = opts.TargetFileSizes[i-1] * 2
 			}
 		case "value-separation":
-			if len(cmdArg.Vals) != 5 {
-				return errors.New("value-separation expects 5 arguments: (enabled, minimum-size, max-blob-reference-depth, rewrite-minimum-age, target-garbage-ratio)")
-			}
 			var policy ValueSeparationPolicy
-			var err error
-			policy.Enabled, err = strconv.ParseBool(cmdArg.Vals[0])
-			if err != nil {
-				return err
-			}
-			policy.MinimumSize, err = strconv.Atoi(cmdArg.Vals[1])
-			if err != nil {
-				return err
-			}
-			policy.MaxBlobReferenceDepth, err = strconv.Atoi(cmdArg.Vals[2])
-			if err != nil {
-				return err
-			}
-			policy.RewriteMinimumAge, err = time.ParseDuration(cmdArg.Vals[3])
-			if err != nil {
-				return err
-			}
-			policy.TargetGarbageRatio, err = strconv.ParseFloat(cmdArg.Vals[4], 64)
-			if err != nil {
-				return err
+			if len(cmdArg.Vals) == 1 && cmdArg.Vals[0] == "off" || cmdArg.Vals[0] == "disabled" {
+				policy.Enabled = false
+			} else {
+				if len(cmdArg.Vals) != 5 {
+					return errors.New("value-separation expects 5 arguments: (enabled, minimum-size, max-blob-reference-depth, rewrite-minimum-age, target-garbage-ratio)")
+				}
+				var err error
+				policy.Enabled, err = strconv.ParseBool(cmdArg.Vals[0])
+				if err != nil {
+					return err
+				}
+				policy.MinimumSize, err = strconv.Atoi(cmdArg.Vals[1])
+				if err != nil {
+					return err
+				}
+				policy.MaxBlobReferenceDepth, err = strconv.Atoi(cmdArg.Vals[2])
+				if err != nil {
+					return err
+				}
+				policy.RewriteMinimumAge, err = time.ParseDuration(cmdArg.Vals[3])
+				if err != nil {
+					return err
+				}
+				policy.TargetGarbageRatio, err = strconv.ParseFloat(cmdArg.Vals[4], 64)
+				if err != nil {
+					return err
+				}
 			}
 			opts.Experimental.ValueSeparationPolicy = func() ValueSeparationPolicy {
 				return policy

testdata/compaction/set_with_del_sstable_Pebblev5

@@ -89,6 +89,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1536
+compression: None:79
 
 compact a-e L1
 ----
@@ -107,6 +108,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 768
+compression: None:87,Snappy:76/87
 
 # Same as above, except range tombstone covers multiple grandparent file boundaries.
 

testdata/compaction/set_with_del_sstable_Pebblev6

@@ -89,6 +89,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1570
+compression: None:79
 
 compact a-e L1
 ----
@@ -107,6 +108,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 785
+compression: None:87,Snappy:76/87
 
 # Same as above, except range tombstone covers multiple grandparent file boundaries.
 

testdata/compaction/set_with_del_sstable_Pebblev7

@@ -89,6 +89,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 1518
+compression: None:79
 
 compact a-e L1
 ----
@@ -107,6 +108,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 759
+compression: None:87,Snappy:76/87
 
 # Same as above, except range tombstone covers multiple grandparent file boundaries.
 

testdata/compaction_picker_scores

@@ -38,6 +38,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 328899
+compression: None:79
 
 scores
 ----
@@ -94,6 +95,7 @@ num-deletions: 5
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 163850
 range-deletions-bytes-estimate: 0
+compression: None:128
 
 scores
 ----
@@ -147,6 +149,7 @@ num-deletions: 5
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 163860
 range-deletions-bytes-estimate: 0
+compression: None:129
 
 maybe-compact
 ----

testdata/compaction_tombstones

@@ -16,6 +16,7 @@ num-deletions: 2
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
+compression: None:85
 
 maybe-compact
 ----
@@ -38,6 +39,7 @@ num-deletions: 2
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
+compression: None:85
 
 maybe-compact
 ----
@@ -59,6 +61,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
+compression: None:87,Snappy:76/87
 
 maybe-compact
 ----
@@ -82,6 +85,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 2
 range-deletions-bytes-estimate: 0
+compression: None:124
 
 maybe-compact
 ----
@@ -121,6 +125,7 @@ num-deletions: 2
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 2
 range-deletions-bytes-estimate: 101
+compression: None:87,Snappy:96/129
 
 maybe-compact
 ----
@@ -154,6 +159,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 2
 range-deletions-bytes-estimate: 0
+compression: None:36,Snappy:131/169
 
 close-snapshot
 15
@@ -198,6 +204,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 16824
+compression: None:16929
 
 # Because we set max bytes low, maybe-compact will trigger an automatic
 # compaction in preference over an elision-only compaction.
@@ -235,6 +242,7 @@ num-deletions: 3
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 6150
 range-deletions-bytes-estimate: 0
+compression: None:128
 
 # By plain file size, 000005 should be picked because it is larger and
 # overlaps the same amount of data in L6. However, 000004 has a high
@@ -269,6 +277,7 @@ num-deletions: 0
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
+compression: None:79
 
 wait-pending-table-stats
 000005
@@ -278,6 +287,7 @@ num-deletions: 0
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
+compression: None:83
 
 wait-pending-table-stats
 000006
@@ -287,6 +297,7 @@ num-deletions: 0
 num-range-key-sets: 1
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 0
+compression: None:83
 
 maybe-compact
 ----
@@ -320,6 +331,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 94
+compression: None:227
 
 maybe-compact
 ----
@@ -361,6 +373,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 2459
 range-deletions-bytes-estimate: 0
+compression: None:120
 
 wait-pending-table-stats
 000005
@@ -370,6 +383,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 8380
+compression: None:87,Snappy:73/84
 
 # With multiple compactions, there is non-determinism in the output table
 # numbers, so the test overwrites them to 0.
@@ -407,6 +421,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 2459
 range-deletions-bytes-estimate: 0
+compression: None:120
 
 wait-pending-table-stats
 000005
@@ -416,6 +431,7 @@ num-deletions: 1
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 0
 range-deletions-bytes-estimate: 8380
+compression: None:87,Snappy:73/84
 
 # With multiple compactions, there is non-determinism in the output table
 # numbers, so the test overwrites them to 0.
@@ -478,6 +494,7 @@ num-deletions: 3
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 7
 range-deletions-bytes-estimate: 0
+compression: None:36,Snappy:95/108
 
 # Force a high tombstone density ratio to trigger the compaction
 # In a real scenario, this would be calculated based on the actual
@@ -491,6 +508,7 @@ num-range-key-sets: 0
 point-deletions-bytes-estimate: 7
 range-deletions-bytes-estimate: 0
 tombstone-dense-blocks-ratio: 0.9
+compression: None:36,Snappy:95/108
 
 # Now the compaction should be triggered with tombstone-density type
 # since the file has a high tombstone density. The compaction log
@@ -540,6 +558,7 @@ num-deletions: 3
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 7
 range-deletions-bytes-estimate: 0
+compression: None:36,Snappy:95/108
 
 # Force a high tombstone density ratio to trigger the compaction
 wait-pending-table-stats force-tombstone-density-ratio=0.9
@@ -551,6 +570,7 @@ num-range-key-sets: 0
 point-deletions-bytes-estimate: 7
 range-deletions-bytes-estimate: 0
 tombstone-dense-blocks-ratio: 0.9
+compression: None:36,Snappy:95/108
 
 # A regular tombstone density compaction should be triggered (not a move optimization)
 # because there are overlapping files in L5 that prevent the optimization
@@ -599,6 +619,7 @@ num-deletions: 3
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 857149
 range-deletions-bytes-estimate: 0
+compression: None:36,Snappy:95/108
 
 # Force a high tombstone density ratio to trigger the compaction
 wait-pending-table-stats force-tombstone-density-ratio=0.9
@@ -610,6 +631,7 @@ num-range-key-sets: 0
 point-deletions-bytes-estimate: 857149
 range-deletions-bytes-estimate: 0
 tombstone-dense-blocks-ratio: 0.9
+compression: None:36,Snappy:95/108
 
 # No compaction is triggered because the overlapping bytes in L6 exceed MaxOverlapBytes.
 # Pebble avoids triggering a compaction in this case to prevent excessive overlap in the

testdata/table_stats

@@ -980,3 +980,69 @@ num-deletions: 3
 num-range-key-sets: 0
 point-deletions-bytes-estimate: 3942
 range-deletions-bytes-estimate: 0
+compression: None:36,Snappy:73/84
+
+define format-major-version=25 block-size=100 value-separation=off
+----
+
+batch
+set a <rand-bytes=4096>
+set b <rand-bytes=4096>
+set c <rand-bytes=4096>
+set d <rand-bytes=4096>
+set e <rand-bytes=4096>
+----
+
+flush
+----
+L0.0:
+  000005:[a#10,SET-e#14,SET]
+
+compact a-c
+----
+L6:
+  000005:[a#10,SET-e#14,SET]
+
+wait-pending-table-stats
+000005
+----
+num-entries: 5
+num-deletions: 0
+num-range-key-sets: 0
+point-deletions-bytes-estimate: 0
+range-deletions-bytes-estimate: 0
+compression: None:20986
+
+build dummy
+set c1 c1
+----
+
+ingest-and-excise dummy excise=b-e
+----
+
+lsm
+----
+L6:
+  000007(000005):[a#10,SET-a#10,SET]
+  000006:[c1#16,SET-c1#16,SET]
+  000008(000005):[e#14,SET-e#14,SET]
+
+wait-pending-table-stats
+000007
+----
+num-entries: 1
+num-deletions: 0
+num-range-key-sets: 0
+point-deletions-bytes-estimate: 0
+range-deletions-bytes-estimate: 0
+compression: None:20986
+
+wait-pending-table-stats
+000008
+----
+num-entries: 1
+num-deletions: 0
+num-range-key-sets: 0
+point-deletions-bytes-estimate: 0
+range-deletions-bytes-estimate: 0
+compression: None:20986