db: add SpanPolicy flag to prefer faster compression

For the same reasons we don't want value separation for some of the
local keys, we want to use the faster compression since these keys
shouldn't take up a significant amount of space anyway.

More generally, we probably want to use the fastest compression for
all local keys.

Author: RaduBerinde

compaction.go

@@ -3304,6 +3304,9 @@ func (d *DB) compactAndWrite(
 		if spanPolicy.DisableValueSeparationBySuffix {
 			writerOpts.DisableValueBlocks = true
 		}
+		if spanPolicy.PreferFastCompression && writerOpts.Compression != block.NoCompression {
+			writerOpts.Compression = block.FastestCompression
+		}
 		vSep := valueSeparation
 		if spanPolicy.ValueStoragePolicy == ValueStorageLowReadLatency {
 			vSep = compact.NeverSeparateValues{}

options.go

@@ -1186,6 +1186,12 @@ type ValueSeparationPolicy struct {
 // SpanPolicy contains policies that can vary by key range. The zero value is
 // the default value.
 type SpanPolicy struct {
+	// Prefer a faster compression algorithm for the keys in this span.
+	//
+	// This is useful for keys that are frequently read or written but which don't
+	// amount to a significant amount of space.
+	PreferFastCompression bool
+
 	// DisableValueSeparationBySuffix disables discriminating KVs depending on
 	// suffix.
 	//