manifest: explicitly encode excise operations in the MANIFEST

This adds a new struct field that maps excise span to the sequence number
at which it was performed, this will help in benchmarking of replay package as
it was not updated with excise operations, this field is implemented in
version_edit_apply and version_edit_decode tests.

Addresses #4798

Author: Sachuman

compaction.go

@@ -1588,6 +1588,11 @@ func (d *DB) runIngestFlush(c *tableCompaction) (*manifest.VersionEdit, error) {
 	}
 	if ingestFlushable.exciseSpan.Valid() {
 		exciseBounds := ingestFlushable.exciseSpan.UserKeyBounds()
+		ve.ExciseBoundsRecord = append(ve.ExciseBoundsRecord, manifest.ExciseOpEntry{
+			Bounds: exciseBounds,
+			SeqNum: ingestFlushable.exciseSeqNum,
+		})
+
 		// Iterate through all levels and find files that intersect with exciseSpan.
 		for layer, ls := range version.AllLevelsAndSublevels() {
 			for m := range ls.Overlaps(d.cmp, ingestFlushable.exciseSpan.UserKeyBounds()).All() {

format_major_version.go

@@ -235,6 +235,9 @@ const (
 	FormatValueSeparation
 
 	// -- Add new versions here --
+	// FormatExciseBoundsRecord is a format major version that adds support for
+	// persisting excise bounds records in the manifest (VersionEdit).
+	FormatExciseBoundsRecord
 
 	// FormatNewest is the most recent format major version.
 	FormatNewest FormatMajorVersion = iota - 1
@@ -276,7 +279,7 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 		return sstable.TableFormatPebblev5
 	case FormatTableFormatV6, formatDeprecatedExperimentalValueSeparation:
 		return sstable.TableFormatPebblev6
-	case formatFooterAttributes, FormatValueSeparation:
+	case formatFooterAttributes, FormatValueSeparation, FormatExciseBoundsRecord:
 		return sstable.TableFormatPebblev7
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -290,7 +293,7 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
 	case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
 		FormatDeleteSizedAndObsolete, FormatVirtualSSTables, FormatSyntheticPrefixSuffix,
 		FormatFlushableIngestExcises, FormatColumnarBlocks, FormatWALSyncChunks,
-		FormatTableFormatV6, formatDeprecatedExperimentalValueSeparation, formatFooterAttributes,
+		FormatExciseBoundsRecord, FormatTableFormatV6, formatDeprecatedExperimentalValueSeparation, formatFooterAttributes,
 		FormatValueSeparation:
 		return sstable.TableFormatPebblev1
 	default:
@@ -349,6 +352,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 	FormatValueSeparation: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatValueSeparation)
 	},
+	FormatExciseBoundsRecord: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatExciseBoundsRecord)
+	},
 }
 
 const formatVersionMarkerName = `format-version`

format_major_version_test.go

@@ -35,8 +35,9 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
 
 	// When we add a new version, we should add a check for the new version in
 	// addition to updating these expected values.
-	require.Equal(t, FormatNewest, FormatMajorVersion(24))
-	require.Equal(t, internalFormatNewest, FormatMajorVersion(24))
+	require.Equal(t, FormatNewest, FormatMajorVersion(25))
+	require.Equal(t, internalFormatNewest, FormatMajorVersion(25))
+	require.Equal(t, FormatExciseBoundsRecord, FormatMajorVersion(25))
 }
 
 func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -77,6 +78,8 @@ func TestRatchetFormat(t *testing.T) {
 	require.Equal(t, formatFooterAttributes, d.FormatMajorVersion())
 	require.NoError(t, d.RatchetFormatMajorVersion(FormatValueSeparation))
 	require.Equal(t, FormatValueSeparation, d.FormatMajorVersion())
+	require.NoError(t, d.RatchetFormatMajorVersion(FormatExciseBoundsRecord))
+	require.Equal(t, FormatExciseBoundsRecord, d.FormatMajorVersion())
 
 	require.NoError(t, d.Close())
 
@@ -239,6 +242,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
 		formatDeprecatedExperimentalValueSeparation: {sstable.TableFormatPebblev1, sstable.TableFormatPebblev6},
 		formatFooterAttributes:                      {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
 		FormatValueSeparation:                       {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
+		FormatExciseBoundsRecord:                    {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
 	}
 
 	// Valid versions.

ingest.go

@@ -2108,6 +2108,10 @@ func (d *DB) ingestApply(
 					updateLevelMetricsOnExcise(m, layer.Level(), newFiles)
 				}
 			}
+			ve.ExciseBoundsRecord = append(ve.ExciseBoundsRecord, manifest.ExciseOpEntry{
+				Bounds: exciseBounds,
+				SeqNum: exciseSeqNum,
+			})
 		}
 		if len(filesToSplit) > 0 {
 			// For the same reasons as the above call to excise, we hold the db mutex

internal/manifest/testdata/version_edit_apply

@@ -187,6 +187,7 @@ apply v6
   add-table: L1 000003(000009):[a#2,SET-b#2,SET]
   add-table: L1 000004(000009):[c#2,SET-e#2,SET]
   add-backing: 000009
+  excise-op: [b, c] #3
 ----
 L1:
   000003(000009):[a#2,SET-b#2,SET] seqnums:[0-0] points:[a#2,SET-b#2,SET]

internal/manifest/testdata/version_edit_decode

@@ -64,31 +64,37 @@ f9a006                       #   . ValueSize    = 102521
 6c                           # <tagDeleteBlobFile>
 21                           #   . BlobFileID   = 000033
 21                           #   . FileNum      = 000033
+0a016201630103               # <tagExciseBoundsRecord>, [b, c] #3
 ----
-021903660484106b2929d8a301f9a0069f9485bd066c2121
+021903660484106b2929d8a301f9a0069f9485bd066c21210a0162016301
+03
   log-num:       25
   next-file-num: 102
   last-seq-num:  2052
   add-blob-file: B000041 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
   del-blob-file: B000033 000033
+  excise-op:     [b, c] #3
 
 encode
   add-table:     L6 000029:[bar#14,DEL-foo#13,SET] blobrefs:[(B000041: 20952); depth:1]
   add-blob-file: B000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
   del-blob-file: B000033 000033
+  excise-op: [b, c] #3
 ----
 67061d000b626172000e0000000000000b666f6f010d0000000000000000
-45010129d8a301016baf0729d8a301f9a006006c2121
+45010129d8a301016baf0729d8a301f9a006006c21210a016201630103
   add-table:     L6 000029:[bar#14,DEL-foo#13,SET] seqnums:[0-0] points:[bar#14,DEL-foo#13,SET] blobrefs:[(B000041: 20952); depth:1]
   add-blob-file: B000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
   del-blob-file: B000033 000033
+  excise-op:     [b, c] #3
 
 decode
 67061d000b626172000e0000000000000b666f6f010d0000000000000000
-45010129d8a301016baf0729d8a301f9a006006c2121
+45010129d8a301016baf0729d8a301f9a006006c21210a016201630103
 ----
 67061d000b626172000e0000000000000b666f6f010d0000000000000000
-45010129d8a301016baf0729d8a301f9a006006c2121
+45010129d8a301016baf0729d8a301f9a006006c21210a016201630103
   add-table:     L6 000029:[bar#14,DEL-foo#13,SET]
   add-blob-file: B000943 physical:{000041 size:[20952 (20KB)] vals:[102521 (100KB)]}
   del-blob-file: B000033 000033
+  excise-op:     [b, c] #3

internal/manifest/version_edit.go

@@ -35,14 +35,15 @@ type byteReader interface {
 // Tag 8 is no longer used.
 const (
 	// LevelDB tags.
-	tagComparator     = 1
-	tagLogNumber      = 2
-	tagNextFileNumber = 3
-	tagLastSequence   = 4
-	tagCompactPointer = 5
-	tagDeletedFile    = 6
-	tagNewFile        = 7
-	tagPrevLogNumber  = 9
+	tagComparator         = 1
+	tagLogNumber          = 2
+	tagNextFileNumber     = 3
+	tagLastSequence       = 4
+	tagCompactPointer     = 5
+	tagDeletedFile        = 6
+	tagNewFile            = 7
+	tagPrevLogNumber      = 9
+	tagExciseBoundsRecord = 10
 
 	// RocksDB tags.
 	tagNewFile2         = 100
@@ -98,6 +99,12 @@ type NewTableEntry struct {
 	BackingFileNum base.DiskFileNum
 }
 
+// ExciseOpEntry holds the bounds and sequence number for an excise operation.
+type ExciseOpEntry struct {
+	Bounds base.UserKeyBounds
+	SeqNum base.SeqNum
+}
+
 // VersionEdit holds the state for an edit to a Version along with other
 // on-disk state (log numbers, next file number, and the last sequence number).
 type VersionEdit struct {
@@ -179,6 +186,9 @@ type VersionEdit struct {
 	// While replaying a MANIFEST, the values are nil. Otherwise the values must
 	// not be nil.
 	DeletedBlobFiles map[DeletedBlobFileEntry]*PhysicalBlobFile
+
+	// ExciseBoundsRecord holds excise operations as a slice of entries.
+	ExciseBoundsRecord []ExciseOpEntry
 }
 
 // Decode decodes an edit from the specified reader.
@@ -575,6 +585,32 @@ func (v *VersionEdit) Decode(r io.Reader) error {
 			}
 			v.ObsoletePrevLogNum = n
 
+		case tagExciseBoundsRecord:
+			start, err := d.readBytes()
+			if err != nil {
+				return err
+			}
+			end, err := d.readBytes()
+			if err != nil {
+				return err
+			}
+			kind, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			seqNum, err := d.readUvarint()
+			if err != nil {
+				return err
+			}
+			bounds := base.UserKeyBounds{
+				Start: start,
+				End:   base.UserKeyExclusiveIf(end, base.BoundaryKind(kind) == base.Exclusive),
+			}
+			v.ExciseBoundsRecord = append(v.ExciseBoundsRecord, ExciseOpEntry{
+				Bounds: bounds,
+				SeqNum: base.SeqNum(seqNum),
+			})
+
 		case tagColumnFamily, tagColumnFamilyAdd, tagColumnFamilyDrop, tagMaxColumnFamily:
 			return base.CorruptionErrorf("column families are not supported")
 
@@ -641,6 +677,10 @@ func (v *VersionEdit) string(verbose bool, fmtKey base.FormatKey) string {
 	for _, df := range deletedBlobFileEntries {
 		fmt.Fprintf(&buf, "  del-blob-file: %s %s\n", df.FileID, df.FileNum)
 	}
+	for _, exciseBounds := range v.ExciseBoundsRecord {
+		fmt.Fprintf(&buf, "  excise-op:     %s #%d\n", exciseBounds.Bounds.String(), exciseBounds.SeqNum)
+	}
+
 	return buf.String()
 }
 
@@ -728,6 +768,14 @@ func ParseVersionEditDebug(s string) (_ *VersionEdit, err error) {
 				FileNum: p.DiskFileNum(),
 			}] = nil
 
+		case "excise-op":
+			bounds := p.UserKeyBounds()
+			seqNum := p.HashSeqNum()
+			ve.ExciseBoundsRecord = append(ve.ExciseBoundsRecord, ExciseOpEntry{
+				Bounds: bounds,
+				SeqNum: seqNum,
+			})
+
 		default:
 			return nil, errors.Errorf("field %q not implemented", field)
 		}
@@ -869,6 +917,13 @@ func (v *VersionEdit) Encode(w io.Writer) error {
 		e.writeUvarint(uint64(x.FileID))
 		e.writeUvarint(uint64(x.FileNum))
 	}
+	for _, entry := range v.ExciseBoundsRecord {
+		e.writeUvarint(tagExciseBoundsRecord)
+		e.writeBytes(entry.Bounds.Start)
+		e.writeBytes(entry.Bounds.End.Key)
+		e.writeUvarint(uint64(entry.Bounds.End.Kind))
+		e.writeUvarint(uint64(entry.SeqNum))
+	}
 	_, err := w.Write(e.Bytes())
 	return err
 }

internal/strparse/strparse.go

@@ -109,6 +109,36 @@ func (p *Parser) TryLevel() (level int, ok bool) {
 	return 0, false
 }
 
+// BracketedRange parses UserKeyBounds as string form [a, b], [bb, cc),
+// handles exclusive and inclusive cases of excise bounds.
+func (p *Parser) BracketedRange() string {
+	if p.Done() {
+		p.Errf("expected bracketed range, but no tokens found")
+	}
+	open := p.Peek()
+	if !(open == "[") {
+		p.Errf("expected opening bracket, got %q", open)
+	}
+
+	p.Next()
+	var builder strings.Builder
+	builder.WriteString(open)
+	first := true
+	for !p.Done() {
+		tok := p.Next()
+		if tok == "]" || tok == ")" {
+			builder.WriteString(tok)
+			break
+		}
+		if !first {
+			builder.WriteString(" ")
+		}
+		builder.WriteString(tok)
+		first = false
+	}
+	return builder.String()
+}
+
 // Level parses the next token as a level.
 func (p *Parser) Level() int {
 	level, ok := p.TryLevel()
@@ -150,6 +180,12 @@ func (p *Parser) SeqNum() base.SeqNum {
 	return base.ParseSeqNum(p.Next())
 }
 
+// Uint64 parses the next token as a sequence number with a "#" prefix.
+func (p *Parser) HashSeqNum() base.SeqNum {
+	p.tokens[0] = p.tokens[0][1:]
+	return base.ParseSeqNum(p.Next())
+}
+
 // BlobFileID parses the next token as a BlobFileID.
 func (p *Parser) BlobFileID() base.BlobFileID {
 	s := p.Next()
@@ -178,6 +214,11 @@ func (p *Parser) InternalKey() base.InternalKey {
 	return base.ParseInternalKey(p.Next())
 }
 
+// UserKeyBounds parses the next token as an user key interval.
+func (p *Parser) UserKeyBounds() base.UserKeyBounds {
+	return base.ParseUserKeyBounds(p.BracketedRange())
+}
+
 // Errf panics with an error which includes the original string and the last
 // token.
 func (p *Parser) Errf(format string, args ...any) {

open_test.go

@@ -502,7 +502,7 @@ func TestNewDBFilenames(t *testing.T) {
 			"LOCK",
 			"MANIFEST-000001",
 			"OPTIONS-000003",
-			"marker.format-version.000011.024",
+			"marker.format-version.000012.025",
 			"marker.manifest.000001.MANIFEST-000001",
 		},
 	}