sstable: add checksum to footer

Introduce a new FMV and table format to add a checksum to the footer
of an sstable as stated in #4344

Author: EdwardX29

cockroachkvs/cockroachkvs_bench_test.go

@@ -54,6 +54,18 @@ func BenchmarkRandSeekInSST(b *testing.B) {
 			valueLen: 128,        // ~200 KVs per data block
 			version:  sstable.TableFormatPebblev5,
 		},
+		{
+			name:     "v6/single-level",
+			numKeys:  200 * 100, // ~100 data blocks.
+			valueLen: 128,       // ~200 KVs per data block
+			version:  sstable.TableFormatPebblev6,
+		},
+		{
+			name:     "v6/two-level",
+			numKeys:  200 * 5000, // ~5000 data blocks
+			valueLen: 128,        // ~200 KVs per data block
+			version:  sstable.TableFormatPebblev6,
+		},
 	}
 	keyCfg := keyGenConfig{
 		PrefixAlphabetLen: 26,

compaction_test.go

@@ -1276,6 +1276,12 @@ func TestManualCompaction(t *testing.T) {
 		{
 			testData:   "testdata/manual_compaction_set_with_del_sstable_Pebblev5",
 			minVersion: FormatColumnarBlocks,
+			maxVersion: FormatColumnarBlocks,
+		},
+		{
+			testData:   "testdata/manual_compaction_set_with_del_sstable_Pebblev6",
+			minVersion: formatChecksumFooter,
+			maxVersion: formatChecksumFooter,
 		},
 	}
 

format_major_version.go

@@ -214,6 +214,11 @@ const (
 
 	// -- Add experimental versions here --
 
+	// formatChecksumFooter is a format major version enabling use of the
+	// TableFormatPebblev6 table format. It is a format allowing for the checksum
+	// of sstable footers.
+	formatChecksumFooter
+
 	// internalFormatNewest is the most recent, possibly experimental format major
 	// version.
 	internalFormatNewest FormatMajorVersion = iota - 2
@@ -244,6 +249,8 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
 		return sstable.TableFormatPebblev4
 	case FormatColumnarBlocks, FormatWALSyncChunks:
 		return sstable.TableFormatPebblev5
+	case formatChecksumFooter:
+		return sstable.TableFormatPebblev6
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
 	}
@@ -255,7 +262,8 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
 	switch v {
 	case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
 		FormatDeleteSizedAndObsolete, FormatVirtualSSTables, FormatSyntheticPrefixSuffix,
-		FormatFlushableIngestExcises, FormatColumnarBlocks, FormatWALSyncChunks:
+		FormatFlushableIngestExcises, FormatColumnarBlocks, FormatWALSyncChunks,
+		formatChecksumFooter:
 		return sstable.TableFormatPebblev1
 	default:
 		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
@@ -301,6 +309,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 	FormatWALSyncChunks: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatWALSyncChunks)
 	},
+	formatChecksumFooter: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(formatChecksumFooter)
+	},
 }
 
 const formatVersionMarkerName = `format-version`

format_major_version_test.go

@@ -26,11 +26,12 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
 	require.Equal(t, FormatSyntheticPrefixSuffix, FormatMajorVersion(17))
 	require.Equal(t, FormatFlushableIngestExcises, FormatMajorVersion(18))
 	require.Equal(t, FormatColumnarBlocks, FormatMajorVersion(19))
+	require.Equal(t, FormatWALSyncChunks, FormatMajorVersion(20))
 
 	// When we add a new version, we should add a check for the new version in
 	// addition to updating these expected values.
 	require.Equal(t, FormatNewest, FormatMajorVersion(20))
-	require.Equal(t, internalFormatNewest, FormatMajorVersion(20))
+	require.Equal(t, internalFormatNewest, FormatMajorVersion(21))
 }
 
 func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -63,6 +64,8 @@ func TestRatchetFormat(t *testing.T) {
 	require.Equal(t, FormatColumnarBlocks, d.FormatMajorVersion())
 	require.NoError(t, d.RatchetFormatMajorVersion(FormatWALSyncChunks))
 	require.Equal(t, FormatWALSyncChunks, d.FormatMajorVersion())
+	require.NoError(t, d.RatchetFormatMajorVersion(formatChecksumFooter))
+	require.Equal(t, formatChecksumFooter, d.FormatMajorVersion())
 
 	require.NoError(t, d.Close())
 
@@ -221,6 +224,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
 		FormatFlushableIngestExcises:     {sstable.TableFormatPebblev1, sstable.TableFormatPebblev4},
 		FormatColumnarBlocks:             {sstable.TableFormatPebblev1, sstable.TableFormatPebblev5},
 		FormatWALSyncChunks:              {sstable.TableFormatPebblev1, sstable.TableFormatPebblev5},
+		formatChecksumFooter:             {sstable.TableFormatPebblev1, sstable.TableFormatPebblev6},
 	}
 
 	// Valid versions.

open_test.go

@@ -332,7 +332,7 @@ func TestNewDBFilenames(t *testing.T) {
 			"LOCK",
 			"MANIFEST-000001",
 			"OPTIONS-000003",
-			"marker.format-version.000007.020",
+			"marker.format-version.000008.021",
 			"marker.manifest.000001.MANIFEST-000001",
 		},
 	}

sstable/copier_test.go

@@ -53,6 +53,8 @@ func TestCopySpan(t *testing.T) {
 						tableFormat = TableFormatPebblev4
 					case "pebblev5":
 						tableFormat = TableFormatPebblev5
+					case "pebblev6":
+						tableFormat = TableFormatPebblev6
 					}
 				case "block_size":
 					var err error

sstable/format.go

@@ -29,6 +29,7 @@ const (
 	TableFormatPebblev3 // Value blocks.
 	TableFormatPebblev4 // DELSIZED tombstones.
 	TableFormatPebblev5 // Columnar blocks.
+	TableFormatPebblev6 // Checksum Footer.
 	NumTableFormats
 
 	TableFormatMax = NumTableFormats - 1
@@ -236,6 +237,8 @@ func parseTableFormat(magic []byte, version uint32) (TableFormat, error) {
 			return TableFormatPebblev4, nil
 		case 5:
 			return TableFormatPebblev5, nil
+		case 6:
+			return TableFormatPebblev6, nil
 		default:
 			return TableFormatUnspecified, base.CorruptionErrorf(
 				"(unsupported pebble format version %d)", errors.Safe(version))
@@ -276,6 +279,8 @@ func (f TableFormat) AsTuple() (string, uint32) {
 		return pebbleDBMagic, 4
 	case TableFormatPebblev5:
 		return pebbleDBMagic, 5
+	case TableFormatPebblev6:
+		return pebbleDBMagic, 6
 	default:
 		panic("sstable: unknown table format version tuple")
 	}
@@ -300,6 +305,8 @@ func (f TableFormat) String() string {
 		return "(Pebble,v4)"
 	case TableFormatPebblev5:
 		return "(Pebble,v5)"
+	case TableFormatPebblev6:
+		return "(Pebble,v6)"
 	default:
 		panic("sstable: unknown table format version tuple")
 	}

sstable/format_test.go

@@ -65,6 +65,12 @@ func TestTableFormat_RoundTrip(t *testing.T) {
 			version: 5,
 			want:    TableFormatPebblev5,
 		},
+		{
+			name:    "PebbleDBv6",
+			magic:   pebbleDBMagic,
+			version: 6,
+			want:    TableFormatPebblev6,
+		},
 		// Invalid cases.
 		{
 			name:    "Invalid RocksDB version",
@@ -75,8 +81,8 @@ func TestTableFormat_RoundTrip(t *testing.T) {
 		{
 			name:    "Invalid PebbleDB version",
 			magic:   pebbleDBMagic,
-			version: 6,
-			wantErr: "pebble/table: invalid table 000001: (unsupported pebble format version 6)",
+			version: 7,
+			wantErr: "pebble/table: invalid table 000001: (unsupported pebble format version 7)",
 		},
 		{
 			name:    "Unknown magic string",

sstable/layout.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/binfmt"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
+	"github.com/cockroachdb/pebble/internal/crc"
 	"github.com/cockroachdb/pebble/internal/sstableinternal"
 	"github.com/cockroachdb/pebble/internal/treeprinter"
 	"github.com/cockroachdb/pebble/objstorage"
@@ -140,6 +141,24 @@ func (l *Layout) Describe(
 			trailer, offset := make([]byte, b.Length), 0
 			_ = r.blockReader.Readable().ReadAt(ctx, trailer, int64(b.Offset))
 
+			// In all cases, we know the version is right before the magic.
+			version := binary.LittleEndian.Uint32(trailer[len(trailer)-magicLen-versionLen:])
+			magicNumber := trailer[len(trailer)-magicLen:]
+			format, err := parseTableFormat(magicNumber, version)
+			if err != nil {
+				panic("Error parsing table format.")
+			}
+
+			var computedChecksum uint32
+			var encodedChecksum uint32
+			if format >= TableFormatPebblev6 {
+				computedChecksum = crc.CRC(0).
+					Update(trailer[:checkedPebbleDBChecksumOffset]).
+					Update(trailer[checkedPebbleDBVersionOffset:]).
+					Value()
+				encodedChecksum = binary.LittleEndian.Uint32(trailer[checkedPebbleDBChecksumOffset:])
+			}
+
 			if b.Name == "footer" {
 				checksumType := block.ChecksumType(trailer[0])
 				tpNode.Childf("%03d  checksum type: %s", offset, checksumType)
@@ -160,22 +179,21 @@ func (l *Layout) Describe(
 			if b.Name == "leveldb-footer" {
 				trailing = 8
 			}
-
 			offset += len(trailer) - trailing
-			trailer = trailer[len(trailer)-trailing:]
 
-			if b.Name == "footer" {
-				version := trailer[:4]
-				tpNode.Childf("%03d  version: %d", offset, binary.LittleEndian.Uint32(version))
-				trailer, offset = trailer[4:], offset+4
+			if format >= TableFormatPebblev6 {
+				if computedChecksum == encodedChecksum {
+					tpNode.Childf("%03d  footer checksum: 0x%04x", offset-4, encodedChecksum)
+				} else {
+					tpNode.Childf("%03d  invalid footer checksum: 0x%04x, expected: 0x%04x", offset-4, encodedChecksum, computedChecksum)
+				}
 			}
 
-			magicNumber := trailer
+			tpNode.Childf("%03d  version: %d", offset, version)
+			offset = offset + 4
 			tpNode.Childf("%03d  magic number: 0x%x", offset, magicNumber)
-
 			continue
 		}
-
 		// Read the block and format it. Returns an error if we couldn't read the
 		// block.
 		err := func() error {