blob: add FileFormatV2 which has a properties block

We extend the blob file format by adding a new V2 footer (and
associated magic signature) which encodes a properties block handle
(and a metaindex block handle just in case we need one in the future).
The properties block contains a key/value for compression statistics.

Author: RaduBerinde

blob_rewrite.go

@@ -291,7 +291,7 @@ func (d *DB) runBlobFileRewriteLocked(
 		env,
 		objMeta.DiskFileNum,
 		writable,
-		d.opts.MakeBlobWriterOptions(6),
+		d.opts.MakeBlobWriterOptions(6, d.BlobFileFormat()),
 		c.referencingTables,
 		c.input,
 	)

data_test.go

@@ -1163,12 +1163,18 @@ func runDBDefineCmdReuseFS(td *datadriven.TestData, opts *Options) (*DB, error)
 		return nil, err
 	}
 
-	if len(ve.NewTables) > 0 {
-		// Collect any blob files created.
-		fileStats, err := valueSeparator.bv.WriteFiles(func(fileNum base.DiskFileNum) (objstorage.Writable, error) {
+	if len(ve.NewTables) == 0 {
+		return d, nil
+	}
+
+	// Collect any blob files created.
+	if !valueSeparator.bv.IsEmpty() {
+		newBlobObject := func(fileNum base.DiskFileNum) (objstorage.Writable, error) {
 			writable, _, err := d.objProvider.Create(context.Background(), base.FileTypeBlob, fileNum, objstorage.CreateOptions{})
 			return writable, err
-		}, d.opts.MakeBlobWriterOptions(0))
+		}
+		blobWriterOpts := d.opts.MakeBlobWriterOptions(0, d.BlobFileFormat())
+		fileStats, err := valueSeparator.bv.WriteFiles(newBlobObject, blobWriterOpts)
 		if err != nil {
 			return nil, err
 		}
@@ -1184,22 +1190,22 @@ func runDBDefineCmdReuseFS(td *datadriven.TestData, opts *Options) (*DB, error)
 				Physical: m,
 			})
 		}
+	}
 
-		jobID := d.newJobIDLocked()
-		_, err = d.mu.versions.UpdateVersionLocked(func() (versionUpdate, error) {
-			return versionUpdate{
-				VE:                      ve,
-				JobID:                   jobID,
-				Metrics:                 newFileMetrics(ve.NewTables),
-				InProgressCompactionsFn: func() []compactionInfo { return nil },
-			}, nil
-		})
-		if err != nil {
-			return nil, err
-		}
-		d.updateReadStateLocked(nil)
-		d.updateTableStatsLocked(ve.NewTables)
+	jobID := d.newJobIDLocked()
+	_, err = d.mu.versions.UpdateVersionLocked(func() (versionUpdate, error) {
+		return versionUpdate{
+			VE:                      ve,
+			JobID:                   jobID,
+			Metrics:                 newFileMetrics(ve.NewTables),
+			InProgressCompactionsFn: func() []compactionInfo { return nil },
+		}, nil
+	})
+	if err != nil {
+		return nil, err
 	}
+	d.updateReadStateLocked(nil)
+	d.updateTableStatsLocked(ve.NewTables)
 
 	return d, nil
 }

format_major_version.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/manifest"
 	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/sstable"
+	"github.com/cockroachdb/pebble/sstable/blob"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/vfs/atomicfs"
 )
@@ -234,11 +235,16 @@ const (
 	// participate in every compaction.
 	FormatValueSeparation
 
-	// -- Add new versions here --
 	// FormatExciseBoundsRecord is a format major version that adds support for
 	// persisting excise bounds records in the manifest (VersionEdit).
 	FormatExciseBoundsRecord
 
+	// FormatV2BlobFiles is a format major version that adds support for V2 blob
+	// file format (which adds compression statistics).
+	FormatV2BlobFiles
+
+	// -- Add new versions here --
+
 	// FormatNewest is the most recent format major version.
 	FormatNewest FormatMajorVersion = iota - 1
 
@@ -260,44 +266,55 @@ const FormatMinSupported = FormatFlushableIngest
 // objects (see CreateOnShared option).
 const FormatMinForSharedObjects = FormatVirtualSSTables
 
-// IsSupported returns true if the version is supported by the current Pebble
-// version.
-func (v FormatMajorVersion) IsSupported() bool {
-	return v == FormatDefault && v >= FormatMinSupported && v <= internalFormatNewest
+// resolveDefault asserts that the given version is supported, and returns the
+// given version, replacing FormatDefault with FormatMinSupported.
+func (v FormatMajorVersion) resolveDefault() FormatMajorVersion {
+	if v == FormatDefault {
+		return FormatMinSupported
+	}
+	if v < FormatMinSupported || v > internalFormatNewest {
+		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
+	}
+	return v
 }
 
 // MaxTableFormat returns the maximum sstable.TableFormat that can be used at
 // this FormatMajorVersion.
 func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat {
-	switch v {
-	case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted:
-		return sstable.TableFormatPebblev3
-	case FormatDeleteSizedAndObsolete, FormatVirtualSSTables, FormatSyntheticPrefixSuffix,
-		FormatFlushableIngestExcises:
-		return sstable.TableFormatPebblev4
-	case FormatColumnarBlocks, FormatWALSyncChunks:
-		return sstable.TableFormatPebblev5
-	case FormatTableFormatV6, formatDeprecatedExperimentalValueSeparation:
-		return sstable.TableFormatPebblev6
-	case formatFooterAttributes, FormatValueSeparation, FormatExciseBoundsRecord:
+	v = v.resolveDefault()
+	switch {
+	case v >= formatFooterAttributes:
 		return sstable.TableFormatPebblev7
+	case v >= FormatTableFormatV6:
+		return sstable.TableFormatPebblev6
+	case v >= FormatColumnarBlocks:
+		return sstable.TableFormatPebblev5
+	case v >= FormatDeleteSizedAndObsolete:
+		return sstable.TableFormatPebblev4
 	default:
-		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
+		return sstable.TableFormatPebblev3
 	}
 }
 
 // MinTableFormat returns the minimum sstable.TableFormat that can be used at
 // this FormatMajorVersion.
 func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat {
-	switch v {
-	case FormatDefault, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted,
-		FormatDeleteSizedAndObsolete, FormatVirtualSSTables, FormatSyntheticPrefixSuffix,
-		FormatFlushableIngestExcises, FormatColumnarBlocks, FormatWALSyncChunks,
-		FormatExciseBoundsRecord, FormatTableFormatV6, formatDeprecatedExperimentalValueSeparation, formatFooterAttributes,
-		FormatValueSeparation:
-		return sstable.TableFormatPebblev1
+	_ = v.resolveDefault()
+	return sstable.TableFormatPebblev1
+}
+
+// MaxBlobFileFormat returns the maximum blob.FileFormat that can be used at
+// this FormatMajorVersion. It can only be used on versions that support value
+// separation.
+func (v FormatMajorVersion) MaxBlobFileFormat() blob.FileFormat {
+	v = v.resolveDefault()
+	switch {
+	case v >= FormatV2BlobFiles:
+		return blob.FileFormatV2
+	case v >= FormatValueSeparation:
+		return blob.FileFormatV1
 	default:
-		panic(fmt.Sprintf("pebble: unsupported format major version: %s", v))
+		panic(fmt.Sprintf("pebble: format major version %s does not support blob files", v))
 	}
 }
 
@@ -355,6 +372,9 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{
 	FormatExciseBoundsRecord: func(d *DB) error {
 		return d.finalizeFormatVersUpgrade(FormatExciseBoundsRecord)
 	},
+	FormatV2BlobFiles: func(d *DB) error {
+		return d.finalizeFormatVersUpgrade(FormatV2BlobFiles)
+	},
 }
 
 const formatVersionMarkerName = `format-version`
@@ -426,6 +446,12 @@ func (d *DB) TableFormat() sstable.TableFormat {
 	return f
 }
 
+// BlobFileFormat returns the blob.FileFormat that the database is currently
+// using when writing blob files.
+func (d *DB) BlobFileFormat() blob.FileFormat {
+	return d.FormatMajorVersion().MaxBlobFileFormat()
+}
+
 // shouldCreateShared returns true if the database should use shared objects
 // when creating new objects on the given level.
 func (d *DB) shouldCreateShared(targetLevel int) bool {

format_major_version_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/cockroachdb/pebble/sstable"
+	"github.com/cockroachdb/pebble/sstable/blob"
 	"github.com/cockroachdb/pebble/vfs"
 	"github.com/cockroachdb/pebble/vfs/atomicfs"
 	"github.com/stretchr/testify/require"
@@ -32,12 +33,13 @@ func TestFormatMajorVersionStableValues(t *testing.T) {
 	require.Equal(t, formatDeprecatedExperimentalValueSeparation, FormatMajorVersion(22))
 	require.Equal(t, formatFooterAttributes, FormatMajorVersion(23))
 	require.Equal(t, FormatValueSeparation, FormatMajorVersion(24))
-
-	// When we add a new version, we should add a check for the new version in
-	// addition to updating these expected values.
-	require.Equal(t, FormatNewest, FormatMajorVersion(25))
-	require.Equal(t, internalFormatNewest, FormatMajorVersion(25))
 	require.Equal(t, FormatExciseBoundsRecord, FormatMajorVersion(25))
+	require.Equal(t, FormatV2BlobFiles, FormatMajorVersion(26))
+
+	// When we add a new version, we should add a check for the new version above
+	// in addition to updating the expected values below.
+	require.Equal(t, FormatNewest, FormatMajorVersion(26))
+	require.Equal(t, internalFormatNewest, FormatMajorVersion(26))
 }
 
 func TestFormatMajorVersion_MigrationDefined(t *testing.T) {
@@ -55,36 +57,18 @@ func TestRatchetFormat(t *testing.T) {
 	d, err := Open("", opts)
 	require.NoError(t, err)
 	require.NoError(t, d.Set([]byte("foo"), []byte("bar"), Sync))
-	require.Equal(t, FormatFlushableIngest, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatPrePebblev1MarkedCompacted))
-	require.Equal(t, FormatPrePebblev1MarkedCompacted, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatDeleteSizedAndObsolete))
-	require.Equal(t, FormatDeleteSizedAndObsolete, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatVirtualSSTables))
-	require.Equal(t, FormatVirtualSSTables, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatSyntheticPrefixSuffix))
-	require.Equal(t, FormatSyntheticPrefixSuffix, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatFlushableIngestExcises))
-	require.Equal(t, FormatFlushableIngestExcises, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatColumnarBlocks))
-	require.Equal(t, FormatColumnarBlocks, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatWALSyncChunks))
-	require.Equal(t, FormatWALSyncChunks, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatTableFormatV6))
-	require.Equal(t, FormatTableFormatV6, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(formatDeprecatedExperimentalValueSeparation))
-	require.Equal(t, formatDeprecatedExperimentalValueSeparation, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(formatFooterAttributes))
-	require.Equal(t, formatFooterAttributes, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatValueSeparation))
-	require.Equal(t, FormatValueSeparation, d.FormatMajorVersion())
-	require.NoError(t, d.RatchetFormatMajorVersion(FormatExciseBoundsRecord))
-	require.Equal(t, FormatExciseBoundsRecord, d.FormatMajorVersion())
+
+	// Ratchet through all supported versions.
+	require.Equal(t, FormatMinSupported, d.FormatMajorVersion())
+	for v := FormatMinSupported + 1; v <= internalFormatNewest; v++ {
+		require.NoError(t, d.RatchetFormatMajorVersion(v))
+		require.Equal(t, v, d.FormatMajorVersion())
+	}
 
 	require.NoError(t, d.Close())
 
 	// If we Open the database again, leaving the default format, the
-	// database should Open using the persisted FormatNewest.
+	// database should Open using the persisted internalFormatNewest.
 	opts = &Options{FS: fs, Logger: testLogger{t}}
 	opts.WithFSDefaults()
 	d, err = Open("", opts)
@@ -243,6 +227,7 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
 		formatFooterAttributes:                      {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
 		FormatValueSeparation:                       {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
 		FormatExciseBoundsRecord:                    {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
+		FormatV2BlobFiles:                           {sstable.TableFormatPebblev1, sstable.TableFormatPebblev7},
 	}
 
 	// Valid versions.
@@ -258,6 +243,29 @@ func TestFormatMajorVersions_TableFormat(t *testing.T) {
 	require.Panics(t, func() { _ = fmv.MinTableFormat() })
 }
 
+func TestFormatMajorVersions_BlobFileFormat(t *testing.T) {
+	// NB: This test is intended to validate the mapping between every
+	// FormatMajorVersion and blob.FileFormat exhaustively. This serves as a
+	// sanity check that new versions have a corresponding mapping. The test
+	// fixture is intentionally verbose.
+
+	m := map[FormatMajorVersion]blob.FileFormat{
+		FormatValueSeparation:    blob.FileFormatV1,
+		FormatExciseBoundsRecord: blob.FileFormatV1,
+		FormatV2BlobFiles:        blob.FileFormatV2,
+	}
+
+	// Valid versions.
+	for fmv := FormatValueSeparation; fmv <= internalFormatNewest; fmv++ {
+		got := fmv.MaxBlobFileFormat()
+		require.Equalf(t, m[fmv], got, "got %s; want %s", got, m[fmv])
+	}
+
+	// Invalid versions.
+	require.Panics(t, func() { _ = (internalFormatNewest + 1).MaxBlobFileFormat() })
+	require.Panics(t, func() { _ = (FormatValueSeparation - 1).MaxBlobFileFormat() })
+}
+
 // TestFormatMajorVersions_ColumnarBlocks ensures that
 // Experimental.EnableColumnarBlocks is respected on recent format major
 // versions.

internal/blobtest/handles.go

@@ -212,6 +212,11 @@ func (bv *Values) ParseInlineHandle(
 	return h, remaining, nil
 }
 
+// IsEmpty returns true if the Values has no tracked handles.
+func (bv *Values) IsEmpty() bool {
+	return len(bv.trackedHandles) == 0
+}
+
 // WriteFiles writes all the blob files referenced by Values, using
 // newBlobObject to construct new objects.
 //

metrics.go

@@ -1061,7 +1061,7 @@ func (m *Metrics) StringForTests() string {
 	// We recalculate the file cache size using the 64-bit sizes, and we ignore
 	// the genericcache metadata size which is harder to adjust.
 	const sstableReaderSize64bit = 280
-	const blobFileReaderSize64bit = 96
+	const blobFileReaderSize64bit = 112
 	mCopy.FileCache.Size = mCopy.FileCache.TableCount*sstableReaderSize64bit + mCopy.FileCache.BlobFileCount*blobFileReaderSize64bit
 	if math.MaxInt == math.MaxInt64 {
 		// Verify the 64-bit sizes, so they are kept updated.

open_test.go

@@ -502,7 +502,7 @@ func TestNewDBFilenames(t *testing.T) {
 			"LOCK",
 			"MANIFEST-000001",
 			"OPTIONS-000003",
-			"marker.format-version.000012.025",
+			"marker.format-version.000013.026",
 			"marker.manifest.000001.MANIFEST-000001",
 		},
 	}

options.go

@@ -2543,9 +2543,10 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab
 
 // MakeBlobWriterOptions constructs blob.FileWriterOptions from the corresponding
 // options in the receiver.
-func (o *Options) MakeBlobWriterOptions(level int) blob.FileWriterOptions {
+func (o *Options) MakeBlobWriterOptions(level int, format blob.FileFormat) blob.FileWriterOptions {
 	lo := o.Levels[level]
 	return blob.FileWriterOptions{
+		Format:       format,
 		Compression:  lo.Compression(),
 		ChecksumType: block.ChecksumTypeCRC32c,
 		FlushGovernor: block.MakeFlushGovernor(