objstorageprovider: add dual-write for cold metadata

This commit adds logic to write the metadata portion of cold blob
files on both tiers, and to transparently read the metadata from the
hot tier.

We assume that the metadata is a suffix of the file; the metadata
filenames encode the start offset, for example "000001.blobmeta.1234"
means that the file contains the data from "00001.blob" starting at
offset 1234.

We add a `Writable` implementation which writes the metadata portion
of a file to both cold and hot storage and a `Readable` implementation
which reads data from a cold tier Readable but uses a `vfs.File` for
the metadata.

The provider keeps track internally of which hot metadata files
exist and their start offsets.

Author: RaduBerinde

internal/base/filenames.go

@@ -128,6 +128,15 @@ const (
 	FileTypeOldTemp
 	FileTypeTemp
 	FileTypeBlob
+	// FileTypeBlobMeta is a file that contains only the metadata portion of a
+	// blob file (used when the blob file in on cold storage). The filename for
+	// blobmeta files is of the form `<file-num>.blobmeta.<offset>`, where
+	// <offset> indicates that the file mirrors the contents of the corresponding
+	// blob file starting at this offset.
+	//
+	// The lifetime of this type of file is managed internally by the objstorage
+	// provider.
+	FileTypeBlobMeta
 )
 
 var fileTypeStrings = [...]string{
@@ -139,6 +148,7 @@ var fileTypeStrings = [...]string{
 	FileTypeOldTemp:  "old-temp",
 	FileTypeTemp:     "temp",
 	FileTypeBlob:     "blob",
+	FileTypeBlobMeta: "blobmeta",
 }
 
 // FileTypeFromName parses a FileType from its string representation.
@@ -166,6 +176,8 @@ func (ft FileType) String() string {
 }
 
 // MakeFilename builds a filename from components.
+//
+// Note that for FileTypeBlobMeta, ".<offset>" must be appended to the filename.
 func MakeFilename(fileType FileType, dfn DiskFileNum) string {
 	// Make a buffer sufficiently large for most possible filenames, especially
 	// the common case of a numbered table or blob file.
@@ -192,18 +204,25 @@ func appendFilename(buf []byte, fileType FileType, dfn DiskFileNum) []byte {
 		buf = fmt.Appendf(buf, "temporary.%06d.dbtmp", uint64(dfn))
 	case FileTypeBlob:
 		buf = fmt.Appendf(buf, "%06d.blob", uint64(dfn))
+	case FileTypeBlobMeta:
+		buf = fmt.Appendf(buf, "%06d.blobmeta", uint64(dfn))
 	default:
 		panic("unreachable")
 	}
 	return buf
 }
 
 // MakeFilepath builds a filepath from components.
+//
+// Note that for FileTypeBlobMeta, ".<offset>" must be appended to the filepath.
 func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
 	return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
 }
 
 // ParseFilename parses the components from a filename.
+//
+// Note that the offset component of a FileTypeBlobMeta is not parsed by this
+// function.
 func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
 	filename = fs.PathBase(filename)
 	switch {
@@ -250,6 +269,9 @@ func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileN
 		case "blob":
 			return FileTypeBlob, dfn, true
 		}
+		if strings.HasPrefix(filename[i+1:], "blobmeta.") {
+			return FileTypeBlobMeta, dfn, true
+		}
 	}
 	return 0, dfn, false
 }

internal/base/filenames_test.go

@@ -7,6 +7,7 @@ package base
 import (
 	"bytes"
 	"fmt"
+	"math/rand/v2"
 	"os"
 	"testing"
 
@@ -49,6 +50,7 @@ func TestParseFilename(t *testing.T) {
 		"000000.blob":            true,
 		"000001.blob":            true,
 		"935203523.blob":         true,
+		"000001.blobmeta.0":      true,
 	}
 	fs := vfs.NewMem()
 	for tc, want := range testCases {
@@ -95,6 +97,17 @@ func TestFilenameRoundTrip(t *testing.T) {
 	}
 }
 
+func TestFilenameBlobMeta(t *testing.T) {
+	fileNum := DiskFileNum(rand.Uint64())
+	offset := rand.Int64()
+	fs := vfs.NewMem()
+	path := fmt.Sprintf("%s.%d", MakeFilepath(fs, "foo", FileTypeBlobMeta, fileNum), offset)
+	typ, fn, ok := ParseFilename(fs, path)
+	require.True(t, ok)
+	require.Equal(t, FileTypeBlobMeta, typ)
+	require.Equal(t, fileNum, fn)
+}
+
 type bufferFataler struct {
 	buf bytes.Buffer
 }

objstorage/objstorage.go

@@ -109,10 +109,27 @@ type Writable interface {
 
 	// Finish completes the object and makes the data durable.
 	// No further calls are allowed after calling Finish.
+	//
+	// An error during Finish does not mean that the object is deleted and removed
+	// from the provider. It is expected that the caller will do that separately
+	// (via Provider.Remove).
 	Finish() error
 
-	// Abort gives up on finishing the object. There is no guarantee about whether
-	// the object exists after calling Abort.
+	// StartMetadataPortion signals to the writer that the metadata part of the
+	// object starts here. If the object is being written to the cold tier, data
+	// in subsequent Write() calls will also be written to the hot tier.
+	//
+	// The function should be called at most one time.
+	//
+	// An error means that we won't be able to successfully finish this object.
+	StartMetadataPortion() error
+
+	// Abort gives up on finishing the object.
+	//
+	// Aborting does not mean that the object is deleted and removed from the
+	// provider. It is expected that the caller will do that separately (via
+	// Provider.Remove).
+	//
 	// No further calls are allowed after calling Abort.
 	Abort()
 }

objstorage/objstorageprovider/cold_readable.go

@@ -0,0 +1,136 @@
+// Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"sync"
+
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/vfs"
+)
+
+// newColdReadableWithHotMeta returns an objstorage.Readable that reads the main
+// data from the wrapped "cold storage" readable, and the metadata from a
+// separate file in a local filesystem. The separate file contains a suffix of
+// the full file, starting at metaStartOffset.
+func newColdReadableWithHotMeta(
+	cold objstorage.Readable, metaFS vfs.FS, metaFilepath string, metaStartOffset int64,
+) *coldReadableWithHotMeta {
+	r := &coldReadableWithHotMeta{
+		cold: cold,
+	}
+	r.meta.fs = metaFS
+	r.meta.filepath = metaFilepath
+	r.meta.startOffset = metaStartOffset
+	return r
+}
+
+type coldReadableWithHotMeta struct {
+	cold objstorage.Readable
+
+	meta struct {
+		fs          vfs.FS
+		filepath    string
+		startOffset int64
+		once        struct {
+			sync.Once
+			file vfs.File
+			err  error
+		}
+	}
+}
+
+var _ objstorage.Readable = (*coldReadableWithHotMeta)(nil)
+
+// readMetaAt reads from the metadata file at the given offset.
+func (r *coldReadableWithHotMeta) readMetaAt(p []byte, off int64) error {
+	r.meta.once.Do(func() {
+		r.meta.once.file, r.meta.once.err = r.meta.fs.Open(r.meta.filepath, vfs.RandomReadsOption)
+	})
+	if r.meta.once.err != nil {
+		return r.meta.once.err
+	}
+	_, err := r.meta.once.file.ReadAt(p, off)
+	return err
+}
+
+// ReadAt is part of the objstorage.Readable interface.
+func (r *coldReadableWithHotMeta) ReadAt(ctx context.Context, p []byte, off int64) error {
+	// We don't expect reads that span both regions, but in that case it is
+	// correct to read it all from the cold file (which contains all the data).
+	if off < r.meta.startOffset {
+		return r.cold.ReadAt(ctx, p, off)
+	}
+	return r.readMetaAt(p, off-r.meta.startOffset)
+}
+
+// Close is part of the objstorage.Readable interface.
+func (r *coldReadableWithHotMeta) Close() error {
+	err := r.cold.Close()
+	if r.meta.once.file != nil {
+		err = firstError(err, r.meta.once.file.Close())
+		r.meta.once.file = nil
+	}
+	return err
+}
+
+// Size is part of the objstorage.Readable interface.
+func (r *coldReadableWithHotMeta) Size() int64 {
+	return r.cold.Size()
+}
+
+// NewReadHandle is part of the objstorage.Readable interface.
+func (r *coldReadableWithHotMeta) NewReadHandle(
+	readBeforeSize objstorage.ReadBeforeSize,
+) objstorage.ReadHandle {
+	// The readBeforeSize is used to optimize reading the metadata suffix of a
+	// file, for cases where small reads are expensive. In our case, the metadata
+	// is specifically in a hot tier file, for which small reads are cheap (so we
+	// ignore readBeforeSize, like fileReadable).
+	//
+	// Since we are using the local file for metadata, we pass NoReadBefore for
+	// the cold file.
+	return &coldReadHandle{
+		r:    r,
+		cold: r.cold.NewReadHandle(objstorage.NoReadBefore),
+	}
+}
+
+type coldReadHandle struct {
+	r    *coldReadableWithHotMeta
+	cold objstorage.ReadHandle
+}
+
+var _ objstorage.ReadHandle = (*coldReadHandle)(nil)
+
+// ReadAt is part of the objstorage.ReadHandle interface.
+func (rh *coldReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error {
+	if off < rh.r.meta.startOffset {
+		// Read from cold storage only.
+		return rh.cold.ReadAt(ctx, p, off)
+	}
+	// Read from metadata only.
+	return rh.r.readMetaAt(p, off-rh.r.meta.startOffset)
+}
+
+// Close is part of the objstorage.ReadHandle interface.
+func (rh *coldReadHandle) Close() error {
+	return rh.cold.Close()
+}
+
+// SetupForCompaction is part of the objstorage.ReadHandle interface.
+func (rh *coldReadHandle) SetupForCompaction() {
+	rh.cold.SetupForCompaction()
+}
+
+// RecordCacheHit is part of the objstorage.ReadHandle interface.
+func (rh *coldReadHandle) RecordCacheHit(ctx context.Context, offset, size int64) {
+	// We don't use prefetching for the metadata portion, so we only need to
+	// report cache hits to the cold readable.
+	if offset < rh.r.meta.startOffset {
+		rh.cold.RecordCacheHit(ctx, offset, min(size, rh.r.meta.startOffset-offset))
+	}
+}