objstorage: support cold tier

This change adds objstorage provider support for a cold tier.

`CreateOptions.UseColdTier` can be used to create cold objects; the
tier becomes part of the `ObjectMetadata`.

Author: RaduBerinde

internal/base/internal.go

@@ -774,5 +774,6 @@ type StorageTier uint8
 
 const (
 	HotTier StorageTier = iota
+	ColdTier
 	NumStorageTiers
 )

objstorage/objstorage.go

@@ -120,6 +120,11 @@ type ObjectMetadata struct {
 	DiskFileNum base.DiskFileNum
 	FileType    base.FileType
 
+	// The fields below are only set if the object is on local storage.
+	Local struct {
+		Tier base.StorageTier
+	}
+
 	// The fields below are only set if the object is on remote storage.
 	Remote struct {
 		// CreatorID identifies the DB instance that originally created the object.
@@ -243,6 +248,11 @@ type CreateOptions struct {
 	// The default (zero) value is SharedRefTracking.
 	SharedCleanupMethod SharedCleanupMethod
 
+	// Tier is the storage tier for this object. If Tier is ColdTier and the
+	// provider has a local cold tier configured (and PreferSharedStorage is
+	// false), the object will be created on the local cold tier.
+	Tier base.StorageTier
+
 	// WriteCategory is used for the object when it is created on local storage
 	// to collect aggregated write metrics for each write source.
 	WriteCategory vfs.DiskWriteCategory

objstorage/objstorageprovider/provider.go

@@ -19,7 +19,6 @@ import (
 	"github.com/cockroachdb/pebble/internal/invariants"
 	"github.com/cockroachdb/pebble/objstorage"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
-	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedcache"
 	"github.com/cockroachdb/pebble/objstorage/remote"
 	"github.com/cockroachdb/pebble/vfs"
@@ -61,6 +60,12 @@ type Settings struct {
 		FS        vfs.FS
 		FSDirName string
 
+		// ColdTier is set only when a secondary "cold" storage tier is to be used.
+		ColdTier struct {
+			FS        vfs.FS
+			FSDirName string
+		}
+
 		// FSDirInitialListing is a listing of FSDirName at the time of calling Open.
 		//
 		// This is an optional optimization to avoid double listing on Open when the
@@ -275,7 +280,7 @@ func (p *provider) OpenForReading(
 
 	var r objstorage.Readable
 	if !meta.IsRemote() {
-		r, err = p.localOpenForReading(ctx, fileType, fileNum, opts)
+		r, err = p.localOpenForReading(ctx, fileType, fileNum, meta.Local.Tier, opts)
 	} else {
 		r, err = p.remoteOpenForReading(ctx, meta, opts)
 		if err != nil && p.isNotExistError(meta, err) {
@@ -312,7 +317,11 @@ func (p *provider) Create(
 		} else {
 			category = vfs.WriteCategoryUnspecified
 		}
-		w, meta, err = p.vfsCreate(ctx, fileType, fileNum, category)
+		tier := opts.Tier
+		if tier == base.ColdTier && p.st.Local.ColdTier.FS == nil {
+			tier = base.HotTier
+		}
+		w, meta, err = p.vfsCreate(ctx, fileType, fileNum, tier, category)
 	}
 	if err != nil {
 		err = errors.Wrapf(err, "creating object %s", fileNum)
@@ -339,7 +348,7 @@ func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) erro
 	}
 
 	if !meta.IsRemote() {
-		err = p.localRemove(fileType, fileNum)
+		err = p.localRemove(fileType, fileNum, meta.Local.Tier)
 	} else {
 		// TODO(radu): implement remote object removal (i.e. deref).
 		err = p.sharedUnref(meta)
@@ -398,25 +407,32 @@ func (p *provider) LinkOrCopyFromLocal(
 	dstFileNum base.DiskFileNum,
 	opts objstorage.CreateOptions,
 ) (objstorage.ObjectMetadata, error) {
+	tier := opts.Tier
+	if tier == base.ColdTier && p.st.Local.ColdTier.FS == nil {
+		tier = base.HotTier
+	}
 	shared := opts.PreferSharedStorage && p.st.Remote.CreateOnShared != remote.CreateOnSharedNone
-	if !shared && srcFS == p.st.Local.FS {
-		// Wrap the normal filesystem with one which wraps newly created files with
-		// vfs.NewSyncingFile.
-		fs := vfs.NewSyncingFS(p.st.Local.FS, vfs.SyncingFileOptions{
-			NoSyncOnClose: p.st.Local.NoSyncOnClose,
-			BytesPerSync:  p.st.Local.BytesPerSync,
-		})
-		dstPath := p.localPath(dstFileType, dstFileNum)
-		if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
-			return objstorage.ObjectMetadata{}, err
-		}
+	if !shared && (srcFS == p.st.Local.FS || srcFS == p.st.Local.ColdTier.FS) {
+		fs, dstPath := p.localPath(dstFileType, dstFileNum, tier)
+		if srcFS == fs {
+			// Wrap the normal filesystem with one which wraps newly created files with
+			// vfs.NewSyncingFile.
+			fs = vfs.NewSyncingFS(fs, vfs.SyncingFileOptions{
+				NoSyncOnClose: p.st.Local.NoSyncOnClose,
+				BytesPerSync:  p.st.Local.BytesPerSync,
+			})
+			if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil {
+				return objstorage.ObjectMetadata{}, err
+			}
 
-		meta := objstorage.ObjectMetadata{
-			DiskFileNum: dstFileNum,
-			FileType:    dstFileType,
+			meta := objstorage.ObjectMetadata{
+				DiskFileNum: dstFileNum,
+				FileType:    dstFileType,
+			}
+			meta.Local.Tier = tier
+			p.addMetadata(meta)
+			return meta, nil
 		}
-		p.addMetadata(meta)
-		return meta, nil
 	}
 	// Create the object and copy the data.
 	w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts)
@@ -479,15 +495,16 @@ func (p *provider) Lookup(
 // Path is part of the objstorage.Provider interface.
 func (p *provider) Path(meta objstorage.ObjectMetadata) string {
 	if !meta.IsRemote() {
-		return p.localPath(meta.FileType, meta.DiskFileNum)
+		_, path := p.localPath(meta.FileType, meta.DiskFileNum, meta.Local.Tier)
+		return path
 	}
 	return p.remotePath(meta)
 }
 
 // Size returns the size of the object.
 func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) {
 	if !meta.IsRemote() {
-		return p.localSize(meta.FileType, meta.DiskFileNum)
+		return p.localSize(meta.FileType, meta.DiskFileNum, meta.Local.Tier)
 	}
 	return p.remoteSize(meta)
 }
@@ -548,20 +565,9 @@ func (p *provider) addMetadataLocked(meta objstorage.ObjectMetadata) {
 	}
 	p.mu.knownObjects[meta.DiskFileNum] = meta
 	if meta.IsRemote() {
-		p.mu.remote.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
-			FileNum:          meta.DiskFileNum,
-			FileType:         meta.FileType,
-			CreatorID:        meta.Remote.CreatorID,
-			CreatorFileNum:   meta.Remote.CreatorFileNum,
-			Locator:          meta.Remote.Locator,
-			CleanupMethod:    meta.Remote.CleanupMethod,
-			CustomObjectName: meta.Remote.CustomObjectName,
-		})
-		if meta.IsExternal() {
-			p.mu.remote.addExternalObject(meta)
-		}
+		p.mu.remote.addObject(meta)
 	} else {
-		p.mu.local.objChangeCounter++
+		p.mu.local.objChanged(meta)
 	}
 }
 
@@ -574,13 +580,10 @@ func (p *provider) removeMetadata(fileNum base.DiskFileNum) {
 		return
 	}
 	delete(p.mu.knownObjects, fileNum)
-	if meta.IsExternal() {
-		p.mu.remote.removeExternalObject(meta)
-	}
 	if meta.IsRemote() {
-		p.mu.remote.catalogBatch.DeleteObject(fileNum)
+		p.mu.remote.removeObject(meta)
 	} else {
-		p.mu.local.objChangeCounter++
+		p.mu.local.objChanged(meta)
 	}
 }
 

objstorage/objstorageprovider/provider_test.go

@@ -63,7 +63,7 @@ func TestProvider(t *testing.T) {
 			log.Reset()
 			switch d.Cmd {
 			case "open":
-				var fsDir string
+				var fsDir, coldDir string
 				var creatorID objstorage.CreatorID
 				d.CmdArgs = slices.DeleteFunc(d.CmdArgs, func(arg datadriven.CmdArg) bool {
 					switch arg.Key {
@@ -72,19 +72,29 @@ func TestProvider(t *testing.T) {
 						arg.Scan(t, 0, &id)
 						creatorID = objstorage.CreatorID(id)
 						return true
+					case "cold-tier":
+						coldDir = arg.SingleVal(t)
+						return true
 					}
 					return false
 				})
 				scanArgs("<fs-dir> [creator-id=X]", &fsDir)
 
+				require.NoError(t, fs.MkdirAll(fsDir, 0755))
 				st := DefaultSettings(fs, fsDir)
+				if coldDir != "" {
+					st.Local.ColdTier.FS = fs
+					st.Local.ColdTier.FSDirName = coldDir
+					require.NoError(t, fs.MkdirAll(coldDir, 0755))
+				}
 				if creatorID != 0 {
 					st.Remote.StorageFactory = sharedFactory
 					st.Remote.CreateOnShared = remote.CreateOnSharedAll
 					st.Remote.CreateOnSharedLocator = ""
 				}
 				st.Local.ReadaheadConfig = readaheadConfig
-				require.NoError(t, fs.MkdirAll(fsDir, 0755))
+				if coldDir != "" {
+				}
 				p, err := Open(st)
 				require.NoError(t, err)
 				if creatorID != 0 {
@@ -122,18 +132,24 @@ func TestProvider(t *testing.T) {
 					SharedCleanupMethod: objstorage.SharedRefTracking,
 				}
 				ft := base.FileTypeTable
-				if len(d.CmdArgs) > 0 && d.CmdArgs[0].Key == "file-type" {
-					ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t))
-					d.CmdArgs = d.CmdArgs[1:]
-				}
-				if len(d.CmdArgs) == 5 && d.CmdArgs[4].Key == "no-ref-tracking" {
-					d.CmdArgs = d.CmdArgs[:4]
-					opts.SharedCleanupMethod = objstorage.SharedNoCleanup
-				}
+				d.CmdArgs = slices.DeleteFunc(d.CmdArgs, func(arg datadriven.CmdArg) bool {
+					switch arg.Key {
+					case "file-type":
+						ft = base.FileTypeFromName(d.CmdArgs[0].FirstVal(t))
+						return true
+					case "no-ref-tracking":
+						opts.SharedCleanupMethod = objstorage.SharedNoCleanup
+						return true
+					case "cold-tier":
+						opts.Tier = base.ColdTier
+						return true
+					}
+					return false
+				})
 				var fileNum base.DiskFileNum
 				var typ string
 				var salt, size int
-				scanArgs("[file-type=sstable|blob] <file-num> <local|shared> <salt> <size> [no-ref-tracking]", &fileNum, &typ, &salt, &size)
+				scanArgs("[file-type=sstable|blob] <file-num> <local|shared> <salt> <size> [no-ref-tracking] [cold-tier]", &fileNum, &typ, &salt, &size)
 				switch typ {
 				case "local":
 				case "shared":

objstorage/objstorageprovider/remote.go

@@ -56,6 +56,28 @@ type remoteLockedState struct {
 	externalObjects map[remote.ObjectKey][]base.DiskFileNum
 }
 
+func (rs *remoteLockedState) addObject(meta objstorage.ObjectMetadata) {
+	rs.catalogBatch.AddObject(remoteobjcat.RemoteObjectMetadata{
+		FileNum:          meta.DiskFileNum,
+		FileType:         meta.FileType,
+		CreatorID:        meta.Remote.CreatorID,
+		CreatorFileNum:   meta.Remote.CreatorFileNum,
+		Locator:          meta.Remote.Locator,
+		CleanupMethod:    meta.Remote.CleanupMethod,
+		CustomObjectName: meta.Remote.CustomObjectName,
+	})
+	if meta.IsExternal() {
+		rs.addExternalObject(meta)
+	}
+}
+
+func (rs *remoteLockedState) removeObject(meta objstorage.ObjectMetadata) {
+	if meta.IsExternal() {
+		rs.removeExternalObject(meta)
+	}
+	rs.catalogBatch.DeleteObject(meta.DiskFileNum)
+}
+
 func (rs *remoteLockedState) addExternalObject(meta objstorage.ObjectMetadata) {
 	if rs.externalObjects == nil {
 		rs.externalObjects = make(map[remote.ObjectKey][]base.DiskFileNum)

objstorage/objstorageprovider/testdata/provider/cold_tier

@@ -0,0 +1,84 @@
+open p1 cold-tier=cold1
+----
+<local fs> mkdir-all: p1 0755
+<local fs> mkdir-all: cold1 0755
+<local fs> open-dir: p1
+<local fs> open-dir: cold1
+
+# Create a cold file.
+create file-type=blob 1 local 1 1024 cold-tier
+----
+<local fs> create: cold1/000001.blob
+<local fs> sync-data: cold1/000001.blob
+<local fs> close: cold1/000001.blob
+
+read file-type=blob 1
+0 500
+512 1024
+----
+<local fs> open: cold1/000001.blob (options: *vfs.randomReadsOption)
+size: 1024
+<local fs> read-at(0, 500): cold1/000001.blob
+0 500: ok (salt 1)
+<local fs> read-at(512, 1024): cold1/000001.blob
+512 1024: EOF
+<local fs> close: cold1/000001.blob
+
+create 2 local 2 1024
+----
+<local fs> create: p1/000002.sst
+<local fs> sync-data: p1/000002.sst
+<local fs> close: p1/000002.sst
+
+read 2
+0 500
+512 1024
+----
+<local fs> open: p1/000002.sst (options: *vfs.randomReadsOption)
+size: 1024
+<local fs> read-at(0, 500): p1/000002.sst
+0 500: ok (salt 2)
+<local fs> read-at(512, 1024): p1/000002.sst
+512 1024: EOF
+<local fs> close: p1/000002.sst
+
+list
+----
+000001 -> cold1/000001.blob
+000002 -> p1/000002.sst
+
+remove file-type=blob 1
+----
+<local fs> remove: cold1/000001.blob
+
+remove 2
+----
+<local fs> remove: p1/000002.sst
+
+# Verify that we can request cold tier even if it is not configured.
+open p2
+----
+<local fs> mkdir-all: p2 0755
+<local fs> open-dir: p2
+
+create file-type=blob 3 local 3 1024 cold-tier
+----
+<local fs> create: p2/000003.blob
+<local fs> sync-data: p2/000003.blob
+<local fs> close: p2/000003.blob
+
+read file-type=blob 3
+0 500
+512 1024
+----
+<local fs> open: p2/000003.blob (options: *vfs.randomReadsOption)
+size: 1024
+<local fs> read-at(0, 500): p2/000003.blob
+0 500: ok (salt 3)
+<local fs> read-at(512, 1024): p2/000003.blob
+512 1024: EOF
+<local fs> close: p2/000003.blob
+
+remove file-type=blob 3
+----
+<local fs> remove: p2/000003.blob