tool: separate blob mode flags

This allows us to specify a relative path for our scan
tool instead of enforcing some rule about the order of arguments to
scan.

Author: annrpom

sstable/values.go

@@ -35,7 +35,16 @@ var DebugHandlesBlobContext = TableBlobContext{
 			InlineHandlePreface: preface,
 			HandleSuffix:        handleSuffix,
 		}
-		return base.MakeInPlaceValue([]byte(ih.String()))
+		// Prepend "blob-value:" to the handle to help identify blob values
+		// from inline values later on (we want to be able to format the former
+		// differently during value formatting).
+		//
+		// TODO(annie): Revisit this once we support determining the type
+		// of value we are dealing with in our sstable internal iterator.
+		// We can use some information about the ValuePrefix to separate
+		// how we want to format in-place values and blob value handles
+		// (via a new formatter) instead of prepending "blob-value:".
+		return base.MakeInPlaceValue([]byte("blob-value:" + ih.String()))
 	},
 }
 

tool/find.go

@@ -56,7 +56,7 @@ type findT struct {
 	fmtKey       keyFormatter
 	fmtValue     valueFormatter
 	verbose      bool
-	blobMode     string
+	blobModeLoad bool
 
 	// Map from file num to version edit index which references the file num.
 	editRefs map[base.DiskFileNum][]int
@@ -121,8 +121,8 @@ provenance of the sstables (flushed, ingested, compacted).
 		&f.fmtKey, "key", "key formatter")
 	f.Root.Flags().Var(
 		&f.fmtValue, "value", "value formatter")
-	f.Root.Flags().StringVar(
-		&f.blobMode, "blob-mode", "none", "blob value formatter")
+	f.Root.Flags().BoolVar(
+		&f.blobModeLoad, "load-blobs", false, "load values from blob file when encountered")
 	return f
 }
 
@@ -483,16 +483,9 @@ func (f *findT) searchTables(stdout io.Writer, searchKey []byte, refs []findRef)
 				fragTransforms = m.FragmentIterTransforms()
 			}
 
-			var blobContext sstable.TableBlobContext
-			switch ConvertToBlobRefMode(f.blobMode) {
-			case BlobRefModePrint:
-				f.fmtValue.mustSet("[%s]")
-				blobContext = sstable.DebugHandlesBlobContext
-			case BlobRefModeLoad:
-				f.fmtValue.mustSet("[%s]")
+			blobContext := sstable.DebugHandlesBlobContext
+			if f.blobModeLoad {
 				blobContext = f.blobMappings.LoadValueBlobContext(base.PhysicalTableFileNum(fl.DiskFileNum))
-			default:
-				blobContext = sstable.AssertNoBlobHandles
 			}
 			iter, err := r.NewIter(transforms, nil, nil, blobContext)
 			if err != nil {

tool/sstable.go

@@ -44,14 +44,14 @@ type sstableT struct {
 	mergers   sstable.Mergers
 
 	// Flags.
-	fmtKey   keyFormatter
-	fmtValue valueFormatter
-	start    key
-	end      key
-	filter   key
-	count    int64
-	verbose  bool
-	blobMode string
+	fmtKey       keyFormatter
+	fmtValue     valueFormatter
+	start        key
+	end          key
+	filter       key
+	count        int64
+	verbose      bool
+	blobModeLoad string
 }
 
 func newSSTable(
@@ -103,9 +103,6 @@ properties are pretty-printed or displayed in a verbose/raw format.
 Print the records in the sstables. The sstables are scanned in command line
 order which means the records will be printed in that order. Raw range
 tombstones are displayed interleaved with point records.
-
-When --blob-mode=load is specified, the path to a directory containing a
-manifest and blob file must be provided as the last argument.
 `,
 		Args: cobra.MinimumNArgs(1),
 		Run:  s.runScan,
@@ -145,7 +142,8 @@ inclusive-inclusive range specified by --start and --end.
 	s.Scan.Flags().Int64Var(
 		&s.count, "count", 0, "key count for scan (0 is unlimited)")
 	s.Scan.Flags().StringVar(
-		&s.blobMode, "blob-mode", "none", "blob value formatter")
+		&s.blobModeLoad, "load-blobs-from", "", "relative path to a directory containing a manifest and "+
+			"blob file to load values from")
 
 	return s
 }
@@ -339,25 +337,14 @@ func (s *sstableT) runProperties(cmd *cobra.Command, args []string) {
 
 func (s *sstableT) runScan(cmd *cobra.Command, args []string) {
 	stdout, stderr := cmd.OutOrStdout(), cmd.OutOrStderr()
-	// If in blob load mode, the last argument is the path to our directory
-	// containing the manifest(s) and blob file(s).
-	blobMode := ConvertToBlobRefMode(s.blobMode)
-	var blobDir string
 	var blobMappings *blobFileMappings
-	if blobMode == BlobRefModeLoad {
-		if len(args) < 2 {
-			fmt.Fprintf(stderr, "when --blob-mode=load is specified, the path to a "+
-				"directory containing a manifest and blob file must be provided as the last argument")
-			return
-		}
-		blobDir = args[len(args)-1]
-		args = args[:len(args)-1]
-		manifests, err := findManifests(stderr, s.opts.FS, blobDir)
+	if s.blobModeLoad != "" {
+		manifests, err := findManifests(stderr, s.opts.FS, s.blobModeLoad)
 		if err != nil {
 			fmt.Fprintf(stderr, "%s\n", err)
 			return
 		}
-		blobMappings, err = newBlobFileMappings(stderr, s.opts.FS, blobDir, manifests)
+		blobMappings, err = newBlobFileMappings(stderr, s.opts.FS, s.blobModeLoad, manifests)
 		if err != nil {
 			fmt.Fprintf(stderr, "%s\n", err)
 			return
@@ -379,22 +366,15 @@ func (s *sstableT) runScan(cmd *cobra.Command, args []string) {
 			prefix = fmt.Sprintf("%s: ", path)
 		}
 
-		var blobContext sstable.TableBlobContext
-		switch blobMode {
-		case BlobRefModePrint:
-			s.fmtValue.mustSet("[%s]")
-			blobContext = sstable.DebugHandlesBlobContext
-		case BlobRefModeLoad:
+		blobContext := sstable.DebugHandlesBlobContext
+		if s.blobModeLoad != "" {
 			// If the file number is unset, we are likely trying to read a
 			// non numerically named file.
 			if r.BlockReader().FileNum() == 0 {
 				fmt.Fprintf(stderr, "unset file in path %s\n", path)
 				return
 			}
-			s.fmtValue.mustSet("[%s]")
 			blobContext = blobMappings.LoadValueBlobContext(base.PhysicalTableFileNum(r.BlockReader().FileNum()))
-		default:
-			blobContext = sstable.AssertNoBlobHandles
 		}
 		iter, err := r.NewIter(sstable.NoTransforms, nil, s.end, blobContext)
 		if err != nil {

tool/testdata/find

@@ -142,15 +142,13 @@ Unable to decode sstable find-mixed/000001.sst, pebble/table: invalid table 0000
 find
 testdata/find-db
 eee
---blob-mode=print
 ----
 000004.log
     bbb-eee#19,RANGEDEL
 
 find
 testdata/find-val-sep-db
 crdb:aaa
---blob-mode=print
 ----
 000005.sst [aaa\x00#10,SET-ddd\x00#13,SET]
     (flushed to L0)
@@ -159,48 +157,46 @@ crdb:aaa
 find
 testdata/find-val-sep-db
 crdb:eee
---blob-mode=print
 ----
 000004.log
-    eee\x00#14,SET [pigeon]
+    eee\x00#14,SET [706967656f6e]
 000008.sst [eee\x00#14,SET-fff\x00#15,SET]
     (flushed to L0)
     eee\x00#14,SET [(f0,blk0,id0,len6)]
 
 find
 testdata/find-val-sep-db
 crdb:ddd
---blob-mode=print
 ----
 000005.sst [aaa\x00#10,SET-ddd\x00#13,SET]
     (flushed to L0)
-    ddd\x00#13,SET [6]
+    ddd\x00#13,SET [36]
 
 find
 testdata/find-val-sep-db
 crdb:aaa
---blob-mode=load
+--load-blobs
 ----
 000005.sst [aaa\x00#10,SET-ddd\x00#13,SET]
     (flushed to L0)
-    aaa\x00#10,SET [yuumi]
+    aaa\x00#10,SET [7975756d69]
 
 find
 testdata/find-val-sep-db
 crdb:ddd
---blob-mode=load
+--load-blobs
 ----
 000005.sst [aaa\x00#10,SET-ddd\x00#13,SET]
     (flushed to L0)
-    ddd\x00#13,SET [6]
+    ddd\x00#13,SET [36]
 
 find
 testdata/find-val-sep-db
 crdb:eee
---blob-mode=load
+--load-blobs
 ----
 000004.log
-    eee\x00#14,SET [pigeon]
+    eee\x00#14,SET [706967656f6e]
 000008.sst [eee\x00#14,SET-fff\x00#15,SET]
     (flushed to L0)
-    eee\x00#14,SET [pigeon]
+    eee\x00#14,SET [706967656f6e]

tool/testdata/sstable_scan

@@ -415,46 +415,36 @@ tcrsfsqc@1366.233420574,0#0,SET [1b22f2128e75f0433e8eda52c41872e6]
 
 sstable scan
 ./testdata/find-val-sep-db
---blob-mode=print
 ----
 find-val-sep-db/000005.sst
 aaa\x00#10,SET [(f0,blk0,id0,len5)]
 bbb\x00#11,SET [(f0,blk0,id1,len3)]
 ccc\x00#12,SET [(f0,blk0,id2,len10)]
-ddd\x00#13,SET [6]
+ddd\x00#13,SET [36]
 find-val-sep-db/000008.sst
 eee\x00#14,SET [(f0,blk0,id0,len6)]
 fff\x00#15,SET [(f0,blk0,id1,len7)]
 
-sstable scan
---filter=armed
---blob-mode=print
-../sstable/testdata/hamlet-sst/000002.sst
-----
-000002.sst: armed#0,SET [2]
-
 sstable scan
 ./testdata/find-val-sep-db/000005.sst
---blob-mode=load
+--load-blobs-from
 ----
-when --blob-mode=load is specified, the path to a directory containing a manifest and blob file must be provided as the last argument
+flag needs an argument: --load-blobs-from
 
 sstable scan
 ./testdata/find-val-sep-db/000005.sst
-./testdata/find-val-sep-db
---blob-mode=load
+--load-blobs-from=find-val-sep-db
 ----
 000005.sst
-aaa\x00#10,SET [yuumi]
-bbb\x00#11,SET [mai]
-ccc\x00#12,SET [poiandyaya]
-ddd\x00#13,SET [6]
+aaa\x00#10,SET [7975756d69]
+bbb\x00#11,SET [6d6169]
+ccc\x00#12,SET [706f69616e6479617961]
+ddd\x00#13,SET [36]
 
 sstable scan
 ./testdata/find-val-sep-db/000008.sst
-./testdata/find-val-sep-db
---blob-mode=load
+--load-blobs-from=find-val-sep-db
 ----
 000008.sst
-eee\x00#14,SET [pigeon]
-fff\x00#15,SET [chicken]
+eee\x00#14,SET [706967656f6e]
+fff\x00#15,SET [636869636b656e]

tool/tool.go

@@ -200,30 +200,6 @@ func (t *T) ConfigureSharedStorage(
 	t.opts.Experimental.CreateOnSharedLocator = createOnSharedLocator
 }
 
-// BlobRefMode specifies how blob references should be handled.
-type BlobRefMode int
-
-const (
-	// BlobRefModeNone specifies the AssertNoBlobHandles TableBlobContext.
-	BlobRefModeNone BlobRefMode = iota
-	// BlobRefModePrint specifies the DebugHandlesBlobContext TableBlobContext.
-	BlobRefModePrint
-	// BlobRefModeLoad specifies the LoadValBlobContext
-	// TableBlobContext.
-	BlobRefModeLoad
-)
-
-func ConvertToBlobRefMode(s string) BlobRefMode {
-	switch s {
-	case "print":
-		return BlobRefModePrint
-	case "load":
-		return BlobRefModeLoad
-	default:
-		return BlobRefModeNone
-	}
-}
-
 // debugReaderProvider is a cache-less ReaderProvider meant for debugging blob
 // files.
 type debugReaderProvider struct {

tool/util.go

@@ -5,6 +5,7 @@
 package tool
 
 import (
+	"bytes"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -176,6 +177,15 @@ func (f *valueFormatter) Set(spec string) error {
 			return fmtFormatter{f.spec, v}
 		}
 	}
+	fn := f.fn
+	// Format blob values differently from inline values.
+	f.fn = func(k, v []byte) fmt.Formatter {
+		valuePrefix := []byte("blob-value:")
+		if bytes.HasPrefix(v, valuePrefix) {
+			return fmtFormatter{"[%s]", v[len(valuePrefix):]}
+		}
+		return fn(k, v)
+	}
 	return nil
 }