Skip to content
Permalink
Browse files

Introduce Formatter function for formatting user keys

Use the configured format when formatting keys for error messages.

Clarify some comments about the keys passed to Comparer functions.
  • Loading branch information
petermattis committed Sep 5, 2019
1 parent 2010d29 commit 455817cb5a900187197236cef3ef6186964262f7
@@ -9,6 +9,7 @@ import (
"testing"
"time"

"github.com/cockroachdb/pebble/internal/base"
"github.com/cockroachdb/pebble/internal/datadriven"
"github.com/cockroachdb/pebble/vfs"
)
@@ -37,7 +38,7 @@ func TestManualFlush(t *testing.T) {
}

d.mu.Lock()
s := d.mu.versions.currentVersion().DebugString()
s := d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
d.mu.Unlock()
return s

@@ -63,7 +64,7 @@ func TestManualFlush(t *testing.T) {
}

d.mu.Lock()
s := d.mu.versions.currentVersion().DebugString()
s := d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
d.mu.Unlock()
return s

@@ -519,7 +519,7 @@ func TestGetIter(t *testing.T) {
v.Files[tt.level] = append(v.Files[tt.level], meta)
}

err := v.CheckOrdering(cmp)
err := v.CheckOrdering(cmp, base.DefaultFormatter)
if tc.badOrdering && err == nil {
t.Errorf("desc=%q: want bad ordering, got nil error", desc)
continue
@@ -7,20 +7,17 @@ package base
import (
"bytes"
"encoding/binary"
"fmt"
"strconv"
"unicode/utf8"
)

// TODO(tbg): introduce a FeasibleKey type to make things clearer.

// Compare returns -1, 0, or +1 depending on whether a is 'less than',
// 'equal to' or 'greater than' b. The two arguments can only be 'equal'
// if their contents are exactly equal. Furthermore, the empty slice
// must be 'less than' any non-empty slice.
//
// TODO(tbg): clarify what keys this needs to compare. It definitely needs to
// compare feasible keys (i.e. any user keys, but also those returned by
// Successor and Separator), as well as keys returned by Split (which are not
// themselves feasible, but are feasible keys with their version suffix
// trimmmed). Anything else?
// Compare returns -1, 0, or +1 depending on whether a is 'less than', 'equal
// to' or 'greater than' b. The two arguments can only be 'equal' if their
// contents are exactly equal. Furthermore, the empty slice must be 'less than'
// any non-empty slice. Compare is used to compare user keys, such as those
// passed as arguments to the various DB methods, as well as those returned
// from Separator, Successor, and Split.
type Compare func(a, b []byte) int

// Equal returns true if a and b are equivalent. For a given Compare,
@@ -38,47 +35,46 @@ type Equal func(a, b []byte) bool
// of the user key prefix in the order that gives the correct ordering.
type AbbreviatedKey func(key []byte) uint64

// Given feasible keys a, b for which Compare(a, b) < 0, Separator returns a
// feasible key k such that:
// Formatter returns a formatter for the user key.
type Formatter func(key []byte) fmt.Formatter

// Separator is used to construct SSTable index blocks. A trivial implementation
// is `return a`, but appending fewer bytes leads to smaller SSTables.
//
// Given keys a, b for which Compare(a, b) < 0, Separator returns a key k such
// that:
//
// 1. Compare(a, k) <= 0, and
// 2. Compare(k, b) < 0.
//
// As a special case, b may be nil in which case the second condition is dropped.
//
// Separator is used to construct SSTable index blocks. A trivial implementation
// is `return a`, but appending fewer bytes leads to smaller SSTables.
//
// For example, if dst, a and b are the []byte equivalents of the strings
// "aqua", "black" and "blue", then the result may be "aquablb".
// Similarly, if the arguments were "aqua", "green" and "", then the result
// may be "aquah".
type Separator func(dst, a, b []byte) []byte

// Given a feasible key a, Successor returns feasible key k such that Compare(k,
// a) >= 0. A simple implementation may return a unchanged. The dst parameter
// may be used to store the returned key, though it is valid to pass a nil. The
// returned key must be feasible.
//
// TODO(tbg) it seems that Successor is just the special case of Separator in
// which b is nil. Can we remove this?
// Successor returns a shortened key given a key a, such that Compare(k, a) >=
// 0. A simple implementation may return a unchanged. The dst parameter may be
// used to store the returned key, though it is valid to pass nil. The returned
// key must be valid to pass to Compare.
type Successor func(dst, a []byte) []byte

// Split returns the length of the prefix of the user key that corresponds to
// the key portion of an MVCC encoding scheme to enable the use of prefix bloom
// filters.
//
// The method will only ever be called with feasible keys, that is, keys that
// the user could potentially store in the database. Typically this means
// that the method must only handle valid MVCC encoded keys and should panic
// on any other input.
// The method will only ever be called with valid MVCC keys, that is, keys that
// the user could potentially store in the database. Typically this means that
// the method must only handle valid MVCC encoded keys and should panic on any
// other input.
//
// A trivial MVCC scheme is one in which Split() returns len(a). This
// corresponds to assigning a constant version to each key in the database. For
// performance reasons, it is preferable to use a `nil` split in this case.
//
// The returned prefix must have the following properties (where a and b are
// feasible):
// The returned prefix must have the following properties:
//
// 1) bytes.HasPrefix(a, prefix(a))
// 2) Compare(prefix(a), a) <= 0,
@@ -92,6 +88,7 @@ type Comparer struct {
Compare Compare
Equal Equal
AbbreviatedKey AbbreviatedKey
Format Formatter
Separator Separator
Split Split
Successor Successor
@@ -104,6 +101,12 @@ type Comparer struct {
Name string
}

// DefaultFormatter is the default implementation of user key formatting:
// non-ASCII data is formatted as escaped hexadecimal values.
var DefaultFormatter = func(key []byte) fmt.Formatter {
return FormatBytes(key)
}

// DefaultComparer is the default implementation of the Comparer interface.
// It uses the natural ordering, consistent with bytes.Compare.
var DefaultComparer = &Comparer{
@@ -122,6 +125,8 @@ var DefaultComparer = &Comparer{
return v << uint(8*(8-len(key)))
},

Format: DefaultFormatter,

Separator: func(dst, a, b []byte) []byte {
i, n := SharedPrefixLen(a, b), len(dst)
dst = append(dst, a...)
@@ -156,14 +161,15 @@ var DefaultComparer = &Comparer{
return dst
},

Successor: func(dst, a []byte) []byte {
Successor: func(dst, a []byte) (ret []byte) {
for i := 0; i < len(a); i++ {
if a[i] != 0xff {
dst = append(dst, a[:i+1]...)
dst[len(dst)-1]++
return dst
}
}
// a is a run of 0xffs, leave it alone.
return append(dst, a...)
},

@@ -184,3 +190,24 @@ func SharedPrefixLen(a, b []byte) int {
}
return i
}

// FormatBytes formats a byte slice using hexadecimal escapes for non-ASCII
// data.
type FormatBytes []byte

const lowerhex = "0123456789abcdef"

// Format implements the fmt.Formatter interface.
func (p FormatBytes) Format(s fmt.State, c rune) {
buf := make([]byte, 0, len(p))
for _, b := range p {
if b < utf8.RuneSelf && strconv.IsPrint(rune(b)) {
buf = append(buf, b)
continue
}
buf = append(buf, `\x`...)
buf = append(buf, lowerhex[byte(b)>>4])
buf = append(buf, lowerhex[byte(b)&0xF])
}
s.Write(buf)
}
@@ -302,10 +302,19 @@ func (k InternalKey) Clone() InternalKey {

// String returns a string representation of the key.
func (k InternalKey) String() string {
return fmt.Sprintf("%s#%d,%d", k.UserKey, k.SeqNum(), k.Kind())
return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())
}

// Pretty returns a pretty-printed string representation of the key.
func (k InternalKey) Pretty(f func([]byte) string) string {
return fmt.Sprintf("%s#%d,%d", f(k.UserKey), k.SeqNum(), k.Kind())
// Pretty returns a formatter for the key.
func (k InternalKey) Pretty(f Formatter) fmt.Formatter {
return prettyInternalKey{k, f}
}

type prettyInternalKey struct {
InternalKey
formatter Formatter
}

func (k prettyInternalKey) Format(s fmt.State, c rune) {
fmt.Fprintf(s, "%s#%d,%d", k.formatter(k.UserKey), k.SeqNum(), k.Kind())
}
@@ -163,6 +163,11 @@ type Version struct {
}

func (v *Version) String() string {
return v.Pretty(base.DefaultFormatter)
}

// Pretty returns a string representation of the version.
func (v *Version) Pretty(format base.Formatter) string {
var buf bytes.Buffer
for level := 0; level < NumLevels; level++ {
if len(v.Files[level]) == 0 {
@@ -171,7 +176,7 @@ func (v *Version) String() string {
fmt.Fprintf(&buf, "%d:", level)
for j := range v.Files[level] {
f := &v.Files[level][j]
fmt.Fprintf(&buf, " %s-%s", f.Smallest.UserKey, f.Largest.UserKey)
fmt.Fprintf(&buf, " %s-%s", format(f.Smallest.UserKey), format(f.Largest.UserKey))
}
fmt.Fprintf(&buf, "\n")
}
@@ -180,7 +185,7 @@ func (v *Version) String() string {

// DebugString returns an alternative format to String() which includes
// sequence number and kind information for the sstable boundaries.
func (v *Version) DebugString() string {
func (v *Version) DebugString(format base.Formatter) string {
var buf bytes.Buffer
for level := 0; level < NumLevels; level++ {
if len(v.Files[level]) == 0 {
@@ -189,7 +194,7 @@ func (v *Version) DebugString() string {
fmt.Fprintf(&buf, "%d:", level)
for j := range v.Files[level] {
f := &v.Files[level][j]
fmt.Fprintf(&buf, " %s-%s", f.Smallest, f.Largest)
fmt.Fprintf(&buf, " %s-%s", f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
fmt.Fprintf(&buf, "\n")
}
@@ -316,7 +321,7 @@ func (v *Version) Overlaps(
// CheckOrdering checks that the files are consistent with respect to
// increasing file numbers (for level 0 files) and increasing and non-
// overlapping internal key ranges (for level non-0 files).
func (v *Version) CheckOrdering(cmp Compare) error {
func (v *Version) CheckOrdering(cmp Compare, format base.Formatter) error {
for level, ff := range v.Files {
if level == 0 {
for i := 1; i < len(ff); i++ {
@@ -337,11 +342,11 @@ func (v *Version) CheckOrdering(cmp Compare) error {
f := &ff[i]
if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 {
return fmt.Errorf("level non-0 files are not in increasing ikey order: %s, %s\n%s",
prev.Largest, f.Smallest, v.DebugString())
prev.Largest.Pretty(format), f.Smallest.Pretty(format), v.DebugString(format))
}
if base.InternalCompare(cmp, f.Smallest, f.Largest) > 0 {
return fmt.Errorf("level non-0 file has inconsistent bounds: %s, %s",
f.Smallest, f.Largest)
f.Smallest.Pretty(format), f.Largest.Pretty(format))
}
}
}
@@ -405,7 +405,7 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) {
//
// base may be nil, which is equivalent to a pointer to a zero version.
func (b *BulkVersionEdit) Apply(
opts *Options, base *Version, cmp Compare,
base *Version, cmp Compare, format base.Formatter,
) (*Version, error) {
v := new(Version)
for level := range v.Files {
@@ -460,7 +460,7 @@ func (b *BulkVersionEdit) Apply(
SortBySmallest(v.Files[level], cmp)
}
}
if err := v.CheckOrdering(cmp); err != nil {
if err := v.CheckOrdering(cmp, format); err != nil {
return nil, fmt.Errorf("pebble: internal error: %v", err)
}
return v, nil
@@ -40,3 +40,20 @@ func (t Tombstone) String() string {
}
return fmt.Sprintf("%s-%s#%d", t.Start.UserKey, t.End, t.Start.SeqNum())
}

// Pretty returns a formatter for the tombstone.
func (t Tombstone) Pretty(f base.Formatter) fmt.Formatter {
return prettyTombstone{t, f}
}

type prettyTombstone struct {
Tombstone
formatter base.Formatter
}

func (t prettyTombstone) Format(s fmt.State, c rune) {
if t.Empty() {
fmt.Fprintf(s, "<empty>")
}
fmt.Fprintf(s, "%s-%s#%d", t.formatter(t.Start.UserKey), t.formatter(t.End), t.Start.SeqNum())
}
@@ -409,7 +409,7 @@ func TestIteratorTableFilter(t *testing.T) {
d.mu.Lock()
// Disable the "dynamic base level" code for this test.
d.mu.versions.picker.baseLevel = 1
s := d.mu.versions.currentVersion().DebugString()
s := d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
d.mu.Unlock()
return s

@@ -137,7 +137,7 @@ func TestRangeDelCompactionTruncation(t *testing.T) {

lsm := func() string {
d.mu.Lock()
s := d.mu.versions.currentVersion().DebugString()
s := d.mu.versions.currentVersion().DebugString(base.DefaultFormatter)
d.mu.Unlock()
return s
}

0 comments on commit 455817c

Please sign in to comment.
You can’t perform that action at this time.