store: order-preserving varint key encoding (#5771)

CHANGELOG_PENDING.md

@@ -38,6 +38,9 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 
 - Blockchain Protocol
 
+- Data Storage
+  - [store/state/evidence/light] \#5771 Use an order-preserving varint key encoding (@cmwaters)
+
 ### FEATURES
 
 ### IMPROVEMENTS

evidence/pool.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	gogotypes "github.com/gogo/protobuf/types"
+	"github.com/google/orderedcode"
 	dbm "github.com/tendermint/tm-db"
 
 	clist "github.com/tendermint/tendermint/libs/clist"
@@ -21,8 +22,9 @@ import (
 )
 
 const (
-	baseKeyCommitted = byte(0x00)
-	baseKeyPending   = byte(0x01)
+	// prefixes are unique across all tm db's
+	prefixCommitted = int64(8)
+	prefixPending   = int64(9)
 )
 
 // Pool maintains a pool of valid evidence to be broadcasted and committed
@@ -67,7 +69,7 @@ func NewPool(evidenceDB dbm.DB, stateDB sm.Store, blockStore BlockStore) (*Pool,
 	// if pending evidence already in db, in event of prior failure, then check for expiration,
 	// update the size and load it back to the evidenceList
 	pool.pruningHeight, pool.pruningTime = pool.removeExpiredPendingEvidence()
-	evList, _, err := pool.listEvidence(baseKeyPending, -1)
+	evList, _, err := pool.listEvidence(prefixPending, -1)
 	if err != nil {
 		return nil, err
 	}
@@ -84,7 +86,7 @@ func (evpool *Pool) PendingEvidence(maxBytes int64) ([]types.Evidence, int64) {
 	if evpool.Size() == 0 {
 		return []types.Evidence{}, 0
 	}
-	evidence, size, err := evpool.listEvidence(baseKeyPending, maxBytes)
+	evidence, size, err := evpool.listEvidence(prefixPending, maxBytes)
 	if err != nil {
 		evpool.logger.Error("Unable to retrieve pending evidence", "err", err)
 	}
@@ -402,19 +404,20 @@ func (evpool *Pool) markEvidenceAsCommitted(evidence types.EvidenceList) {
 
 // listEvidence retrieves lists evidence from oldest to newest within maxBytes.
 // If maxBytes is -1, there's no cap on the size of returned evidence.
-func (evpool *Pool) listEvidence(prefixKey byte, maxBytes int64) ([]types.Evidence, int64, error) {
+func (evpool *Pool) listEvidence(prefixKey int64, maxBytes int64) ([]types.Evidence, int64, error) {
 	var (
 		evSize    int64
 		totalSize int64
 		evidence  []types.Evidence
 		evList    tmproto.EvidenceList // used for calculating the bytes size
 	)
 
-	iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{prefixKey})
+	iter, err := dbm.IteratePrefix(evpool.evidenceStore, prefixToBytes(prefixKey))
 	if err != nil {
 		return nil, totalSize, fmt.Errorf("database error: %v", err)
 	}
 	defer iter.Close()
+
 	for ; iter.Valid(); iter.Next() {
 		var evpb tmproto.Evidence
 		err := evpb.Unmarshal(iter.Value())
@@ -446,7 +449,7 @@ func (evpool *Pool) listEvidence(prefixKey byte, maxBytes int64) ([]types.Eviden
 }
 
 func (evpool *Pool) removeExpiredPendingEvidence() (int64, time.Time) {
-	iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{baseKeyPending})
+	iter, err := dbm.IteratePrefix(evpool.evidenceStore, prefixToBytes(prefixPending))
 	if err != nil {
 		evpool.logger.Error("Unable to iterate over pending evidence", "err", err)
 		return evpool.State().LastBlockHeight, evpool.State().LastBlockTime
@@ -511,19 +514,28 @@ func evMapKey(ev types.Evidence) string {
 	return string(ev.Hash())
 }
 
-// big endian padded hex
-func bE(h int64) string {
-	return fmt.Sprintf("%0.16X", h)
+func prefixToBytes(prefix int64) []byte {
+	key, err := orderedcode.Append(nil, prefix)
+	if err != nil {
+		panic(err)
+	}
+	return key
 }
 
 func keyCommitted(evidence types.Evidence) []byte {
-	return append([]byte{baseKeyCommitted}, keySuffix(evidence)...)
+	var height int64 = evidence.Height()
+	key, err := orderedcode.Append(nil, prefixCommitted, height, string(evidence.Hash()))
+	if err != nil {
+		panic(err)
+	}
+	return key
 }
 
 func keyPending(evidence types.Evidence) []byte {
-	return append([]byte{baseKeyPending}, keySuffix(evidence)...)
-}
-
-func keySuffix(evidence types.Evidence) []byte {
-	return []byte(fmt.Sprintf("%s/%X", bE(evidence.Height()), evidence.Hash()))
+	var height int64 = evidence.Height()
+	key, err := orderedcode.Append(nil, prefixPending, height, string(evidence.Hash()))
+	if err != nil {
+		panic(err)
+	}
+	return key
 }

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/go-logfmt/logfmt v0.5.0
 	github.com/gogo/protobuf v1.3.1
 	github.com/golang/protobuf v1.4.3
+	github.com/google/orderedcode v0.0.1
 	github.com/gorilla/websocket v1.4.2
 	github.com/gtank/merlin v0.1.1
 	github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff

go.sum

@@ -221,6 +221,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/orderedcode v0.0.1 h1:UzfcAexk9Vhv8+9pNOgRu41f16lHq725vPwnSeiG/Us=
+github.com/google/orderedcode v0.0.1/go.mod h1:iVyU4/qPKHY5h/wSd6rZZCDcLJNxiWO6dvsYES2Sb20=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=

light/store/db/db.go

@@ -3,9 +3,8 @@ package db
 import (
 	"encoding/binary"
 	"fmt"
-	"regexp"
-	"strconv"
 
+	"github.com/google/orderedcode"
 	dbm "github.com/tendermint/tm-db"
 
 	tmsync "github.com/tendermint/tendermint/libs/sync"
@@ -14,8 +13,9 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-var (
-	sizeKey = []byte("size")
+const (
+	prefixLightBlock = int64(0x0a)
+	prefixSize       = int64(0x0b)
 )
 
 type dbs struct {
@@ -30,13 +30,17 @@ type dbs struct {
 // want to use one DB with many light clients).
 func New(db dbm.DB, prefix string) store.Store {
 
+	lightStore := &dbs{db: db, prefix: prefix}
+
+	// retrieve the size of the db
 	size := uint16(0)
-	bz, err := db.Get(sizeKey)
+	bz, err := lightStore.db.Get(lightStore.sizeKey())
 	if err == nil && len(bz) > 0 {
 		size = unmarshalSize(bz)
 	}
+	lightStore.size = size
 
-	return &dbs{db: db, prefix: prefix, size: size}
+	return lightStore
 }
 
 // SaveLightBlock persists LightBlock to the db.
@@ -65,7 +69,7 @@ func (s *dbs) SaveLightBlock(lb *types.LightBlock) error {
 	if err = b.Set(s.lbKey(lb.Height), lbBz); err != nil {
 		return err
 	}
-	if err = b.Set(sizeKey, marshalSize(s.size+1)); err != nil {
+	if err = b.Set(s.sizeKey(), marshalSize(s.size+1)); err != nil {
 		return err
 	}
 	if err = b.WriteSync(); err != nil {
@@ -93,7 +97,7 @@ func (s *dbs) DeleteLightBlock(height int64) error {
 	if err := b.Delete(s.lbKey(height)); err != nil {
 		return err
 	}
-	if err := b.Set(sizeKey, marshalSize(s.size-1)); err != nil {
+	if err := b.Set(s.sizeKey(), marshalSize(s.size-1)); err != nil {
 		return err
 	}
 	if err := b.WriteSync(); err != nil {
@@ -147,13 +151,8 @@ func (s *dbs) LastLightBlockHeight() (int64, error) {
 	}
 	defer itr.Close()
 
-	for itr.Valid() {
-		key := itr.Key()
-		_, height, ok := parseLbKey(key)
-		if ok {
-			return height, nil
-		}
-		itr.Next()
+	if itr.Valid() {
+		return s.decodeLbKey(itr.Key())
 	}
 
 	return -1, itr.Error()
@@ -172,13 +171,8 @@ func (s *dbs) FirstLightBlockHeight() (int64, error) {
 	}
 	defer itr.Close()
 
-	for itr.Valid() {
-		key := itr.Key()
-		_, height, ok := parseLbKey(key)
-		if ok {
-			return height, nil
-		}
-		itr.Next()
+	if itr.Valid() {
+		return s.decodeLbKey(itr.Key())
 	}
 
 	return -1, itr.Error()
@@ -202,13 +196,12 @@ func (s *dbs) LightBlockBefore(height int64) (*types.LightBlock, error) {
 	}
 	defer itr.Close()
 
-	for itr.Valid() {
-		key := itr.Key()
-		_, existingHeight, ok := parseLbKey(key)
-		if ok {
-			return s.LightBlock(existingHeight)
+	if itr.Valid() {
+		existingHeight, err := s.decodeLbKey(itr.Key())
+		if err != nil {
+			return nil, err
 		}
-		itr.Next()
+		return s.LightBlock(existingHeight)
 	}
 	if err = itr.Error(); err != nil {
 		return nil, err
@@ -248,11 +241,12 @@ func (s *dbs) Prune(size uint16) error {
 	pruned := 0
 	for itr.Valid() && numToPrune > 0 {
 		key := itr.Key()
-		_, height, ok := parseLbKey(key)
-		if ok {
-			if err = b.Delete(s.lbKey(height)); err != nil {
-				return err
-			}
+		height, err := s.decodeLbKey(key)
+		if err != nil {
+			return err
+		}
+		if err = b.Delete(s.lbKey(height)); err != nil {
+			return err
 		}
 		itr.Next()
 		numToPrune--
@@ -273,7 +267,7 @@ func (s *dbs) Prune(size uint16) error {
 
 	s.size -= uint16(pruned)
 
-	if wErr := s.db.SetSync(sizeKey, marshalSize(s.size)); wErr != nil {
+	if wErr := s.db.SetSync(s.sizeKey(), marshalSize(size)); wErr != nil {
 		return fmt.Errorf("failed to persist size: %w", wErr)
 	}
 
@@ -289,32 +283,39 @@ func (s *dbs) Size() uint16 {
 	return s.size
 }
 
-func (s *dbs) lbKey(height int64) []byte {
-	return []byte(fmt.Sprintf("lb/%s/%020d", s.prefix, height))
+func (s *dbs) sizeKey() []byte {
+	key, err := orderedcode.Append(nil, s.prefix, prefixSize)
+	if err != nil {
+		panic(err)
+	}
+	return key
 }
 
-var keyPattern = regexp.MustCompile(`^(lb)/([^/]*)/([0-9]+)$`)
-
-func parseKey(key []byte) (part string, prefix string, height int64, ok bool) {
-	submatch := keyPattern.FindSubmatch(key)
-	if submatch == nil {
-		return "", "", 0, false
-	}
-	part = string(submatch[1])
-	prefix = string(submatch[2])
-	height, err := strconv.ParseInt(string(submatch[3]), 10, 64)
+func (s *dbs) lbKey(height int64) []byte {
+	key, err := orderedcode.Append(nil, s.prefix, prefixLightBlock, height)
 	if err != nil {
-		return "", "", 0, false
+		panic(err)
 	}
-	ok = true // good!
-	return
+	return key
 }
 
-func parseLbKey(key []byte) (prefix string, height int64, ok bool) {
-	var part string
-	part, prefix, height, ok = parseKey(key)
-	if part != "lb" {
-		return "", 0, false
+func (s *dbs) decodeLbKey(key []byte) (height int64, err error) {
+	var (
+		dbPrefix         string
+		lightBlockPrefix int64
+	)
+	remaining, err := orderedcode.Parse(string(key), &dbPrefix, &lightBlockPrefix, &height)
+	if err != nil {
+		err = fmt.Errorf("failed to parse light block key: %w", err)
+	}
+	if len(remaining) != 0 {
+		err = fmt.Errorf("expected no remainder when parsing light block key but got: %s", remaining)
+	}
+	if lightBlockPrefix != prefixLightBlock {
+		err = fmt.Errorf("expected light block prefix but got: %d", lightBlockPrefix)
+	}
+	if dbPrefix != s.prefix {
+		err = fmt.Errorf("parsed key has a different prefix. Expected: %s, got: %s", s.prefix, dbPrefix)
 	}
 	return
 }

light/store/db/db_test.go

@@ -89,6 +89,9 @@ func Test_LightBlockBefore(t *testing.T) {
 	if assert.NotNil(t, h) {
 		assert.EqualValues(t, 2, h.Height)
 	}
+
+	_, err = dbStore.LightBlockBefore(2)
+	require.Error(t, err)
 }
 
 func Test_Prune(t *testing.T) {