feat(orm): add ormkv.EntryCodec and ormkv.IndexCodec's (cosmos#10647)

* feat(orm): add KeyCodec

* WIP

* code coverage

* add DefaultValue test

* fix range key check

* revert DefaultValue

* fix range check

* feat(orm): add ormkv.Codec's

* WIP

* add UniqueKeyCodec

* add IndexKeyCodec

* fixes

* add SeqCodec

* add doc comments

* test fields

* refactor field names

* Update orm/encoding/ormkv/index_key.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/index_key.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/index_key.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update orm/encoding/ormkv/unique_key.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* add tests for entry strings

* address review comments

* fix non-deterministic string rendering and tests

* Update x/auth/middleware/priority_test.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

* Update x/auth/middleware/priority_test.go

Co-authored-by: Tyler <48813565+technicallyty@users.noreply.github.com>

orm/encoding/ormkv/codec.go

@@ -0,0 +1,28 @@
+package ormkv
+
+import "google.golang.org/protobuf/reflect/protoreflect"
+
+// EntryCodec defines an interfaces for decoding and encoding entries in the
+// kv-store backing an ORM instance. EntryCodec's enable full logical decoding
+// of ORM data.
+type EntryCodec interface {
+
+	// DecodeEntry decodes a kv-pair into an Entry.
+	DecodeEntry(k, v []byte) (Entry, error)
+
+	// EncodeEntry encodes an entry into a kv-pair.
+	EncodeEntry(entry Entry) (k, v []byte, err error)
+}
+
+// IndexCodec defines an interfaces for encoding and decoding index-keys in the
+// kv-store.
+type IndexCodec interface {
+	EntryCodec
+
+	// DecodeIndexKey decodes a kv-pair into index-fields and primary-key field
+	// values. These fields may or may not overlap depending on the index.
+	DecodeIndexKey(k, v []byte) (indexFields, primaryKey []protoreflect.Value, err error)
+
+	// EncodeKVFromMessage encodes a kv-pair for the index from a message.
+	EncodeKVFromMessage(message protoreflect.Message) (k, v []byte, err error)
+}

orm/encoding/ormkv/entry.go

@@ -0,0 +1,152 @@
+package ormkv
+
+import (
+	"fmt"
+	"strings"
+
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+// Entry defines a logical representation of a kv-store entry for ORM instances.
+type Entry interface {
+	fmt.Stringer
+
+	// GetTableName returns the table-name (equivalent to the fully-qualified
+	// proto message name) this entry corresponds to.
+	GetTableName() protoreflect.FullName
+
+	// to allow new methods to be added without breakage, this interface
+	// shouldn't be implemented outside this package,
+	// see https://go.dev/blog/module-compatibility
+	doNotImplement()
+}
+
+// PrimaryKeyEntry represents a logically decoded primary-key entry.
+type PrimaryKeyEntry struct {
+
+	// TableName is the table this entry represents.
+	TableName protoreflect.FullName
+
+	// Key represents the primary key values.
+	Key []protoreflect.Value
+
+	// Value represents the message stored under the primary key.
+	Value proto.Message
+}
+
+func (p *PrimaryKeyEntry) GetTableName() protoreflect.FullName {
+	return p.TableName
+}
+
+func (p *PrimaryKeyEntry) String() string {
+	msg := p.Value
+	msgStr := "_"
+	if msg != nil {
+		msgBz, err := protojson.Marshal(msg)
+		if err == nil {
+			msgStr = string(msgBz)
+		} else {
+			msgStr = fmt.Sprintf("ERR:%v", err)
+		}
+	}
+	return fmt.Sprintf("PK:%s/%s:%s", p.TableName, fmtValues(p.Key), msgStr)
+}
+
+func fmtValues(values []protoreflect.Value) string {
+	if len(values) == 0 {
+		return "_"
+	}
+
+	parts := make([]string, len(values))
+	for i, v := range values {
+		val, err := structpb.NewValue(v.Interface())
+		if err != nil {
+			parts[i] = "ERR"
+			continue
+		}
+
+		bz, err := protojson.Marshal(val)
+		if err != nil {
+			parts[i] = "ERR"
+			continue
+		}
+
+		parts[i] = string(bz)
+	}
+
+	return strings.Join(parts, "/")
+}
+
+func (p *PrimaryKeyEntry) doNotImplement() {}
+
+// IndexKeyEntry represents a logically decoded index entry.
+type IndexKeyEntry struct {
+
+	// TableName is the table this entry represents.
+	TableName protoreflect.FullName
+
+	// Fields are the index fields this entry represents.
+	Fields []protoreflect.Name
+
+	// IsUnique indicates whether this index is unique or not.
+	IsUnique bool
+
+	// IndexValues represent the index values.
+	IndexValues []protoreflect.Value
+
+	// PrimaryKey represents the primary key values, it is empty if this is a
+	// prefix key
+	PrimaryKey []protoreflect.Value
+}
+
+func (i *IndexKeyEntry) GetTableName() protoreflect.FullName {
+	return i.TableName
+}
+
+func (i *IndexKeyEntry) doNotImplement() {}
+
+func (i *IndexKeyEntry) string() string {
+	return fmt.Sprintf("%s/%s:%s:%s", i.TableName, fmtFields(i.Fields), fmtValues(i.IndexValues), fmtValues(i.PrimaryKey))
+}
+
+func fmtFields(fields []protoreflect.Name) string {
+	strs := make([]string, len(fields))
+	for i, field := range fields {
+		strs[i] = string(field)
+	}
+	return strings.Join(strs, "/")
+}
+
+func (i *IndexKeyEntry) String() string {
+	if i.IsUnique {
+		return fmt.Sprintf("UNIQ:%s", i.string())
+	} else {
+
+		return fmt.Sprintf("IDX:%s", i.string())
+	}
+}
+
+// SeqEntry represents a sequence for tables with auto-incrementing primary keys.
+type SeqEntry struct {
+
+	// TableName is the table this entry represents.
+	TableName protoreflect.FullName
+
+	// Value is the uint64 value stored for this sequence.
+	Value uint64
+}
+
+func (s *SeqEntry) GetTableName() protoreflect.FullName {
+	return s.TableName
+}
+
+func (s *SeqEntry) doNotImplement() {}
+
+func (s *SeqEntry) String() string {
+	return fmt.Sprintf("SEQ:%s:%d", s.TableName, s.Value)
+}
+
+var _, _, _ Entry = &PrimaryKeyEntry{}, &IndexKeyEntry{}, &SeqEntry{}

orm/encoding/ormkv/entry_test.go

@@ -0,0 +1,77 @@
+package ormkv_test
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+
+	"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
+
+	"gotest.tools/v3/assert"
+
+	"github.com/cosmos/cosmos-sdk/orm/internal/testpb"
+	"github.com/cosmos/cosmos-sdk/orm/internal/testutil"
+)
+
+var aFullName = (&testpb.A{}).ProtoReflect().Descriptor().FullName()
+
+func TestPrimaryKeyEntry(t *testing.T) {
+	entry := &ormkv.PrimaryKeyEntry{
+		TableName: aFullName,
+		Key:       testutil.ValuesOf(uint32(1), "abc"),
+		Value:     &testpb.A{I32: -1},
+	}
+	assert.Equal(t, `PK:testpb.A/1/"abc":{"i32":-1}`, entry.String())
+	assert.Equal(t, aFullName, entry.GetTableName())
+
+	// prefix key
+	entry = &ormkv.PrimaryKeyEntry{
+		TableName: aFullName,
+		Key:       testutil.ValuesOf(uint32(1), "abc"),
+		Value:     nil,
+	}
+	assert.Equal(t, `PK:testpb.A/1/"abc":_`, entry.String())
+	assert.Equal(t, aFullName, entry.GetTableName())
+}
+
+func TestIndexKeyEntry(t *testing.T) {
+	entry := &ormkv.IndexKeyEntry{
+		TableName:   aFullName,
+		Fields:      []protoreflect.Name{"u32", "i32", "str"},
+		IsUnique:    false,
+		IndexValues: testutil.ValuesOf(uint32(10), int32(-1), "abc"),
+		PrimaryKey:  testutil.ValuesOf("abc", int32(-1)),
+	}
+	assert.Equal(t, `IDX:testpb.A/u32/i32/str:10/-1/"abc":"abc"/-1`, entry.String())
+	assert.Equal(t, aFullName, entry.GetTableName())
+
+	entry = &ormkv.IndexKeyEntry{
+		TableName:   aFullName,
+		Fields:      []protoreflect.Name{"u32"},
+		IsUnique:    true,
+		IndexValues: testutil.ValuesOf(uint32(10)),
+		PrimaryKey:  testutil.ValuesOf("abc", int32(-1)),
+	}
+	assert.Equal(t, `UNIQ:testpb.A/u32:10:"abc"/-1`, entry.String())
+	assert.Equal(t, aFullName, entry.GetTableName())
+
+	// prefix key
+	entry = &ormkv.IndexKeyEntry{
+		TableName:   aFullName,
+		Fields:      []protoreflect.Name{"u32", "i32", "str"},
+		IsUnique:    false,
+		IndexValues: testutil.ValuesOf(uint32(10), int32(-1)),
+	}
+	assert.Equal(t, `IDX:testpb.A/u32/i32/str:10/-1:_`, entry.String())
+	assert.Equal(t, aFullName, entry.GetTableName())
+
+	// prefix key
+	entry = &ormkv.IndexKeyEntry{
+		TableName:   aFullName,
+		Fields:      []protoreflect.Name{"str", "i32"},
+		IsUnique:    true,
+		IndexValues: testutil.ValuesOf("abc", int32(1)),
+	}
+	assert.Equal(t, `UNIQ:testpb.A/str/i32:"abc"/1:_`, entry.String())
+	assert.Equal(t, aFullName, entry.GetTableName())
+}

orm/encoding/ormkv/index_key.go

@@ -0,0 +1,120 @@
+package ormkv
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// IndexKeyCodec is the codec for (non-unique) index keys.
+type IndexKeyCodec struct {
+	*KeyCodec
+	tableName    protoreflect.FullName
+	pkFieldOrder []int
+}
+
+var _ IndexCodec = &IndexKeyCodec{}
+
+// NewIndexKeyCodec creates a new IndexKeyCodec with an optional prefix for the
+// provided message descriptor, index and primary key fields.
+func NewIndexKeyCodec(prefix []byte, messageDescriptor protoreflect.MessageDescriptor, indexFields, primaryKeyFields []protoreflect.Name) (*IndexKeyCodec, error) {
+	indexFieldMap := map[protoreflect.Name]int{}
+
+	keyFields := make([]protoreflect.Name, 0, len(indexFields)+len(primaryKeyFields))
+	for i, f := range indexFields {
+		indexFieldMap[f] = i
+		keyFields = append(keyFields, f)
+	}
+
+	numIndexFields := len(indexFields)
+	numPrimaryKeyFields := len(primaryKeyFields)
+	pkFieldOrder := make([]int, numPrimaryKeyFields)
+	k := 0
+	for j, f := range primaryKeyFields {
+		if i, ok := indexFieldMap[f]; ok {
+			pkFieldOrder[j] = i
+			continue
+		}
+		keyFields = append(keyFields, f)
+		pkFieldOrder[j] = numIndexFields + k
+		k++
+	}
+
+	cdc, err := NewKeyCodec(prefix, messageDescriptor, keyFields)
+	if err != nil {
+		return nil, err
+	}
+
+	return &IndexKeyCodec{
+		KeyCodec:     cdc,
+		pkFieldOrder: pkFieldOrder,
+		tableName:    messageDescriptor.FullName(),
+	}, nil
+}
+
+func (cdc IndexKeyCodec) DecodeIndexKey(k, _ []byte) (indexFields, primaryKey []protoreflect.Value, err error) {
+
+	values, err := cdc.Decode(bytes.NewReader(k))
+	// got prefix key
+	if err == io.EOF {
+		return values, nil, nil
+	} else if err != nil {
+		return nil, nil, err
+	}
+
+	// got prefix key
+	if len(values) < len(cdc.fieldCodecs) {
+		return values, nil, nil
+	}
+
+	numPkFields := len(cdc.pkFieldOrder)
+	pkValues := make([]protoreflect.Value, numPkFields)
+
+	for i := 0; i < numPkFields; i++ {
+		pkValues[i] = values[cdc.pkFieldOrder[i]]
+	}
+
+	return values, pkValues, nil
+}
+
+func (cdc IndexKeyCodec) DecodeEntry(k, v []byte) (Entry, error) {
+	idxValues, pk, err := cdc.DecodeIndexKey(k, v)
+	if err != nil {
+		return nil, err
+	}
+
+	return &IndexKeyEntry{
+		TableName:   cdc.tableName,
+		Fields:      cdc.fieldNames,
+		IndexValues: idxValues,
+		PrimaryKey:  pk,
+	}, nil
+}
+
+func (cdc IndexKeyCodec) EncodeEntry(entry Entry) (k, v []byte, err error) {
+	indexEntry, ok := entry.(*IndexKeyEntry)
+	if !ok {
+		return nil, nil, ormerrors.BadDecodeEntry
+	}
+
+	if indexEntry.TableName != cdc.tableName {
+		return nil, nil, ormerrors.BadDecodeEntry
+	}
+
+	bz, err := cdc.KeyCodec.Encode(indexEntry.IndexValues)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return bz, sentinel, nil
+}
+
+var sentinel = []byte{0}
+
+func (cdc IndexKeyCodec) EncodeKVFromMessage(message protoreflect.Message) (k, v []byte, err error) {
+	_, k, err = cdc.EncodeFromMessage(message)
+	return k, sentinel, err
+}