Skip to content

Commit

Permalink
Merge pull request #15 from stackb/hash-oneof
Browse files Browse the repository at this point in the history 
Add hashOneOf test cases
  • Loading branch information
@pcj
pcj committed Jun 21, 2023
2 parents fbf579e + e04dd43 commit d9be408
Show file tree
Hide file tree
Showing 2 changed files with 178 additions and 71 deletions.
122 changes: 51 additions & 71 deletions hasher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func (h *hasher) hashMessage(msg protoreflect.Message) ([]byte, error) {
return nil, nil
}

var hashes []fieldHashEntry
var hashes []*fieldHashEntry

fieldHashes, err := h.hashFields(msg, md.Fields())
if err != nil {
Expand All @@ -80,89 +80,60 @@ func (h *hasher) hashMessage(msg protoreflect.Message) ([]byte, error) {
buf.Write(hash.vhash)
}

// ohash, err := h.hashOneofs(msg, md.Oneofs())
// if err != nil {
// return nil, fmt.Errorf("hashing fields: %w", err)
// }
// buf.Write(ohash)

identifier := mapIdentifier
// if hasher.messageIdentifier != "" {
// identifier = hasher.messageIdentifier
// }
return hash(identifier, buf.Bytes())
}

func (h *hasher) hashOneofs(msg protoreflect.Message, oneofs protoreflect.OneofDescriptors) ([]byte, error) {
type oneOfHash struct {
number int
data []byte
}

hashes := make([]oneOfHash, oneofs.Len())
// for i := 0; i < oneofs.Len(); i++ {
// od := oneofs.Get(i)
// fields := od.Fields()
// data, err := h.hashFields(msg, fields)
// if err != nil {
// return nil, fmt.Errorf("hashing field %d (%s): %w", i, od.FullName(), err)
// }
// hashes[i] = oneOfHash{number: i, v: data}
// }
func (h *hasher) hashFields(msg protoreflect.Message, fields protoreflect.FieldDescriptors) ([]*fieldHashEntry, error) {
hashes := make([]*fieldHashEntry, 0, fields.Len())

var buf bytes.Buffer
for _, hash := range hashes {
buf.Write(hash.data)
}

return buf.Bytes(), nil
}

func (h *hasher) hashFields(msg protoreflect.Message, fields protoreflect.FieldDescriptors) ([]fieldHashEntry, error) {
// type fieldHash struct {
// number int
// data []byte
// }

hashes := make([]fieldHashEntry, 0, fields.Len())
for i := 0; i < fields.Len(); i++ {
fd := fields.Get(i)

if !msg.Has(fd) {
// if we are in this block and the field is a scalar one, it is
// either a proto3 field that was never set or is the empty value
// (indistinguishable) or this is a proto2 field that is nil.
continue
}
value := msg.Get(fd)

var khash []byte
var err error
if h.fieldNamesAsKeys {
khash, err = hashUnicode(string(fd.Name()))
} else {
khash, err = hashInt64(int64(fd.Number()))
}
hash, err := h.hashField(fd, msg.Get(fd))
if err != nil {
return nil, fmt.Errorf("hashing key field %d (%s = %d): %w", i, fd.FullName(), fd.Number(), err)
return nil, err
}
hashes = append(hashes, hash)
}

vhash, err := h.hashField(fd, value)
if err != nil {
return nil, fmt.Errorf("hashing field %d (%s = %d): %w", i, fd.FullName(), fd.Number(), err)
}
return hashes, nil
}

hashes = append(hashes, fieldHashEntry{
number: int32(fd.Number()),
khash: khash,
vhash: vhash,
})
func (h *hasher) hashField(fd protoreflect.FieldDescriptor, value protoreflect.Value) (*fieldHashEntry, error) {
khash, err := h.hashFieldKey(fd)
if err != nil {
return nil, fmt.Errorf("hashing field key %d (%s): %w", fd.Number(), fd.FullName(), err)
}

return hashes, nil
vhash, err := h.hashFieldValue(fd, value)
if err != nil {
return nil, fmt.Errorf("hashing field value %d (%s): %w", fd.Number(), fd.FullName(), err)
}

return &fieldHashEntry{
number: int32(fd.Number()),
khash: khash,
vhash: vhash,
}, nil
}

func (h *hasher) hashFieldKey(fd protoreflect.FieldDescriptor) ([]byte, error) {
if h.fieldNamesAsKeys {
return hashUnicode(string(fd.Name()))
}
return hashInt64(int64(fd.Number()))
}

func (h *hasher) hashField(fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
func (h *hasher) hashFieldValue(fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
if fd.IsList() {
return h.hashList(fd.Kind(), value.List())
}
Expand All @@ -174,32 +145,41 @@ func (h *hasher) hashField(fd protoreflect.FieldDescriptor, value protoreflect.V

func (h *hasher) hashValue(kind protoreflect.Kind, value protoreflect.Value) ([]byte, error) {
switch kind {
case protoreflect.BoolKind:
case
protoreflect.BoolKind:
return h.hashBool(value.Bool())
case protoreflect.EnumKind:
case
protoreflect.EnumKind:
return h.hashEnum(value.Enum())
case protoreflect.Uint32Kind,
case
protoreflect.Uint32Kind,
protoreflect.Uint64Kind,
protoreflect.Fixed32Kind,
protoreflect.Fixed64Kind:
return h.hashUint(value.Uint())
case protoreflect.Int32Kind,
case
protoreflect.Int32Kind,
protoreflect.Int64Kind,
protoreflect.Sint32Kind,
protoreflect.Sint64Kind,
protoreflect.Sfixed32Kind,
protoreflect.Sfixed64Kind:
return h.hashInt(value.Int())
case protoreflect.FloatKind,
case
protoreflect.FloatKind,
protoreflect.DoubleKind:
return h.hashFloat(value.Float())
case protoreflect.StringKind:
case
protoreflect.StringKind:
return h.hashString(value.String())
case protoreflect.BytesKind:
case
protoreflect.BytesKind:
return h.hashBytes(value.Bytes())
case protoreflect.MessageKind:
case
protoreflect.MessageKind:
return h.hashMessage(value.Message())
case protoreflect.GroupKind:
case
protoreflect.GroupKind:
return nil, fmt.Errorf("protoreflect.GroupKind: not implemented: %T", value)
}
return nil, fmt.Errorf("unexpected field kind: %v (%T)", kind, value)
Expand Down Expand Up @@ -259,14 +239,14 @@ func (h *hasher) hashMap(kd, fd protoreflect.FieldDescriptor, m protoreflect.Map
var errValue error
var errKey protoreflect.MapKey
m.Range(func(mk protoreflect.MapKey, v protoreflect.Value) bool {
khash, err := h.hashField(kd, mk.Value())
khash, err := h.hashFieldValue(kd, mk.Value())
if err != nil {
errKey = mk
errValue = err
return false
}

vhash, err := h.hashField(fd, v)
vhash, err := h.hashFieldValue(fd, v)
if err != nil {
errKey = mk
errValue = err
Expand Down
127 changes: 127 additions & 0 deletions hasher_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -749,6 +749,133 @@ func TestHashProto2DefaultFields(t *testing.T) {
}
}

func TestHashOneofFields(t *testing.T) {
for name, tc := range map[string]hashTestCase{
"empty": {
fieldNamesAsKeys: true,
protos: []proto.Message{
&pb2_latest.Singleton{},
&pb3_latest.Singleton{},

&pb2_latest.Empty{},
&pb3_latest.Empty{},
},
obj: map[int64]string{},
json: `{}`,
want: "18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e4",
},
"one selected but empty": {
fieldNamesAsKeys: false,
protos: []proto.Message{
// Only proto2 has empty values.
&pb2_latest.Simple{BoolField: proto.Bool(false)},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheBool{}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheBool{}},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheBool{TheBool: false}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheBool{TheBool: false}},
},
obj: map[int64]bool{1: false},
want: "8a956cfa8e9b45b738cb8dc8a3dc7126dab3cbd2c07c80fa1ec312a1a31ed709",
},
"One of the options selected with content (empty string)": {
fieldNamesAsKeys: false,
protos: []proto.Message{
// Only proto2 has empty values.
&pb2_latest.Simple{StringField: proto.String("")},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{}},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{TheString: ""}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{TheString: ""}},
},
obj: map[int64]string{25: ""},
want: "79cff9d2d0ee6c6071c82b58d1a2fcf056b58c4501606862489e5731644c755a",
},
"One of the options selected with content (ints)": {
fieldNamesAsKeys: false,
protos: []proto.Message{
// Only proto2 has empty values.
&pb2_latest.Simple{Int32Field: proto.Int32(0)},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{}},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{TheInt32: 0}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{TheInt32: 0}},
},
obj: map[int64]int32{13: 0},
want: "bafd42680c987c47a76f72e08ed975877162efdb550d2c564c758dc7d988468f",
},
"One of the options selected with content (strings)": {
fieldNamesAsKeys: false,
protos: []proto.Message{
&pb2_latest.Simple{StringField: proto.String("TEST!")},
&pb3_latest.Simple{StringField: "TEST!"},
//
// For protobufs, it is legal (and backwards-compatible) to update a message by wrapping
// an existing field within a oneof rule. Therefore, both objects (using old schem and
// the new schema) should result in the same objecthash.
//
// Example:
//
// # Old schema: | # New schema:
// message Simple { | message Singleton {
// string string_field = 25; | oneof singleton {
// } | string the_string = 25;
// | }
// | }
//
// The following examples demonstrate this equivalence.
&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheString{TheString: "TEST!"}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheString{TheString: "TEST!"}},
},
obj: map[int64]string{25: "TEST!"},
want: "336cdbca99fd46157bc47bcc456f0ac7f1ef3be7a79acf3535f671434b53944f",
},
"One of the options selected with content (equiv case ints)": {
fieldNamesAsKeys: false,
protos: []proto.Message{
&pb2_latest.Simple{Int32Field: proto.Int32(99)},
&pb3_latest.Simple{Int32Field: 99},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheInt32{TheInt32: 99}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheInt32{TheInt32: 99}},
},
obj: map[int64]int32{13: 99},
want: "65517521bc278528d25caf1643da0f094fd88dad50205c9743e3c984a7c53b7d",
},
"One of the options selected with content (nested)": {
fieldNamesAsKeys: false,
protos: []proto.Message{
&pb2_latest.Simple{SingletonField: &pb2_latest.Singleton{}},
&pb3_latest.Simple{SingletonField: &pb3_latest.Singleton{}},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}},
},
obj: map[int64]map[int64]int64{35: {}},
want: "4967c72525c764229f9fbf1294764c9aedc0d4f9f4c52e04a19c7f35ca65f517",
},
"One of the options selected with content (double nested)": {
fieldNamesAsKeys: false,
protos: []proto.Message{
&pb2_latest.Simple{SingletonField: &pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}}},
&pb3_latest.Simple{SingletonField: &pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}}},

&pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{Singleton: &pb2_latest.Singleton_TheSingleton{TheSingleton: &pb2_latest.Singleton{}}}}},
&pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{Singleton: &pb3_latest.Singleton_TheSingleton{TheSingleton: &pb3_latest.Singleton{}}}}},
},
obj: map[int64]map[int64]map[int64]int64{35: {35: {}}},
want: "8ea95bbda0f42073a61f46f9f375f48d5a7cb034fce56b44f958470fda5236d0",
},
} {
tc.Check(name, t)
}
}

func TestHashMapFields(t *testing.T) {
for name, tc := range map[string]hashTestCase{
"boolean maps": {
Expand Down

0 comments on commit d9be408

Please sign in to comment.