Permalink
Browse files

Simplify mutation protos (#823)

* Move KeyValue into SignedKV

Remove one level of proto encapsulation by embedding the signature in
SignedKV along with the data to be signed.

Before ObjectHash, protos needed to be serialized then signed, forcing
us to create two proto messages for each signed type: a value type and a
signed value type. With ObjectHash we can avoid signing a particular
serialization of a protocol buffer and just sign the object itself,
inserting the signature into the object after signing.  Verification
involves removing the signature before verifying the object.

This is part of the proto cleanup effort in  XXX

* EntryUpdate.Update -> Mutation

* Mutation.Value type []byte -> Entry

It appears that grpc has trouble with proto3 OneOf.
We didn't have problems before when entry was a serialized []byte,
but now that we're passing a regular proto object, the PublicKey type
is coming out the otherside, stripped of it's interior values.
This has been fixed by #821

* cleanup

* Remove spurious gogo dependency
  • Loading branch information...
gdbelvin committed Oct 2, 2017
1 parent dc9204a commit ff7f3bee3cb1da50f0b37faa6309c973698ec899
Showing with 277 additions and 10,883 deletions.
  1. +13 −3 core/client/grpcc/grpc_client.go
  2. +5 −9 core/keyserver/keyserver.go
  3. +3 −8 core/keyserver/validate.go
  4. +5 −9 core/keyserver/validate_test.go
  5. +10 −1 core/monitor/verify.go
  6. +1 −1 core/mutation/mutation.go
  7. +3 −4 core/mutation/mutation_test.go
  8. +13 −0 core/mutator/entry/entry.go
  9. +0 −29 core/mutator/entry/entry_test.go
  10. +10 −15 core/mutator/entry/mutation.go
  11. +9 −9 core/mutator/entry/mutator.go
  12. +9 −3 core/mutator/entry/mutator_test.go
  13. +1 −1 core/mutator/mutator.go
  14. +112 −132 core/proto/keytransparency_v1_types/keytransparency_v1_types.pb.go
  15. +9 −13 core/proto/keytransparency_v1_types/keytransparency_v1_types.proto
  16. +8 −3 core/sequencer/sequencer.go
  17. +3 −2 impl/sql/mutations/mutations.go
  18. +63 −120 impl/sql/mutations/mutations_test.go
  19. +0 −36 vendor/github.com/gogo/protobuf/LICENSE
  20. +0 −43 vendor/github.com/gogo/protobuf/proto/Makefile
  21. +0 −234 vendor/github.com/gogo/protobuf/proto/clone.go
  22. +0 −978 vendor/github.com/gogo/protobuf/proto/decode.go
  23. +0 −172 vendor/github.com/gogo/protobuf/proto/decode_gogo.go
  24. +0 −100 vendor/github.com/gogo/protobuf/proto/duration.go
  25. +0 −203 vendor/github.com/gogo/protobuf/proto/duration_gogo.go
  26. +0 −1,362 vendor/github.com/gogo/protobuf/proto/encode.go
  27. +0 −350 vendor/github.com/gogo/protobuf/proto/encode_gogo.go
  28. +0 −300 vendor/github.com/gogo/protobuf/proto/equal.go
  29. +0 −693 vendor/github.com/gogo/protobuf/proto/extensions.go
  30. +0 −294 vendor/github.com/gogo/protobuf/proto/extensions_gogo.go
  31. +0 −898 vendor/github.com/gogo/protobuf/proto/lib.go
  32. +0 −42 vendor/github.com/gogo/protobuf/proto/lib_gogo.go
  33. +0 −311 vendor/github.com/gogo/protobuf/proto/message_set.go
  34. +0 −484 vendor/github.com/gogo/protobuf/proto/pointer_reflect.go
  35. +0 −85 vendor/github.com/gogo/protobuf/proto/pointer_reflect_gogo.go
  36. +0 −270 vendor/github.com/gogo/protobuf/proto/pointer_unsafe.go
  37. +0 −128 vendor/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go
  38. +0 −968 vendor/github.com/gogo/protobuf/proto/properties.go
  39. +0 −111 vendor/github.com/gogo/protobuf/proto/properties_gogo.go
  40. +0 −119 vendor/github.com/gogo/protobuf/proto/skip_gogo.go
  41. +0 −928 vendor/github.com/gogo/protobuf/proto/text.go
  42. +0 −57 vendor/github.com/gogo/protobuf/proto/text_gogo.go
  43. +0 −1,013 vendor/github.com/gogo/protobuf/proto/text_parser.go
  44. +0 −113 vendor/github.com/gogo/protobuf/proto/timestamp.go
  45. +0 −229 vendor/github.com/gogo/protobuf/proto/timestamp_gogo.go
@@ -33,16 +33,18 @@ import (
"github.com/google/keytransparency/core/mutator"
"github.com/google/keytransparency/core/mutator/entry"
"github.com/google/trillian"
"github.com/google/trillian/client"
"github.com/google/trillian/crypto/keys/der"
"github.com/google/trillian/crypto/keyspb"
"github.com/google/trillian/merkle/hashers"
"github.com/golang/protobuf/proto"
"golang.org/x/net/context"
"google.golang.org/grpc"
tpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types"
spb "github.com/google/keytransparency/impl/proto/keytransparency_v1_service"
"github.com/google/trillian"
)
const (
@@ -250,7 +252,7 @@ func (c *Client) Update(ctx context.Context, userID, appID string, profileData [
if err != nil {
return nil, fmt.Errorf("entry.FromLeafValue: %v", err)
}
if _, err := c.mutator.Mutate(oldLeaf, req.GetEntryUpdate().GetUpdate()); err != nil {
if _, err := c.mutator.Mutate(oldLeaf, req.GetEntryUpdate().GetMutation()); err != nil {
return nil, fmt.Errorf("Mutate: %v", err)
}
@@ -277,8 +279,16 @@ func (c *Client) Retry(ctx context.Context, req *tpb.UpdateEntryRequest, opts ..
return fmt.Errorf("VerifyGetEntryResponse(): %v", err)
}
// Mutations are no longer stable serialized byte slices, so we need to use
// an equality operation on the proto itself.
leafValue, err := entry.FromLeafValue(
updateResp.GetProof().GetLeafProof().GetLeaf().GetLeafValue())
if err != nil {
return fmt.Errorf("failed to decode current entry: %v", err)
}
// Check if the response is a replay.
if got, want := updateResp.GetProof().GetLeafProof().Leaf.LeafValue, req.GetEntryUpdate().GetUpdate().GetKeyValue().GetValue(); !bytes.Equal(got, want) {
if got, want := leafValue, req.GetEntryUpdate().GetMutation().GetValue(); !proto.Equal(got, want) {
return ErrRetry
}
return nil
@@ -277,7 +277,7 @@ func (s *Server) UpdateEntry(ctx context.Context, in *tpb.UpdateEntryRequest) (*
return nil, grpc.Errorf(codes.InvalidArgument, "Invalid request")
}
if err := s.saveCommitment(ctx, in.GetEntryUpdate().GetUpdate().GetKeyValue(), in.GetEntryUpdate().Committed); err != nil {
if err := s.saveCommitment(ctx, in.GetEntryUpdate().GetMutation(), in.GetEntryUpdate().Committed); err != nil {
return nil, err
}
@@ -306,7 +306,7 @@ func (s *Server) UpdateEntry(ctx context.Context, in *tpb.UpdateEntryRequest) (*
glog.Errorf("entry.FromLeafValue: %v", err)
return nil, grpc.Errorf(codes.InvalidArgument, "invalid previous leaf value")
}
if _, err := s.mutator.Mutate(oldEntry, in.GetEntryUpdate().GetUpdate()); err == mutator.ErrReplay {
if _, err := s.mutator.Mutate(oldEntry, in.GetEntryUpdate().GetMutation()); err == mutator.ErrReplay {
glog.Warningf("Discarding request due to replay")
// Return the response. The client should handle the replay case
// by comparing the returned response with the request. Check
@@ -322,7 +322,7 @@ func (s *Server) UpdateEntry(ctx context.Context, in *tpb.UpdateEntryRequest) (*
if err != nil {
return nil, grpc.Errorf(codes.Internal, "Cannot create transaction")
}
if _, err := s.mutations.Write(txn, in.GetEntryUpdate().GetUpdate()); err != nil {
if _, err := s.mutations.Write(txn, in.GetEntryUpdate().GetMutation()); err != nil {
glog.Errorf("mutations.Write failed: %v", err)
if err := txn.Rollback(); err != nil {
glog.Errorf("Cannot rollback the transaction: %v", err)
@@ -367,12 +367,8 @@ func (s *Server) GetDomainInfo(ctx context.Context, in *tpb.GetDomainInfoRequest
}, nil
}
func (s *Server) saveCommitment(ctx context.Context, kv *tpb.KeyValue, committed *tpb.Committed) error {
entry := new(tpb.Entry)
if err := proto.Unmarshal(kv.Value, entry); err != nil {
glog.Warningf("Error unmarshaling entry: %v", err)
return grpc.Errorf(codes.InvalidArgument, "Invalid request")
}
func (s *Server) saveCommitment(ctx context.Context, skv *tpb.SignedKV, committed *tpb.Committed) error {
entry := skv.Value
// Write the commitment.
if err := s.committer.Write(ctx, entry.Commitment, committed.Data, committed.Key); err != nil {
View
@@ -25,8 +25,6 @@ import (
"github.com/google/keytransparency/core/crypto/commitments"
"github.com/google/keytransparency/core/crypto/vrf"
"github.com/golang/protobuf/proto"
tpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types"
)
@@ -72,15 +70,12 @@ func validateKey(userID, appID string, key []byte) error {
// - Commitment in SignedEntryUpdate matches the serialized profile.
// - Profile is a valid.
func validateUpdateEntryRequest(in *tpb.UpdateEntryRequest, vrfPriv vrf.PrivateKey) error {
kv := in.GetEntryUpdate().GetUpdate().GetKeyValue()
entry := new(tpb.Entry)
if err := proto.Unmarshal(kv.Value, entry); err != nil {
return err
}
skv := in.GetEntryUpdate().GetMutation()
entry := skv.GetValue()
// Verify Index / VRF
index, _ := vrfPriv.Evaluate(vrf.UniqueID(in.UserId, in.AppId))
if got, want := kv.Key, index[:]; !bytes.Equal(got, want) {
if got, want := skv.Index, index[:]; !bytes.Equal(got, want) {
return ErrWrongIndex
}
@@ -23,8 +23,6 @@ import (
"github.com/google/keytransparency/core/crypto/vrf"
"github.com/google/keytransparency/core/crypto/vrf/p256"
"github.com/golang/protobuf/proto"
tpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types"
)
@@ -99,19 +97,17 @@ func TestValidateUpdateEntryRequest(t *testing.T) {
{false, userID, index, commitment, nil}, // Incorrect key
{true, userID, index, commitment, nonce},
} {
entry := &tpb.Entry{
Commitment: tc.commitment,
}
entryData, _ := proto.Marshal(entry)
kv := &tpb.KeyValue{Key: tc.index[:], Value: entryData}
signedkv := &tpb.SignedKV{
KeyValue: kv,
Index: tc.index[:],
Value: &tpb.Entry{
Commitment: tc.commitment,
},
}
req := &tpb.UpdateEntryRequest{
UserId: tc.userID,
AppId: appID,
EntryUpdate: &tpb.EntryUpdate{
Update: signedkv,
Mutation: signedkv,
Committed: &tpb.Committed{
Key: tc.nonce,
Data: profileData,
View
@@ -141,12 +141,21 @@ func (m *Monitor) verifyMutations(muts []*ktpb.MutationProof, oldRoot, expectedN
}
// compute the new leaf
newLeaf, err := mutator.Mutate(oldLeaf, mut.GetMutation())
newValue, err := mutator.Mutate(oldLeaf, mut.GetMutation())
if err != nil {
glog.Infof("Mutation did not verify: %v", err)
errList = append(errList, ErrInvalidMutation)
}
newLeafnID := storage.NewNodeIDFromPrefixSuffix(index, storage.Suffix{}, m.mapHasher.BitLen())
newLeaf, err := entry.ToLeafValue(newValue)
if err != nil {
glog.Infof("Failed to serialize: %v", err)
errList = append(errList, err)
}
// BUG(gdbelvin): Proto serializations are not idempotent.
// - Upgrade the hasher to use ObjectHash.
// - Use deep compare between the tree and the computed value.
newLeafHash := m.mapHasher.HashLeaf(mapID, index, newLeaf)
newLeaves = append(newLeaves, merkle.HStar2LeafHash{
Index: newLeafnID.BigInt(),
@@ -108,7 +108,7 @@ func (s *Server) GetMutations(ctx context.Context, in *tpb.GetMutationsRequest)
mutations := make([]*tpb.MutationProof, 0, len(mRange))
for _, m := range mRange {
mutations = append(mutations, &tpb.MutationProof{Mutation: m})
indexes = append(indexes, m.GetKeyValue().GetKey())
indexes = append(indexes, m.GetIndex())
}
// Get leaf proofs.
// TODO: allow leaf proofs to be optional.
@@ -46,10 +46,9 @@ func signedKV(t *testing.T, start, end int) []*tpb.SignedKV {
kvs := make([]*tpb.SignedKV, 0, end-start)
for i := start; i <= end; i++ {
kvs = append(kvs, &tpb.SignedKV{
KeyValue: &tpb.KeyValue{
Key: []byte(fmt.Sprintf("key_%v", i)),
Value: []byte(fmt.Sprintf("value_%v", i)),
}})
Index: []byte(fmt.Sprintf("key_%v", i)),
Value: &tpb.Entry{Commitment: []byte(fmt.Sprintf("value_%v", i))},
})
}
return kvs
}
@@ -16,6 +16,8 @@
package entry
import (
"fmt"
"github.com/google/keytransparency/core/crypto/signatures"
"github.com/google/keytransparency/core/crypto/signatures/factory"
@@ -43,6 +45,17 @@ func FromLeafValue(value []byte) (*tpb.Entry, error) {
return nil, nil
}
// ToLeafValue converts the update object into a serialized object to store in the map.
func ToLeafValue(update proto.Message) ([]byte, error) {
e, ok := update.(*tpb.Entry)
if !ok {
glog.Warning("received proto.Message is not of type *tpb.SignedKV.")
return nil, fmt.Errorf("updateM.(*tpb.SignedKV): _, %v", ok)
}
return proto.Marshal(e)
}
func verifiersFromKeys(keys []*keyspb.PublicKey) (map[string]signatures.Verifier, error) {
verifiers := make(map[string]signatures.Verifier)
for _, key := range keys {
@@ -17,7 +17,6 @@ package entry
import (
"encoding/pem"
"errors"
"fmt"
"reflect"
"testing"
@@ -26,7 +25,6 @@ import (
"github.com/google/keytransparency/core/crypto/signatures/factory"
"github.com/google/trillian/crypto/keyspb"
"github.com/google/trillian/crypto/sigpb"
"github.com/golang/protobuf/proto"
@@ -75,33 +73,6 @@ func createEntry(commitment []byte, pkeys []string) (*tpb.Entry, error) {
}, nil
}
func prepareMutation(key []byte, newEntry *tpb.Entry, previous []byte, signers []signatures.Signer) (*tpb.SignedKV, error) {
newEntry.Previous = previous
entryData, err := proto.Marshal(newEntry)
if err != nil {
return nil, fmt.Errorf("Marshal(%v)=%v", newEntry, err)
}
kv := &tpb.KeyValue{
Key: key,
Value: entryData,
}
// Populate signatures map.
sigs := make(map[string]*sigpb.DigitallySigned)
for _, signer := range signers {
sig, err := signer.Sign(*kv)
if err != nil {
return nil, fmt.Errorf("signerSign() failed: %v", err)
}
sigs[signer.KeyID()] = sig
}
return &tpb.SignedKV{
KeyValue: kv,
Signatures: sigs,
}, nil
}
func signersFromPEMs(t *testing.T, keys [][]byte) []signatures.Signer {
signatures.Rand = dev.Zeros
signers := make([]signatures.Signer, 0, len(keys))
@@ -23,7 +23,6 @@ import (
"github.com/google/trillian/crypto/sigpb"
"github.com/benlaurie/objecthash/go/objecthash"
"github.com/golang/protobuf/proto"
tpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types"
)
@@ -95,9 +94,11 @@ func (m *Mutation) SerializeAndSign(signers []signatures.Signer) (*tpb.UpdateEnt
}
// Check authorization.
skv := *signedkv
skv.Signatures = nil
if err := verifyKeys(m.prevEntry.GetAuthorizedKeys(),
m.entry.GetAuthorizedKeys(),
signedkv.GetKeyValue(),
skv,
signedkv.GetSignatures()); err != nil {
return nil, err
}
@@ -106,7 +107,7 @@ func (m *Mutation) SerializeAndSign(signers []signatures.Signer) (*tpb.UpdateEnt
UserId: m.userID,
AppId: m.appID,
EntryUpdate: &tpb.EntryUpdate{
Update: signedkv,
Mutation: signedkv,
Committed: &tpb.Committed{
Key: m.nonce,
Data: m.data,
@@ -117,26 +118,20 @@ func (m *Mutation) SerializeAndSign(signers []signatures.Signer) (*tpb.UpdateEnt
// Sign produces the SignedKV
func (m *Mutation) sign(signers []signatures.Signer) (*tpb.SignedKV, error) {
entryData, err := proto.Marshal(m.entry)
if err != nil {
return nil, err
}
kv := &tpb.KeyValue{
Key: m.index,
Value: entryData,
skv := &tpb.SignedKV{
Index: m.index,
Value: m.entry,
}
sigs := make(map[string]*sigpb.DigitallySigned)
for _, signer := range signers {
sig, err := signer.Sign(kv)
sig, err := signer.Sign(skv)
if err != nil {
return nil, err
}
sigs[signer.KeyID()] = sig
}
return &tpb.SignedKV{
KeyValue: kv,
Signatures: sigs,
}, nil
skv.Signatures = sigs
return skv, nil
}
@@ -41,8 +41,9 @@ func New() *Mutator {
}
// Mutate verifies that this is a valid mutation for this item and applies
// mutation to value.
func (*Mutator) Mutate(oldValue, update proto.Message) ([]byte, error) {
// mutation to value. Repeated applications of Mutate on the same input produce
// the same output.
func (*Mutator) Mutate(oldValue, update proto.Message) (proto.Message, error) {
// Ensure that the mutation size is within bounds.
if proto.Size(update) > mutator.MaxMutationSize {
glog.Warningf("mutation (%v bytes) is larger than the maximum accepted size (%v bytes).", proto.Size(update), mutator.MaxMutationSize)
@@ -64,11 +65,7 @@ func (*Mutator) Mutate(oldValue, update proto.Message) ([]byte, error) {
oldEntry = old
}
kv := updated.GetKeyValue()
newEntry := new(tpb.Entry)
if err := proto.Unmarshal(kv.Value, newEntry); err != nil {
return nil, err
}
newEntry := updated.Value
// Verify pointer to previous data.
// The very first entry will have oldValue=nil, so its hash is the
@@ -91,13 +88,16 @@ func (*Mutator) Mutate(oldValue, update proto.Message) ([]byte, error) {
return nil, mutator.ErrMissingKey
}
kv := *updated
kv.Signatures = nil
if err := verifyKeys(oldEntry.GetAuthorizedKeys(),
newEntry.GetAuthorizedKeys(),
kv, updated.GetSignatures()); err != nil {
kv,
updated.GetSignatures()); err != nil {
return nil, err
}
return updated.GetKeyValue().GetValue(), nil
return updated.GetValue(), nil
}
// verifyKeys verifies both old and new authorized keys based on the following
Oops, something went wrong.

0 comments on commit ff7f3be

Please sign in to comment.