Monitor verification logic (#768)

* reintroduce verification

* monitor integration tests

*  quickfix for mutator bug

CONTRIBUTORS

@@ -13,3 +13,4 @@ Antonio Marcedone <a.marcedone@gmail.com>
 Cesar Ghali <cghali@uci.edu>
 Daniel Ziegler <dmz@yahoo-inc.com>
 Gary Belvin <gdbelvin@gmail.com>
+Ismail Khoffi <Ismail.Khoffi@gmail.com>

cmd/keytransparency-monitor/main.go

@@ -42,7 +42,10 @@ import (
 	spb "github.com/google/keytransparency/impl/proto/keytransparency_v1_service"
 	mopb "github.com/google/keytransparency/impl/proto/monitor_v1_service"
 	mupb "github.com/google/keytransparency/impl/proto/mutation_v1_service"
-	_ "github.com/google/trillian/merkle/coniks"    // Register coniks
+	tlogcli "github.com/google/trillian/client"
+	"github.com/google/trillian/crypto/keys/der"
+	_ "github.com/google/trillian/merkle/coniks" // Register coniks
+	"github.com/google/trillian/merkle/hashers"
 	_ "github.com/google/trillian/merkle/objhasher" // Register objhasher
 )
 
@@ -141,13 +144,22 @@ func main() {
 	// Insert handlers for other http paths here.
 	mux := http.NewServeMux()
 	mux.Handle("/", gwmux)
+	logHasher, err := hashers.NewLogHasher(logTree.GetHashStrategy())
+	if err != nil {
+		glog.Fatalf("Could not initialize log hasher: %v", err)
+	}
+	logPubKey, err := der.UnmarshalPublicKey(logTree.GetPublicKey().GetDer())
+	if err != nil {
+		glog.Fatalf("Failed parsing Log public key: %v", err)
+	}
+	logVerifier := tlogcli.NewLogVerifier(logHasher, logPubKey)
 
-	// initialize the mutations API client and feed the responses it got
-	// into the monitor:
-	mon, err := cmon.New(logTree, mapTree, crypto.NewSHA256Signer(key), store)
+	mon, err := cmon.New(logVerifier, mapTree, crypto.NewSHA256Signer(key), store)
 	if err != nil {
 		glog.Exitf("Failed to initialize monitor: %v", err)
 	}
+	// initialize the mutations API client and feed the responses it got
+	// into the monitor:
 	mutCli := client.New(mcc, *pollPeriod)
 	responses, errs := mutCli.StartPolling(1)
 	go func() {

core/monitor/monitor.go

@@ -25,39 +25,39 @@ import (
 	ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types"
 
 	"github.com/google/trillian"
+	"github.com/google/trillian/client"
 	tcrypto "github.com/google/trillian/crypto"
-	"github.com/google/trillian/merkle"
+	"github.com/google/trillian/crypto/keys/der"
 	"github.com/google/trillian/merkle/hashers"
 )
 
 // Monitor holds the internal state for a monitor accessing the mutations API
 // and for verifying its responses.
 type Monitor struct {
-	hasher      hashers.MapHasher
-	logPubKey   crypto.PublicKey
+	mapID       int64
+	mapHasher   hashers.MapHasher
 	mapPubKey   crypto.PublicKey
-	logVerifier merkle.LogVerifier
+	logVerifier client.LogVerifier
 	signer      *tcrypto.Signer
-	// TODO(ismail): update last trusted signed log root
-	//trusted     trillian.SignedLogRoot
-	store *storage.Storage
+	trusted     *trillian.SignedLogRoot
+	store       *storage.Storage
 }
 
 // New creates a new instance of the monitor.
-func New(logTree, mapTree *trillian.Tree, signer *tcrypto.Signer, store *storage.Storage) (*Monitor, error) {
-	logHasher, err := hashers.NewLogHasher(logTree.GetHashStrategy())
-	if err != nil {
-		return nil, fmt.Errorf("Failed creating LogHasher: %v", err)
-	}
+func New(logverifierCli client.LogVerifier, mapTree *trillian.Tree, signer *tcrypto.Signer, store *storage.Storage) (*Monitor, error) {
 	mapHasher, err := hashers.NewMapHasher(mapTree.GetHashStrategy())
 	if err != nil {
 		return nil, fmt.Errorf("Failed creating MapHasher: %v", err)
 	}
+	mapPubKey, err := der.UnmarshalPublicKey(mapTree.GetPublicKey().GetDer())
+	if err != nil {
+		return nil, fmt.Errorf("Could not unmarshal map public key: %v", err)
+	}
 	return &Monitor{
-		hasher:      mapHasher,
-		logVerifier: merkle.NewLogVerifier(logHasher),
-		logPubKey:   logTree.GetPublicKey(),
-		mapPubKey:   mapTree.GetPublicKey(),
+		logVerifier: logverifierCli,
+		mapID:       mapTree.TreeId,
+		mapHasher:   mapHasher,
+		mapPubKey:   mapPubKey,
 		signer:      signer,
 		store:       store,
 	}, nil
@@ -70,7 +70,7 @@ func (m *Monitor) Process(resp *ktpb.GetMutationsResponse) error {
 	var smr *trillian.SignedMapRoot
 	var err error
 	seen := time.Now().Unix()
-	errs := m.verifyMutationsResponse(resp)
+	errs := m.VerifyMutationsResponse(resp)
 	if len(errs) == 0 {
 		glog.Infof("Successfully verified mutations response for epoch: %v", resp.Epoch)
 		smr, err = m.signMapRoot(resp)

core/monitor/verify.go

@@ -18,14 +18,190 @@
 package monitor
 
 import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"math/big"
+
+	"github.com/google/keytransparency/core/mutator/entry"
+
+	"github.com/golang/glog"
+	tcrypto "github.com/google/trillian/crypto"
+	"github.com/google/trillian/merkle"
+	"github.com/google/trillian/storage"
+
 	ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types"
 )
 
-// verifyMutationsResponse verifies a response received by the GetMutations API.
+var (
+	// ErrInconsistentProofs occurs when the server returned different hashes
+	// for the same inclusion proof node in the tree.
+	ErrInconsistentProofs = errors.New("inconsistent inclusion proofs")
+	// ErrInvalidLogConsistencyProof occurs when the log consistency proof does
+	// not verify.
+	ErrInvalidLogConsistencyProof = errors.New("invalid log consistency proof")
+	// ErrInvalidLogInclusion occurs if the inclusion proof for the signed map
+	// root into the log does not verify.
+	ErrInvalidLogInclusion = errors.New("invalid log inclusion proof")
+	// ErrInvalidLogSignature occurs if the log roots signature does not verify.
+	ErrInvalidLogSignature = errors.New("invalid signature on log root")
+	// ErrInvalidMapSignature occurs if the map roots signature does not verify.
+	ErrInvalidMapSignature = errors.New("invalid signature on map root")
+	// ErrInvalidMutation occurs when verification failed because of an invalid
+	// mutation.
+	ErrInvalidMutation = errors.New("invalid mutation")
+	// ErrNotMatchingMapRoot occurs when the reconstructed root differs from the
+	// one we received from the server.
+	ErrNotMatchingMapRoot = errors.New("recreated root does not match")
+)
+
+// VerifyMutationsResponse verifies a response received by the GetMutations API.
 // Additionally to the response it takes a complete list of mutations. The list
 // of received mutations may differ from those included in the initial response
-// because of the max. page size. If any verification check failed it returns
-// an error.
-func (m *Monitor) verifyMutationsResponse(in *ktpb.GetMutationsResponse) []error {
+// because of the max. page size.
+func (m *Monitor) VerifyMutationsResponse(in *ktpb.GetMutationsResponse) []error {
+	errList := make([]error, 0)
+
+	if m.trusted == nil {
+		m.trusted = in.GetLogRoot()
+	}
+
+	if err := m.logVerifier.VerifyRoot(m.trusted, in.GetLogRoot(), in.GetLogConsistency()); err != nil {
+		// this could be one of ErrInvalidLogSignature, ErrInvalidLogConsistencyProof
+		errList = append(errList, err)
+	}
+	// updated trusted log root
+	m.trusted = in.GetLogRoot()
+
+	b, err := json.Marshal(in.GetSmr())
+	if err != nil {
+		glog.Errorf("json.Marshal(): %v", err)
+		errList = append(errList, ErrInvalidMapSignature)
+	}
+	leafIndex := in.GetSmr().GetMapRevision()
+	treeSize := in.GetLogRoot().GetTreeSize()
+	err = m.logVerifier.VerifyInclusionAtIndex(in.GetLogRoot(), b, leafIndex, in.GetLogInclusion())
+	if err != nil {
+		glog.Errorf("m.logVerifier.VerifyInclusionAtIndex((%v, %v, _): %v", leafIndex, treeSize, err)
+		errList = append(errList, ErrInvalidLogInclusion)
+	}
+
+	// copy of singed map root
+	smr := *in.GetSmr()
+	// reset to the state before it was signed:
+	smr.Signature = nil
+	// verify signature on map root:
+	if err := tcrypto.VerifyObject(m.mapPubKey, smr, in.GetSmr().GetSignature()); err != nil {
+		glog.Infof("couldn't verify signature on map root: %v", err)
+		errList = append(errList, ErrInvalidMapSignature)
+	}
+
+	// we need the old root for verifying the inclusion of the old leafs in the
+	// previous epoch. Storage always stores the mutations response independent
+	// from if the checks succeeded or not.
+	var oldRoot []byte
+	// mutations happen after epoch 1 which is stored in storage:
+	if m.store.LatestEpoch() > 0 {
+		// retrieve the old root hash from storage
+		monRes, err := m.store.Get(in.Epoch - 1)
+		if err != nil {
+			glog.Infof("Could not retrieve previous monitoring result: %v", err)
+		}
+		oldRoot = monRes.Response.GetSmr().GetRootHash()
+
+		if err := m.verifyMutations(in.GetMutations(), oldRoot,
+			in.GetSmr().GetRootHash(), in.GetSmr().GetMapId()); len(err) > 0 {
+			errList = append(errList, err...)
+		}
+	}
+
+	return errList
+}
+
+func (m *Monitor) verifyMutations(muts []*ktpb.Mutation, oldRoot, expectedNewRoot []byte, mapID int64) []error {
+	errList := make([]error, 0)
+	mutator := entry.New()
+	oldProofNodes := make(map[string][]byte)
+	newLeaves := make([]merkle.HStar2LeafHash, 0, len(muts))
+	glog.Infof("verifyMutations() called with %v mutations.", len(muts))
+
+	for _, mut := range muts {
+		oldLeaf, err := entry.FromLeafValue(mut.GetProof().GetLeaf().GetLeafValue())
+		if err != nil {
+			errList = append(errList, ErrInvalidMutation)
+		}
+
+		// verify that the provided leaf’s inclusion proof goes to epoch e-1:
+		index := mut.GetProof().GetLeaf().GetIndex()
+		leaf := mut.GetProof().GetLeaf().GetLeafValue()
+		if err := merkle.VerifyMapInclusionProof(mapID, index,
+			leaf, oldRoot, mut.GetProof().GetInclusion(), m.mapHasher); err != nil {
+			glog.Infof("VerifyMapInclusionProof(%x): %v", index, err)
+			errList = append(errList, ErrInvalidMutation)
+		}
+
+		// compute the new leaf
+		newLeaf, err := mutator.Mutate(oldLeaf, mut.GetUpdate())
+		if err != nil {
+			glog.Infof("Mutation did not verify: %v", err)
+			errList = append(errList, ErrInvalidMutation)
+		}
+		newLeafnID := storage.NewNodeIDFromPrefixSuffix(index, storage.Suffix{}, m.mapHasher.BitLen())
+		newLeafHash := m.mapHasher.HashLeaf(mapID, index, newLeaf)
+		newLeaves = append(newLeaves, merkle.HStar2LeafHash{
+			Index:    newLeafnID.BigInt(),
+			LeafHash: newLeafHash,
+		})
+
+		// store the proof hashes locally to recompute the tree below:
+		sibIDs := newLeafnID.Siblings()
+		proofs := mut.GetProof().GetInclusion()
+		for level, sibID := range sibIDs {
+			proof := proofs[level]
+			if p, ok := oldProofNodes[sibID.String()]; ok {
+				// sanity check: for each mut overlapping proof nodes should be
+				// equal:
+				if !bytes.Equal(p, proof) {
+					// this is really odd and should never happen
+					errList = append(errList, ErrInconsistentProofs)
+				}
+			} else {
+				if len(proof) > 0 {
+					oldProofNodes[sibID.String()] = proof
+				}
+			}
+		}
+	}
+
+	if err := m.validateMapRoot(expectedNewRoot, mapID, newLeaves, oldProofNodes); err != nil {
+		errList = append(errList, err)
+	}
+
+	return errList
+}
+
+func (m *Monitor) validateMapRoot(expectedRoot []byte, mapID int64, mutatedLeaves []merkle.HStar2LeafHash, oldProofNodes map[string][]byte) error {
+	// compute the new root using local intermediate hashes from epoch e
+	// (above proof hashes):
+	hs2 := merkle.NewHStar2(mapID, m.mapHasher)
+	newRoot, err := hs2.HStar2Nodes([]byte{}, m.mapHasher.BitLen(), mutatedLeaves,
+		func(depth int, index *big.Int) ([]byte, error) {
+			nID := storage.NewNodeIDFromBigInt(depth, index, m.mapHasher.BitLen())
+			if p, ok := oldProofNodes[nID.String()]; ok {
+				return p, nil
+			}
+			return nil, nil
+		}, nil)
+
+	if err != nil {
+		glog.Errorf("hs2.HStar2Nodes(_): %v", err)
+		return ErrNotMatchingMapRoot
+	}
+
+	// verify rootHash
+	if !bytes.Equal(newRoot, expectedRoot) {
+		return ErrNotMatchingMapRoot
+	}
+
 	return nil
 }

core/mutation/mutation.go

@@ -106,18 +106,24 @@ func (s *Server) GetMutations(ctx context.Context, in *tpb.GetMutationsRequest)
 	}
 	// Get leaf proofs.
 	// TODO: allow leaf proofs to be optional.
-	proofs, err := s.inclusionProofs(ctx, indexes, in.Epoch)
+	var epoch int64
+	if in.Epoch > 1 {
+		epoch = in.Epoch - 1
+	} else {
+		epoch = 1
+	}
+	proofs, err := s.inclusionProofs(ctx, indexes, epoch)
 	if err != nil {
 		return nil, err
 	}
 	for i, p := range proofs {
 		mutations[i].Proof = p
 	}
 
-	// MapRevisions start at 1. Log leave's index starts at 0.
+	// MapRevisions start at 1. Log leave's index starts at 1.
 	// MapRevision should be at least 1 since the Signer is
 	// supposed to create at least one revision on startup.
-	respEpoch := resp.GetMapRoot().GetMapRevision() - 1
+	respEpoch := resp.GetMapRoot().GetMapRevision()
 	// Fetch log proofs.
 	logRoot, logConsistency, logInclusion, err := s.logProofs(ctx, in.GetFirstTreeSize(), respEpoch)
 	if err != nil {

impl/monitor/client/client.go

@@ -64,9 +64,9 @@ func (c *Client) StartPolling(startEpoch int64) (<-chan *ktpb.GetMutationsRespon
 			glog.Infof("Polling: %v", now)
 			// time out if we exceed the poll period:
 			ctx, _ := context.WithTimeout(context.Background(), c.pollPeriod)
-			monitorResp, err := c.pollMutations(ctx, epoch)
+			monitorResp, err := c.PollMutations(ctx, epoch)
 			if err != nil {
-				glog.Infof("pollMutations(_): %v", err)
+				glog.Infof("PollMutations(_): %v", err)
 				errChan <- err
 			} else {
 				// only write to results channel and increment epoch
@@ -80,7 +80,9 @@ func (c *Client) StartPolling(startEpoch int64) (<-chan *ktpb.GetMutationsRespon
 	return response, errChan
 }
 
-func (c *Client) pollMutations(ctx context.Context,
+// PollMutations polls GetMutationsResponses from the configured
+// key-transparency server's mutations API.
+func (c *Client) PollMutations(ctx context.Context,
 	queryEpoch int64,
 	opts ...grpc.CallOption) (*ktpb.GetMutationsResponse, error) {
 	response, err := c.client.GetMutations(ctx, &ktpb.GetMutationsRequest{