Skip to content
Permalink
Browse files

Use new TrillianMapWrite API (#1304)

* go get github.com/google/trillian@HEAD

* Use new TrillianMapWrite API
  • Loading branch information...
gdbelvin committed Jul 2, 2019
1 parent ca16bec commit f73ff7048b226dc84a975459f138a9882af1a5e9
Showing with 538 additions and 41 deletions.
  1. +1 −0 cmd/keytransparency-sequencer/main.go
  2. +11 −9 core/sequencer/server.go
  3. +16 −3 core/sequencer/server_test.go
  4. +40 −2 core/sequencer/trillian_client.go
  5. +61 −26 go.mod
  6. +408 −0 go.sum
  7. +1 −1 impl/integration/env.go
@@ -169,6 +169,7 @@ func main() {
directoryStorage,
trillian.NewTrillianLogClient(lconn),
trillian.NewTrillianMapClient(mconn),
trillian.NewTrillianMapWriteClient(mconn),
mutations, mutations,
spb.NewKeyTransparencySequencerClient(conn),
prometheus.MetricFactory{}))
@@ -164,6 +164,7 @@ func NewServer(
directories directory.Storage,
tlog tpb.TrillianLogClient,
tmap tpb.TrillianMapClient,
twrite tpb.TrillianMapWriteClient,
batcher Batcher,
logs LogsReader,
loopback spb.KeyTransparencySequencerClient,
@@ -176,6 +177,7 @@ func NewServer(
directories: directories,
tmap: tmap,
tlog: tlog,
twrite: twrite,
},
batcher: batcher,
logs: logs,
@@ -384,13 +386,13 @@ func (s *Server) ApplyRevision(ctx context.Context, in *spb.ApplyRevisionRequest
}

// Read Map.
mapClient, err := s.trillian.MapClient(ctx, in.DirectoryId)
mapClient, err := s.trillian.MapWriteClient(ctx, in.DirectoryId)
if err != nil {
return nil, err
}
verifyLeafStart := time.Now()
leaves, err := mapClient.GetMapLeavesByRevisionNoProof(ctx, in.Revision-1, indexes)
fnLatency.Observe(time.Since(verifyLeafStart).Seconds(), in.DirectoryId, "GetAndVerifyMapLeavesByRevision")
leaves, err := mapClient.GetLeavesByRevision(ctx, in.Revision-1, indexes)
fnLatency.Observe(time.Since(verifyLeafStart).Seconds(), in.DirectoryId, "GetLeavesByRevision")
if err != nil {
return nil, err
}
@@ -421,20 +423,20 @@ func (s *Server) ApplyRevision(ctx context.Context, in *spb.ApplyRevisionRequest

// Set new leaf values.
setRevisionStart := time.Now()
mapRoot, err := mapClient.SetLeavesAtRevision(ctx, in.Revision, newLeaves, metadata)
fnLatency.Observe(time.Since(setRevisionStart).Seconds(), in.DirectoryId, "SetLeavesAtRevision")
err = mapClient.WriteLeaves(ctx, in.Revision, newLeaves, metadata)
fnLatency.Observe(time.Since(setRevisionStart).Seconds(), in.DirectoryId, "WriteLeaves")
if err != nil {
return nil, status.Errorf(codes.Internal, "VerifySignedMapRoot(): %v", err)
return nil, err
}
glog.V(2).Infof("CreateRevision: SetLeaves:{Revision: %v}", mapRoot.Revision)
glog.V(2).Infof("CreateRevision: WriteLeaves:{Revision: %v}", in.Revision)

for _, s := range meta.Sources {
watermarkApplied.Set(float64(s.HighestExclusive), in.DirectoryId, fmt.Sprintf("%v", s.LogId))
}
mapLeafCount.Add(float64(len(newLeaves)), in.DirectoryId)
mapRevisionCount.Inc(in.DirectoryId)
glog.Infof("ApplyRevision(): dir: %v, rev: %v, root: %x, mutations: %v, indexes: %v, newleaves: %v",
in.DirectoryId, mapRoot.Revision, mapRoot.RootHash, len(logItems), len(indexes), len(newLeaves))
glog.Infof("ApplyRevision(): dir: %v, rev: %v, mutations: %v, indexes: %v, newleaves: %v",
in.DirectoryId, in.Revision, len(logItems), len(indexes), len(newLeaves))
return &spb.ApplyRevisionResponse{
DirectoryId: in.DirectoryId,
Revision: in.Revision,
@@ -23,6 +23,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/trillian/monitoring"
"github.com/google/trillian/types"
"google.golang.org/grpc"

"github.com/google/keytransparency/core/mutator"
"github.com/google/keytransparency/core/sequencer/mapper"
@@ -70,14 +71,17 @@ func (l fakeLogs) HighWatermark(ctx context.Context, directoryID string, logID,
}

type fakeTrillianFactory struct {
tmap trillianMap
tlog trillianLog
tmap trillianMap
tlog trillianLog
twrite *MapWriteClient
}

func (t *fakeTrillianFactory) MapClient(_ context.Context, _ string) (trillianMap, error) {
return t.tmap, nil
}

func (t *fakeTrillianFactory) MapWriteClient(_ context.Context, _ string) (*MapWriteClient, error) {
return t.twrite, nil
}
func (t *fakeTrillianFactory) LogClient(_ context.Context, _ string) (trillianLog, error) {
return t.tlog, nil
}
@@ -91,6 +95,15 @@ func (m *fakeMap) GetAndVerifyLatestMapRoot(_ context.Context) (*tpb.SignedMapRo
return nil, m.latestMapRoot, nil
}

type fakeWrite struct{}

func (m *fakeWrite) GetLeavesByRevision(ctx context.Context, in *tpb.GetMapLeavesByRevisionRequest, opts ...grpc.CallOption) (*tpb.MapLeaves, error) {
return nil, nil
}
func (m *fakeWrite) WriteLeaves(ctx context.Context, in *tpb.WriteMapLeavesRequest, opts ...grpc.CallOption) (*tpb.WriteMapLeavesResponse, error) {
return nil, nil
}

type fakeBatcher struct {
highestRev int64
batches map[int64]*spb.MapMetadata
@@ -31,15 +31,14 @@ import (
// trillianFactory creates verifying clients for Trillian
type trillianFactory interface {
MapClient(ctx context.Context, dirID string) (trillianMap, error)
MapWriteClient(ctx context.Context, dirID string) (*MapWriteClient, error)
LogClient(ctx context.Context, dirID string) (trillianLog, error)
}

// trillianMap communicates with the Trilian map and verifies the responses.
type trillianMap interface {
GetAndVerifyLatestMapRoot(ctx context.Context) (*tpb.SignedMapRoot, *types.MapRootV1, error)
SetLeavesAtRevision(ctx context.Context, rev int64, leaves []*tpb.MapLeaf, meta []byte) (*types.MapRootV1, error)
GetAndVerifyMapRootByRevision(ctx context.Context, rev int64) (*tpb.SignedMapRoot, *types.MapRootV1, error)
GetMapLeavesByRevisionNoProof(ctx context.Context, rev int64, indexes [][]byte) ([]*tpb.MapLeaf, error)
}

// trillianLog communicates with the Trillian log and verifies the responses.
@@ -54,6 +53,21 @@ type Trillian struct {
directories directory.Storage
tmap tpb.TrillianMapClient
tlog tpb.TrillianLogClient
twrite tpb.TrillianMapWriteClient
}

// MapWriteClient returns a connection to the map write API.
func (t *Trillian) MapWriteClient(ctx context.Context, dirID string) (*MapWriteClient, error) {
directory, err := t.directories.Read(ctx, dirID, false)
if err != nil {
glog.Errorf("directories.Read(%v): %v", dirID, err)
return nil, status.Errorf(codes.Internal, "Cannot fetch directory info for %v", dirID)
}

return &MapWriteClient{
MapID: directory.Map.TreeId,
twrite: t.twrite,
}, nil
}

// MapClient returns a verifying MapClient
@@ -84,6 +98,30 @@ func (t *Trillian) LogClient(ctx context.Context, dirID string) (trillianLog, er
return tclient.NewFromTree(t.tlog, directory.Log, trustedRoot)
}

type MapWriteClient struct {
MapID int64
twrite tpb.TrillianMapWriteClient
}

func (c *MapWriteClient) GetLeavesByRevision(ctx context.Context, rev int64, indexes [][]byte) ([]*tpb.MapLeaf, error) {
mapLeaves, err := c.twrite.GetLeavesByRevision(ctx, &tpb.GetMapLeavesByRevisionRequest{
MapId: c.MapID,
Revision: rev,
Index: indexes,
})
return mapLeaves.GetLeaves(), err
}

func (c *MapWriteClient) WriteLeaves(ctx context.Context, rev int64, leaves []*tpb.MapLeaf, metadata []byte) error {
_, err := c.twrite.WriteLeaves(ctx, &tpb.WriteMapLeavesRequest{
MapId: c.MapID,
Leaves: leaves,
ExpectRevision: rev,
Metadata: metadata,
})
return err
}

// MapClient interacts with the Trillian Map and verifies its responses.
type MapClient struct {
*tclient.MapClient
87 go.mod
@@ -3,47 +3,82 @@ module github.com/google/keytransparency
go 1.12

require (
cloud.google.com/go v0.40.0 // indirect
9fans.net/go v0.0.2 // indirect
github.com/OpenPeeDeeP/depguard v1.0.0 // indirect
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/alecthomas/gometalinter v3.0.0+incompatible // indirect
github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1
github.com/coreos/bbolt v1.3.3 // indirect
github.com/coreos/etcd v3.3.13+incompatible // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/go-logfmt/logfmt v0.4.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
github.com/cosiner/argv v0.0.1 // indirect
github.com/davidrjenni/reftools v0.0.0-20190411195930-981bbac422f8 // indirect
github.com/fatih/gomodifytags v0.0.0-20190517204355-df91c5bc7551 // indirect
github.com/fatih/motion v0.0.0-20190527122956-41470362fad4 // indirect
github.com/go-delve/delve v1.2.0 // indirect
github.com/go-kit/kit v0.9.0 // indirect
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/go-sql-driver/mysql v1.4.1
github.com/go-toolsmith/astcast v1.0.0 // indirect
github.com/go-toolsmith/astcopy v1.0.0 // indirect
github.com/go-toolsmith/astfmt v1.0.0 // indirect
github.com/go-toolsmith/astp v1.0.0 // indirect
github.com/go-toolsmith/pkgload v1.0.0 // indirect
github.com/go-toolsmith/typep v1.0.0 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/golang/mock v1.3.1
github.com/golang/protobuf v1.3.1
github.com/google/certificate-transparency-go v1.0.21 // indirect
github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c
github.com/google/go-cmp v0.3.0
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f // indirect
github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect
github.com/google/tink v1.2.1-0.20190523150020-6495d823d968
github.com/google/trillian v1.2.2-0.20190603160524-0cfa53f919f3
github.com/google/trillian v1.2.2-0.20190628151111-5e2c3d733100
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/grpc-ecosystem/grpc-gateway v1.9.0
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.9.3
github.com/josharian/impl v0.0.0-20180228163738-3d0f908298c4 // indirect
github.com/jstemmer/gotags v1.4.1 // indirect
github.com/keegancsmith/rpc v1.1.0 // indirect
github.com/kisielk/errcheck v1.2.0 // indirect
github.com/klauspost/asmfmt v1.2.0 // indirect
github.com/klauspost/compress v1.7.1 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect
github.com/koron/iferr v0.0.0-20180615142939-bb332a3b1d91 // indirect
github.com/kr/pretty v0.1.0
github.com/kr/pty v1.1.7 // indirect
github.com/kylelemons/godebug v1.1.0
github.com/lib/pq v1.1.1 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/logrusorgru/aurora v0.0.0-20190428105938-cea283e61946 // indirect
github.com/mattn/go-sqlite3 v1.10.0
github.com/pelletier/go-toml v1.4.0 // indirect
github.com/pkg/errors v0.8.1 // indirect
github.com/prometheus/client_golang v0.9.4
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spf13/afero v1.2.2 // indirect
github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f // indirect
github.com/peterh/liner v1.1.0 // indirect
github.com/pkg/profile v1.3.0 // indirect
github.com/prometheus/client_golang v1.0.0
github.com/prometheus/common v0.6.0 // indirect
github.com/prometheus/procfs v0.0.3 // indirect
github.com/rogpeppe/fastuuid v1.1.0 // indirect
github.com/rogpeppe/go-internal v1.3.0 // indirect
github.com/rogpeppe/godef v1.1.1 // indirect
github.com/russross/blackfriday v2.0.0+incompatible // indirect
github.com/shirou/gopsutil v2.18.12+incompatible // indirect
github.com/spf13/cobra v0.0.5
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.4.0
go.etcd.io/bbolt v1.3.3 // indirect
go.opencensus.io v0.22.0 // indirect
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5
golang.org/x/net v0.0.0-20190607181551-461777fb6f67 // indirect
github.com/stamblerre/gocode v0.0.0-20190327203809-810592086997 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/ugorji/go v1.1.5-pre // indirect
github.com/valyala/fasthttp v1.3.0 // indirect
github.com/zmb3/gogetdoc v0.0.0-20190228002656-b37376c5da6a // indirect
golang.org/x/arch v0.0.0-20190312162104-788fe5ffcd8c // indirect
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
golang.org/x/exp v0.0.0-20190627132806-fd42eb6b336f // indirect
golang.org/x/image v0.0.0-20190622003408-7e034cad6442 // indirect
golang.org/x/mobile v0.0.0-20190607214518-6fa95d984e88 // indirect
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76 // indirect
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
google.golang.org/api v0.6.0
google.golang.org/appengine v1.6.1 // indirect
google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6
golang.org/x/tools v0.0.0-20190628222527-fb37f6ba8261 // indirect
google.golang.org/api v0.7.0
google.golang.org/genproto v0.0.0-20190627203621-eb59cef1c072
google.golang.org/grpc v1.21.1
gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c // indirect
)

0 comments on commit f73ff70

Please sign in to comment.
You can’t perform that action at this time.