forked from google/certificate-transparency-go
-
Notifications
You must be signed in to change notification settings - Fork 0
/
instance.go
283 lines (262 loc) · 9.29 KB
/
instance.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minimal
import (
"context"
"crypto"
"errors"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/golang/glog"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
"github.com/google/certificate-transparency-go/gossip/minimal/configpb"
"github.com/google/certificate-transparency-go/jsonclient"
"github.com/google/certificate-transparency-go/x509"
"github.com/google/certificate-transparency-go/x509util"
"github.com/google/trillian/crypto/keys"
"github.com/google/trillian/monitoring"
logclient "github.com/google/certificate-transparency-go/client"
hubclient "github.com/google/trillian-examples/gossip/client"
)
const (
defaultRateHz = 1.0
defaultMinInterval = 1 * time.Second
)
// NewGossiperFromFile creates a gossiper from the given filename, which should
// contain text-protobuf encoded configuration data, together with an optional
// http Client.
func NewGossiperFromFile(ctx context.Context, filename string, hc *http.Client, mf monitoring.MetricFactory) (*Gossiper, error) {
return NewBoundaryGossiperFromFile(ctx, filename, hc, hc, mf)
}
// NewBoundaryGossiperFromFile creates a gossiper that uses different
// http.Client instances for source logs and destination hubs, for example to
// allow gossiping across (some kinds of) network boundaries.
func NewBoundaryGossiperFromFile(ctx context.Context, filename string, hcLog, hcHub *http.Client, mf monitoring.MetricFactory) (*Gossiper, error) {
cfgText, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
var cfgProto configpb.GossipConfig
if err := proto.UnmarshalText(string(cfgText), &cfgProto); err != nil {
return nil, fmt.Errorf("%s: failed to parse gossip config: %v", filename, err)
}
cfg, err := NewBoundaryGossiper(ctx, &cfgProto, hcLog, hcHub, mf)
if err != nil {
return nil, fmt.Errorf("%s: config error: %v", filename, err)
}
return cfg, nil
}
// NewGossiper creates a gossiper from the given configuration protobuf and optional
// http client.
func NewGossiper(ctx context.Context, cfg *configpb.GossipConfig, hc *http.Client, mf monitoring.MetricFactory) (*Gossiper, error) {
return NewBoundaryGossiper(ctx, cfg, hc, hc, mf)
}
// NewBoundaryGossiper creates a gossiper from the given configuration protobuf
// and a pair of http.Client instances for source logs and destination hubs,
// to allow (for example) gossiping across (some kinds of) network boundaries.
func NewBoundaryGossiper(ctx context.Context, cfg *configpb.GossipConfig, hcLog, hcHub *http.Client, mf monitoring.MetricFactory) (*Gossiper, error) {
once.Do(func() { setupMetrics(mf) })
if len(cfg.DestHub) == 0 {
return nil, errors.New("no dest hub config found")
}
if len(cfg.SourceLog) == 0 {
return nil, errors.New("no source log config found")
}
needPrivKey := false
for _, destHub := range cfg.DestHub {
if !destHub.IsHub {
// Destinations include at least one CT Log, so need a private key
// for cert generation for all such destinations.
needPrivKey = true
break
}
}
var signer crypto.Signer
var root *x509.Certificate
if needPrivKey {
if cfg.PrivateKey == nil {
return nil, errors.New("no private key found")
}
var keyProto ptypes.DynamicAny
if err := ptypes.UnmarshalAny(cfg.PrivateKey, &keyProto); err != nil {
return nil, fmt.Errorf("failed to unmarshal cfg.PrivateKey: %v", err)
}
var err error
signer, err = keys.NewSigner(ctx, keyProto.Message)
if err != nil {
return nil, fmt.Errorf("failed to load private key: %v", err)
}
root, err = x509util.CertificateFromPEM([]byte(cfg.RootCert))
if err != nil {
return nil, fmt.Errorf("failed to parse root cert: %v", err)
}
}
allSTHsRate := 0.0
srcs := make(map[string]*sourceLog)
for _, lc := range cfg.SourceLog {
base, err := logConfigFromProto(lc, hcLog)
if err != nil {
return nil, fmt.Errorf("failed to parse source log config for %q: %v", lc.Name, err)
}
if _, ok := srcs[base.Name]; ok {
return nil, fmt.Errorf("duplicate source logs for name %q", base.Name)
}
glog.Infof("configured source log %s at %s (%+v)", base.Name, base.URL, base)
srcs[base.Name] = &sourceLog{logConfig: *base}
knownSourceLogs.Set(1.0, base.Name)
// Assume that each source log has a new STH when polled.
sthRate := defaultRateHz
if base.MinInterval > 0 {
sthRate = 1.0 / base.MinInterval.Seconds()
}
allSTHsRate += sthRate
}
dests := make(map[string]*destHub)
for _, lc := range cfg.DestHub {
hub, err := hubFromProto(lc, hcHub)
if err != nil {
return nil, fmt.Errorf("failed to parse dest hub config for %q: %v", lc.Name, err)
}
if _, ok := dests[hub.Name]; ok {
return nil, fmt.Errorf("duplicate dest hubs for name %q", hub.Name)
}
glog.Infof("configured dest Hub %s at %s (%+v)", hub.Name, hub.URL, hub)
dests[hub.Name] = hub
isHub := 0.0
if lc.IsHub {
isHub = 1.0
}
destPureHub.Set(isHub, hub.Name)
submitRate := defaultRateHz
if hub.MinInterval > 0 {
submitRate = 1.0 / hub.MinInterval.Seconds()
}
if allSTHsRate > submitRate {
glog.Errorf("%s: Overall STH retrieval rate (%f Hz) higher than submission limit (%f Hz) for hub, retrieved STHs may be dropped", hub.Name, allSTHsRate, submitRate)
}
}
return &Gossiper{
signer: signer,
root: root,
dests: dests,
srcs: srcs,
bufferSize: int(cfg.BufferSize),
}, nil
}
func logConfigFromProto(cfg *configpb.LogConfig, hc *http.Client) (*logConfig, error) {
if cfg.Name == "" {
return nil, errors.New("no log name provided")
}
interval, err := ptypes.Duration(cfg.MinReqInterval)
if err != nil {
return nil, fmt.Errorf("failed to parse MinReqInterval: %v", err)
}
if interval <= 0 {
interval = defaultMinInterval
}
opts := jsonclient.Options{PublicKeyDER: cfg.PublicKey.GetDer(), UserAgent: "ct-go-gossip-client/1.0"}
client, err := logclient.New(cfg.Url, hc, opts)
if err != nil {
return nil, fmt.Errorf("failed to create log client for %q: %v", cfg.Name, err)
}
if client.Verifier == nil {
glog.Warningf("No public key provided for log %s, signature checks will be skipped", cfg.Name)
}
return &logConfig{
Name: cfg.Name,
URL: cfg.Url,
Log: client,
MinInterval: interval,
}, nil
}
func hubFromProto(cfg *configpb.HubConfig, hc *http.Client) (*destHub, error) {
if cfg.Name == "" {
return nil, errors.New("no source log name provided")
}
interval, err := ptypes.Duration(cfg.MinReqInterval)
if err != nil {
return nil, fmt.Errorf("failed to parse MinReqInterval: %v", err)
}
if interval <= 0 {
interval = defaultMinInterval
}
var submitter hubSubmitter
opts := jsonclient.Options{PublicKeyDER: cfg.PublicKey.GetDer(), UserAgent: "ct-go-gossip-hub/1.0"}
if cfg.IsHub {
cl, err := hubclient.New(cfg.Url, hc, opts)
if err != nil {
return nil, fmt.Errorf("failed to create hub client for %q: %v", cfg.Name, err)
}
if cl.Verifier == nil {
glog.Warningf("No public key provided for hub %s, signature checks will be skipped", cfg.Name)
}
submitter = &pureHubSubmitter{cl}
} else {
cl, err := logclient.New(cfg.Url, hc, opts)
if err != nil {
return nil, fmt.Errorf("failed to create log client for %q: %v", cfg.Name, err)
}
if cl.Verifier == nil {
glog.Warningf("No public key provided for CT log %s, signature checks will be skipped", cfg.Name)
}
submitter = &ctLogSubmitter{cl}
}
return &destHub{
Name: cfg.Name,
URL: cfg.Url,
Submitter: submitter,
MinInterval: interval,
lastHubSubmission: make(map[string]time.Time),
}, nil
}
func hubScannerFromProto(cfg *configpb.HubConfig, hc *http.Client) (*hubScanner, error) {
if cfg.Name == "" {
return nil, errors.New("no hub name provided")
}
interval, err := ptypes.Duration(cfg.MinReqInterval)
if err != nil {
return nil, fmt.Errorf("failed to parse MinReqInterval: %v", err)
}
opts := jsonclient.Options{PublicKeyDER: cfg.PublicKey.GetDer(), UserAgent: "ct-go-gossip-scanner/1.0"}
var fetcher hubFetcher
if cfg.IsHub {
cl, err := hubclient.New(cfg.Url, hc, opts)
if err != nil {
return nil, fmt.Errorf("failed to create hub client for Gossip Hub %q: %v", cfg.Name, err)
}
if cl.Verifier == nil {
glog.Warningf("No public key provided for Gossip Hub %s, signature checks will be skipped", cfg.Name)
}
fetcher = &gossipHubFetcher{Hub: cl}
} else {
cl, err := logclient.New(cfg.Url, hc, opts)
if err != nil {
return nil, fmt.Errorf("failed to create hub client for CT log %q: %v", cfg.Name, err)
}
if cl.Verifier == nil {
glog.Warningf("No public key provided for CT log %s, signature checks will be skipped", cfg.Name)
}
fetcher = &ctHubFetcher{Log: cl}
}
return &hubScanner{
Name: cfg.Name,
URL: cfg.Url,
MinInterval: interval,
cfgStartIndex: cfg.StartIndex,
fetcher: fetcher,
}, nil
}