forked from zmap/zcrypto
/
logclient.go
403 lines (371 loc) · 12.5 KB
/
logclient.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
// Package client is a CT log client implementation and contains types and code
// for interacting with RFC6962-compliant CT Log instances.
// See http://tools.ietf.org/html/rfc6962 for details
package client
import (
"bytes"
"crypto/sha256"
"crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"strconv"
"time"
"github.com/mreiferson/go-httpclient"
"github.com/oof-security/zcrypto/ct"
"golang.org/x/net/context"
)
// URI paths for CT Log endpoints
const (
AddChainPath = "/ct/v1/add-chain"
AddPreChainPath = "/ct/v1/add-pre-chain"
AddJSONPath = "/ct/v1/add-json"
GetSTHPath = "/ct/v1/get-sth"
GetEntriesPath = "/ct/v1/get-entries"
)
// LogClient represents a client for a given CT Log instance
type LogClient struct {
Uri string // the base URI of the log. e.g. http://ct.googleapis/pilot
httpClient *http.Client // used to interact with the log via HTTP
}
//////////////////////////////////////////////////////////////////////////////////
// JSON structures follow.
// These represent the structures returned by the CT Log server.
//////////////////////////////////////////////////////////////////////////////////
// addChainRequest represents the JSON request body sent to the add-chain CT
// method.
type addChainRequest struct {
Chain []string `json:"chain"`
}
// addChainResponse represents the JSON response to the add-chain CT method.
// An SCT represents a Log's promise to integrate a [pre-]certificate into the
// log within a defined period of time.
type addChainResponse struct {
SCTVersion ct.Version `json:"sct_version"` // SCT structure version
ID string `json:"id"` // Log ID
Timestamp uint64 `json:"timestamp"` // Timestamp of issuance
Extensions string `json:"extensions"` // Holder for any CT extensions
Signature string `json:"signature"` // Log signature for this SCT
}
// addJSONRequest represents the JSON request body sent ot the add-json CT
// method.
type addJSONRequest struct {
Data interface{} `json:"data"`
}
// getSTHResponse respresents the JSON response to the get-sth CT method
type getSTHResponse struct {
TreeSize uint64 `json:"tree_size"` // Number of certs in the current tree
Timestamp uint64 `json:"timestamp"` // Time that the tree was created
SHA256RootHash string `json:"sha256_root_hash"` // Root hash of the tree
TreeHeadSignature string `json:"tree_head_signature"` // Log signature for this STH
}
// base64LeafEntry respresents a Base64 encoded leaf entry
type base64LeafEntry struct {
LeafInput string `json:"leaf_input"`
ExtraData string `json:"extra_data"`
}
// getEntriesReponse respresents the JSON response to the CT get-entries method
type getEntriesResponse struct {
Entries []base64LeafEntry `json:"entries"` // the list of returned entries
}
// getConsistencyProofResponse represents the JSON response to the CT get-consistency-proof method
type getConsistencyProofResponse struct {
Consistency []string `json:"consistency"`
}
// getAuditProofResponse represents the JSON response to the CT get-audit-proof method
type getAuditProofResponse struct {
Hash []string `json:"hash"` // the hashes which make up the proof
TreeSize uint64 `json:"tree_size"` // the tree size against which this proof is constructed
}
// getAcceptedRootsResponse represents the JSON response to the CT get-roots method.
type getAcceptedRootsResponse struct {
Certificates []string `json:"certificates"`
}
// getEntryAndProodReponse represents the JSON response to the CT get-entry-and-proof method
type getEntryAndProofResponse struct {
LeafInput string `json:"leaf_input"` // the entry itself
ExtraData string `json:"extra_data"` // any chain provided when the entry was added to the log
AuditPath []string `json:"audit_path"` // the corresponding proof
}
// New constructs a new LogClient instance.
// |uri| is the base URI of the CT log instance to interact with, e.g.
// http://ct.googleapis.com/pilot
func New(uri string) *LogClient {
var c LogClient
c.Uri = uri
transport := &httpclient.Transport{
ConnectTimeout: 10 * time.Second,
RequestTimeout: 30 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConnsPerHost: 10,
DisableKeepAlives: false,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
c.httpClient = &http.Client{Transport: transport}
return &c
}
// Makes a HTTP call to |uri|, and attempts to parse the response as a JSON
// representation of the structure in |res|.
// Returns a non-nil |error| if there was a problem.
func (c *LogClient) fetchAndParse(uri string, res interface{}) error {
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return err
}
resp, err := c.httpClient.Do(req)
var body []byte
if resp != nil {
if resp.StatusCode > 399 {
return errors.New("HTTP error: " + resp.Status)
}
body, err = ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return err
}
}
if err != nil {
return err
}
if err = json.Unmarshal(body, &res); err != nil {
fmt.Println(string(body))
return err
}
return nil
}
// Makes a HTTP POST call to |uri|, and attempts to parse the response as a JSON
// representation of the structure in |res|.
// Returns a non-nil |error| if there was a problem.
func (c *LogClient) postAndParse(uri string, req interface{}, res interface{}) (*http.Response, string, error) {
postBody, err := json.Marshal(req)
if err != nil {
return nil, "", err
}
httpReq, err := http.NewRequest("POST", uri, bytes.NewReader(postBody))
if err != nil {
return nil, "", err
}
//httpReq.Header.Set("Keep-Alive", "timeout=15, max=100")
httpReq.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(httpReq)
// Read all of the body, if there is one, so that the http.Client can do
// Keep-Alive:
var body []byte
if resp != nil {
body, err = ioutil.ReadAll(resp.Body)
resp.Body.Close()
}
if err != nil {
return resp, string(body), err
}
if resp.StatusCode == 200 {
if err != nil {
return resp, string(body), err
}
if err = json.Unmarshal(body, &res); err != nil {
return resp, string(body), err
}
}
return resp, string(body), nil
}
func backoffForRetry(ctx context.Context, d time.Duration) error {
backoffTimer := time.NewTimer(d)
if ctx != nil {
select {
case <-ctx.Done():
return ctx.Err()
case <-backoffTimer.C:
}
} else {
<-backoffTimer.C
}
return nil
}
// Attempts to add |chain| to the log, using the api end-point specified by
// |path|. If provided context expires before submission is complete an
// error will be returned.
func (c *LogClient) addChainWithRetry(ctx context.Context, path string, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) {
var resp addChainResponse
var req addChainRequest
for _, link := range chain {
req.Chain = append(req.Chain, base64.StdEncoding.EncodeToString(link))
}
httpStatus := "Unknown"
httpCode := 0
backoffSeconds := 0
done := false
for !done {
if backoffSeconds > 0 {
log.Printf("Got %s, backing-off %d seconds", httpStatus, backoffSeconds)
}
err := backoffForRetry(ctx, time.Second*time.Duration(backoffSeconds))
if err != nil {
return nil, err, 0
}
if backoffSeconds > 0 {
backoffSeconds = 0
}
httpResp, errorBody, err := c.postAndParse(c.Uri+path, &req, &resp)
if err != nil {
backoffSeconds = 10
continue
}
switch {
case httpResp.StatusCode == 200:
done = true
case httpResp.StatusCode == 408:
// request timeout, retry immediately
case httpResp.StatusCode == 503:
// Retry
backoffSeconds = 10
if retryAfter := httpResp.Header.Get("Retry-After"); retryAfter != "" {
if seconds, err := strconv.Atoi(retryAfter); err == nil {
backoffSeconds = seconds
}
}
default:
return nil, fmt.Errorf("got HTTP Status %s: %s", httpResp.Status, errorBody), httpResp.StatusCode
}
httpStatus = httpResp.Status
httpCode = httpResp.StatusCode
}
rawLogID, err := base64.StdEncoding.DecodeString(resp.ID)
if err != nil {
return nil, err, httpCode
}
rawSignature, err := base64.StdEncoding.DecodeString(resp.Signature)
if err != nil {
return nil, err, httpCode
}
ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(rawSignature))
if err != nil {
return nil, err, httpCode
}
var logID ct.SHA256Hash
copy(logID[:], rawLogID)
return &ct.SignedCertificateTimestamp{
SCTVersion: resp.SCTVersion,
LogID: logID,
Timestamp: resp.Timestamp,
Extensions: ct.CTExtensions(resp.Extensions),
Signature: *ds}, nil, httpCode
}
// AddChain adds the (DER represented) X509 |chain| to the log.
func (c *LogClient) AddChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) {
return c.addChainWithRetry(nil, AddChainPath, chain)
}
// AddPreChain adds the (DER represented) Precertificate |chain| to the log.
func (c *LogClient) AddPreChain(chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) {
return c.addChainWithRetry(nil, AddPreChainPath, chain)
}
// AddChainWithContext adds the (DER represented) X509 |chain| to the log and
// fails if the provided context expires before the chain is submitted.
func (c *LogClient) AddChainWithContext(ctx context.Context, chain []ct.ASN1Cert) (*ct.SignedCertificateTimestamp, error, int) {
return c.addChainWithRetry(ctx, AddChainPath, chain)
}
func (c *LogClient) AddJSON(data interface{}) (*ct.SignedCertificateTimestamp, error) {
req := addJSONRequest{
Data: data,
}
var resp addChainResponse
_, _, err := c.postAndParse(c.Uri+AddJSONPath, &req, &resp)
if err != nil {
return nil, err
}
rawLogID, err := base64.StdEncoding.DecodeString(resp.ID)
if err != nil {
return nil, err
}
rawSignature, err := base64.StdEncoding.DecodeString(resp.Signature)
if err != nil {
return nil, err
}
ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(rawSignature))
if err != nil {
return nil, err
}
var logID ct.SHA256Hash
copy(logID[:], rawLogID)
return &ct.SignedCertificateTimestamp{
SCTVersion: resp.SCTVersion,
LogID: logID,
Timestamp: resp.Timestamp,
Extensions: ct.CTExtensions(resp.Extensions),
Signature: *ds}, nil
}
// GetSTH retrieves the current STH from the log.
// Returns a populated SignedTreeHead, or a non-nil error.
func (c *LogClient) GetSTH() (sth *ct.SignedTreeHead, err error) {
var resp getSTHResponse
if err = c.fetchAndParse(c.Uri+GetSTHPath, &resp); err != nil {
return
}
sth = &ct.SignedTreeHead{
TreeSize: resp.TreeSize,
Timestamp: resp.Timestamp,
}
rawRootHash, err := base64.StdEncoding.DecodeString(resp.SHA256RootHash)
if err != nil {
return nil, fmt.Errorf("invalid base64 encoding in sha256_root_hash: %v", err)
}
if len(rawRootHash) != sha256.Size {
return nil, fmt.Errorf("sha256_root_hash is invalid length, expected %d got %d", sha256.Size, len(rawRootHash))
}
copy(sth.SHA256RootHash[:], rawRootHash)
rawSignature, err := base64.StdEncoding.DecodeString(resp.TreeHeadSignature)
if err != nil {
return nil, errors.New("invalid base64 encoding in tree_head_signature")
}
ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(rawSignature))
if err != nil {
return nil, err
}
// TODO(alcutter): Verify signature
sth.TreeHeadSignature = *ds
return
}
// GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT
// log server. (see section 4.6.)
// Returns a slice of LeafInputs or a non-nil error.
func (c *LogClient) GetEntries(start, end int64) ([]ct.LogEntry, error) {
if end < 0 {
return nil, errors.New("end should be >= 0")
}
if end < start {
return nil, errors.New("start should be <= end")
}
var resp getEntriesResponse
err := c.fetchAndParse(fmt.Sprintf("%s%s?start=%d&end=%d", c.Uri, GetEntriesPath, start, end), &resp)
if err != nil {
return nil, err
}
entries := make([]ct.LogEntry, len(resp.Entries))
for index, entry := range resp.Entries {
leafBytes, err := base64.StdEncoding.DecodeString(entry.LeafInput)
leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewBuffer(leafBytes))
if err != nil {
return nil, err
}
entries[index].Leaf = *leaf
chainBytes, err := base64.StdEncoding.DecodeString(entry.ExtraData)
var chain []ct.ASN1Cert
switch leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
chain, err = ct.UnmarshalX509ChainArray(chainBytes)
case ct.PrecertLogEntryType:
chain, err = ct.UnmarshalPrecertChainArray(chainBytes)
default:
return nil, fmt.Errorf("saw unknown entry type: %v", leaf.TimestampedEntry.EntryType)
}
if err != nil {
return nil, err
}
entries[index].Chain = chain
entries[index].Index = start + int64(index)
}
return entries, nil
}