-
Notifications
You must be signed in to change notification settings - Fork 8
/
crl_monitor.go
350 lines (306 loc) · 10.1 KB
/
crl_monitor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
/* crt.sh: crl_monitor - CRL Monitor
* Written by Rob Stradling
* Copyright (C) 2017-2020 Sectigo Limited
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"bytes"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"database/sql"
"encoding/asn1"
"flag"
"fmt"
"github.com/lib/pq"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
)
type config struct {
// Common configuration parameters shared by all processors.
ConnInfo string
ConnOpen int
ConnIdle int
ConnLife duration
Interval duration
Batch int
Concurrent int
// Processor-specific config.
Chunk int
HTTPTimeout duration
}
type Work struct {
c *config
db *sql.DB
transport http.Transport
http_client http.Client
create_temp_table_statement *sql.Stmt
crl_update_statement *sql.Stmt
}
type WorkItem struct {
work *Work
ca_id int32
distribution_point_url string
crl_size sql.NullInt64
this_update time.Time
next_update time.Time
issuer_cert []byte
error_message sql.NullString
crl_sha256 [sha256.Size]byte
start_time time.Time
already_updated bool
}
func checkRedirectURL(req *http.Request, via []*http.Request) error {
// Fixup incorrectly encoded redirect URLs
req.URL.RawQuery = strings.Replace(req.URL.RawQuery, " ", "%20", -1)
return nil
}
// tomlConfig.DefineCustomFlags() and tomlConfig.PrintCustomFlags()
// Specify command-line flags that are specific to this processor.
func (c *config) DefineCustomFlags() {
flag.DurationVar(&c.HTTPTimeout.Duration, "httptimeout", c.HTTPTimeout.Duration, "HTTP timeout")
}
func (c *config) PrintCustomFlags() string {
return fmt.Sprintf("httptimeout:%s", c.HTTPTimeout.Duration)
}
func (w *Work) Init(c *config) {
w.c = c
w.transport = http.Transport { TLSClientConfig: &tls.Config { InsecureSkipVerify: true } }
w.http_client = http.Client { CheckRedirect: checkRedirectURL, Timeout: c.HTTPTimeout.Duration, Transport: &w.transport }
var err error
w.create_temp_table_statement, err = w.db.Prepare(`
CREATE TEMP TABLE crl_revoked_import_temp (
SERIAL_NUMBER bytea,
REASON_CODE smallint,
REVOCATION_DATE timestamp
) ON COMMIT DROP
`)
checkErr(err)
w.crl_update_statement, err = w.db.Prepare("SELECT crl_update($1,$2,$3,$4,$5,$6,$7,$8)")
checkErr(err)
}
// Work.Begin
// Do any DB stuff that needs to happen before a batch of work.
func (w *Work) Begin(db *sql.DB) {
}
// Work.End
// Do any DB stuff that needs to happen after a batch of work.
func (w *Work) End() {
}
// Work.Exit
// One-time program exit code.
func (w *Work) Exit() {
w.create_temp_table_statement.Close()
w.crl_update_statement.Close()
}
// Work.Prepare()
// Prepare the driving SELECT query.
func (w *Work) SelectQuery(batch_size int) string {
return fmt.Sprintf(`
SELECT crl.CA_ID, crl.DISTRIBUTION_POINT_URL, coalesce(crl.THIS_UPDATE, 'epoch'::timestamp), coalesce(crl.NEXT_UPDATE, 'epoch'::timestamp), coalesce(c2.CERTIFICATE, c1.CERTIFICATE)
FROM crl
LEFT JOIN LATERAL (
SELECT c1.CERTIFICATE, c1.ISSUER_CA_ID
FROM ca_certificate cac1, certificate c1
WHERE crl.CA_ID = cac1.CA_ID
AND cac1.CERTIFICATE_ID = c1.ID
LIMIT 1
) c1 ON TRUE
LEFT JOIN LATERAL (
SELECT c2.CERTIFICATE
FROM ca_certificate cac2, certificate c2
WHERE cac2.CA_ID = c1.ISSUER_CA_ID
AND cac2.CERTIFICATE_ID = c2.ID
AND EXISTS (SELECT 1 FROM x509_extKeyUsages(c1.CERTIFICATE) WHERE x509_extKeyUsages = '1.3.6.1.4.1.11129.2.4.4')
LIMIT 1
) c2 ON TRUE
WHERE crl.IS_ACTIVE = 't'
AND crl.NEXT_CHECK_DUE < now() AT TIME ZONE 'UTC'
ORDER BY crl.IS_ACTIVE, crl.NEXT_CHECK_DUE
LIMIT %d
`, batch_size)
}
// WorkItem.Parse()
// Parse one SELECTed row to configure one work item.
func (wi *WorkItem) Parse(rs *sql.Rows) error {
return rs.Scan(&wi.ca_id, &wi.distribution_point_url, &wi.this_update, &wi.next_update, &wi.issuer_cert)
}
func (wi *WorkItem) checkErr(err error) {
if err != nil {
wi.error_message.String = err.Error()
wi.error_message.Valid = true
panic(err)
}
}
// WorkItem.Perform()
// Do the work for one item.
func (wi *WorkItem) Perform(db *sql.DB, w *Work) {
wi.start_time = time.Now().UTC()
wi.work = w
wi.error_message.String = ""
wi.error_message.Valid = false
wi.crl_size.Valid = false
wi.already_updated = false
// Retrieve the CRL
var err error
var crl *pkix.CertificateList
var body []byte
if strings.HasPrefix(strings.ToLower(wi.distribution_point_url), "ldap") {
// TODO: Support LDAP CRL URLs
wi.error_message.String = "Unsupported URL scheme"
wi.error_message.Valid = true
log.Printf("%s: %s", wi.error_message.String, wi.distribution_point_url)
return
} else {
// Fetch the CRL via HTTP(S)
req, err := http.NewRequest("GET", wi.distribution_point_url, nil)
wi.checkErr(err)
req.Header.Add("User-Agent", "crt.sh")
req.Header.Add("If-Modified-Since", wi.this_update.Format(http.TimeFormat))
resp, err := w.http_client.Do(req)
wi.checkErr(err)
defer resp.Body.Close()
if resp.StatusCode == 304 {
log.Printf("Not Modified (304): %s", wi.distribution_point_url)
return
}
if resp.StatusCode != 200 {
wi.error_message.String = fmt.Sprintf("HTTP %d", resp.StatusCode)
wi.error_message.Valid = true
log.Printf("%s: %s", wi.error_message.String, wi.distribution_point_url)
return
}
// Extract the HTTP response body
body, err = ioutil.ReadAll(resp.Body)
wi.checkErr(err)
}
// Progress report
wi.crl_size.Int64 = int64(len(body))
wi.crl_size.Valid = true
log.Printf("Downloaded (%d bytes): %s", wi.crl_size.Int64, wi.distribution_point_url)
// Calculate SHA-256(CRL)
wi.crl_sha256 = sha256.Sum256(body)
// Parse the CRL
crl, err = x509.ParseCRL(body)
if err != nil {
log.Printf("x509.ParseCRL() => %v", err)
wi.error_message.String = err.Error()
wi.error_message.Valid = true
return
}
// Extract various fields from this CRL
var temp_this_update = wi.this_update
wi.this_update = crl.TBSCertList.ThisUpdate
wi.next_update = crl.TBSCertList.NextUpdate
// Check that this CRL is newer than the last one we processed
if temp_this_update.Sub(wi.this_update) >= 0 {
log.Printf("Not Modified (thisUpdate): %s", wi.distribution_point_url)
return
}
// Parse the supplied issuer certificate
cert, err := x509.ParseCertificate(wi.issuer_cert)
wi.checkErr(err)
// Check this CRL's signature using the supplied issuer certificate
err = cert.CheckCRLSignature(crl)
if err != nil {
log.Printf("cert.CheckCRLSignature() => %v", err)
wi.error_message.String = err.Error()
wi.error_message.Valid = true
return
}
// Show progress report
log.Printf("Verified: %s", wi.distribution_point_url)
// TODO: Check crl.HasExpired(time.Now) ?
// TODO: Set inactive if "latest" CRL is ancient?
// TODO: Deactivate if duplicate of another CDP?
// Begin a new transaction.
tx, err := db.Begin()
wi.checkErr(err)
defer tx.Rollback()
// Prepare some statements for this transaction.
tx_create_temp_table_statement := tx.Stmt(w.create_temp_table_statement)
defer tx_create_temp_table_statement.Close()
tx_crl_update_statement := tx.Stmt(w.crl_update_statement)
defer tx_crl_update_statement.Close()
// Create the temporary "crl_revoked_import_temp" table.
_, err = tx_create_temp_table_statement.Exec()
wi.checkErr(err)
if crl.TBSCertList.RevokedCertificates != nil {
// Prepare the COPY statement.
tx_copy_item_statement, err := tx.Prepare(pq.CopyIn("crl_revoked_import_temp", "serial_number", "reason_code", "revocation_date"))
wi.checkErr(err)
// Loop through the revoked certs, adding each one to the bulk import.
for _, revoked_cert := range crl.TBSCertList.RevokedCertificates {
// Get the CRL Entry Reason Code (if specified)
var reason_code sql.NullInt64
reason_code.Valid = false
for _, ext := range revoked_cert.Extensions {
if ext.Id.Equal([]int{2, 5, 29, 21}) {
if bytes.HasPrefix(ext.Value, []byte{10, 1}) { // ENUMERATED, length=1
reason_code.Int64 = int64(ext.Value[2])
reason_code.Valid = true
}
}
}
// Get the bytes of the encoded serial number
serial_bytes, err := asn1.Marshal(revoked_cert.SerialNumber)
wi.checkErr(err)
if serial_bytes[1] > 0x7F {
log.Printf("Serial number has multiple length octets")
} else {
// The [2:] strips the ASN.1 tag and length octets.
_, err = tx_copy_item_statement.Exec(serial_bytes[2:], reason_code, revoked_cert.RevocationTime)
wi.checkErr(err)
}
}
// Execute the COPY statement, to perform the bulk import.
_, err = tx_copy_item_statement.Exec()
wi.checkErr(err)
}
// Process the bulk-imported data.
_, err = tx_crl_update_statement.Exec(wi.ca_id, wi.distribution_point_url, wi.this_update, wi.next_update, wi.start_time, wi.error_message, wi.crl_sha256[:], wi.crl_size)
wi.checkErr(err)
// Commit the transaction. This will drop the temporary table.
err = tx.Commit()
wi.checkErr(err)
log.Printf("Processed (%d revocations): %s (%v)", len(crl.TBSCertList.RevokedCertificates), wi.distribution_point_url, time.Now().UTC().Sub(wi.start_time))
wi.already_updated = true
}
// Work.UpdateStatement()
// Prepare the UPDATE statement to be run when a CRL has not changed.
func (w *Work) UpdateStatement() string {
return `
UPDATE crl
SET LAST_CHECKED=statement_timestamp() AT TIME ZONE 'UTC',
NEXT_CHECK_DUE=statement_timestamp() AT TIME ZONE 'UTC' + interval '4 hours',
ERROR_MESSAGE=$1::text
WHERE CA_ID=$2
AND DISTRIBUTION_POINT_URL=$3
`
}
// WorkItem.Update()
// Update the DB with the results of the work for this item.
func (wi *WorkItem) Update(update_statement *sql.Stmt) (sql.Result, error) {
if !wi.already_updated {
return update_statement.Exec(wi.error_message, wi.ca_id, wi.distribution_point_url)
} else {
return nil, nil
}
}