forked from caddyserver/caddy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
maintain.go
364 lines (329 loc) · 12.9 KB
/
maintain.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
// Copyright 2015 Light Code Labs, LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package caddytls
import (
"io/ioutil"
"log"
"os"
"path/filepath"
"time"
"github.com/mholt/caddy"
"golang.org/x/crypto/ocsp"
)
func init() {
// maintain assets while this package is imported, which is
// always. we don't ever stop it, since we need it running.
go maintainAssets(make(chan struct{}))
}
const (
// RenewInterval is how often to check certificates for renewal.
RenewInterval = 12 * time.Hour
// RenewDurationBefore is how long before expiration to renew certificates.
RenewDurationBefore = (24 * time.Hour) * 30
// RenewDurationBeforeAtStartup is how long before expiration to require
// a renewed certificate when the process is first starting up (see #1680).
// A wider window between RenewDurationBefore and this value will allow
// Caddy to start under duress but hopefully this duration will give it
// enough time for the blockage to be relieved.
RenewDurationBeforeAtStartup = (24 * time.Hour) * 7
// OCSPInterval is how often to check if OCSP stapling needs updating.
OCSPInterval = 1 * time.Hour
)
// maintainAssets is a permanently-blocking function
// that loops indefinitely and, on a regular schedule, checks
// certificates for expiration and initiates a renewal of certs
// that are expiring soon. It also updates OCSP stapling and
// performs other maintenance of assets. It should only be
// called once per process.
//
// You must pass in the channel which you'll close when
// maintenance should stop, to allow this goroutine to clean up
// after itself and unblock. (Not that you HAVE to stop it...)
func maintainAssets(stopChan chan struct{}) {
renewalTicker := time.NewTicker(RenewInterval)
ocspTicker := time.NewTicker(OCSPInterval)
for {
select {
case <-renewalTicker.C:
log.Println("[INFO] Scanning for expiring certificates")
RenewManagedCertificates(false)
log.Println("[INFO] Done checking certificates")
case <-ocspTicker.C:
log.Println("[INFO] Scanning for stale OCSP staples")
UpdateOCSPStaples()
DeleteOldStapleFiles()
log.Println("[INFO] Done checking OCSP staples")
case <-stopChan:
renewalTicker.Stop()
ocspTicker.Stop()
log.Println("[INFO] Stopped background maintenance routine")
return
}
}
}
// RenewManagedCertificates renews managed certificates,
// including ones loaded on-demand.
func RenewManagedCertificates(allowPrompts bool) (err error) {
for _, inst := range caddy.Instances() {
inst.StorageMu.RLock()
certCache, ok := inst.Storage[CertCacheInstStorageKey].(*certificateCache)
inst.StorageMu.RUnlock()
if !ok || certCache == nil {
continue
}
// we use the queues for a very important reason: to do any and all
// operations that could require an exclusive write lock outside
// of the read lock! otherwise we get a deadlock, yikes. in other
// words, our first iteration through the certificate cache does NOT
// perform any operations--only queues them--so that more fine-grained
// write locks may be obtained during the actual operations.
var renewQueue, reloadQueue, deleteQueue []Certificate
certCache.RLock()
for certKey, cert := range certCache.cache {
if len(cert.configs) == 0 {
// this is bad if this happens, probably a programmer error (oops)
log.Printf("[ERROR] No associated TLS config for certificate with names %v; unable to manage", cert.Names)
continue
}
if !cert.configs[0].Managed || cert.configs[0].SelfSigned {
continue
}
// the list of names on this cert should never be empty... programmer error?
if cert.Names == nil || len(cert.Names) == 0 {
log.Printf("[WARNING] Certificate keyed by '%s' has no names: %v - removing from cache", certKey, cert.Names)
deleteQueue = append(deleteQueue, cert)
continue
}
// if time is up or expires soon, we need to try to renew it
timeLeft := cert.NotAfter.Sub(time.Now().UTC())
if timeLeft < RenewDurationBefore {
// see if the certificate in storage has already been renewed, possibly by another
// instance of Caddy that didn't coordinate with this one; if so, just load it (this
// might happen if another instance already renewed it - kinda sloppy but checking disk
// first is a simple way to possibly drastically reduce rate limit problems)
storedCertExpiring, err := managedCertInStorageExpiresSoon(cert)
if err != nil {
// hmm, weird, but not a big deal, maybe it was deleted or something
log.Printf("[NOTICE] Error while checking if certificate for %v in storage is also expiring soon: %v",
cert.Names, err)
} else if !storedCertExpiring {
// if the certificate is NOT expiring soon and there was no error, then we
// are good to just reload the certificate from storage instead of repeating
// a likely-unnecessary renewal procedure
reloadQueue = append(reloadQueue, cert)
continue
}
// the certificate in storage has not been renewed yet, so we will do it
// NOTE 1: This is not correct 100% of the time, if multiple Caddy instances
// happen to run their maintenance checks at approximately the same times;
// both might start renewal at about the same time and do two renewals and one
// will overwrite the other. Hence TLS storage plugins. This is sort of a TODO.
// NOTE 2: It is super-important to note that the TLS-SNI challenge requires
// a write lock on the cache in order to complete its challenge, so it is extra
// vital that this renew operation does not happen inside our read lock!
renewQueue = append(renewQueue, cert)
}
}
certCache.RUnlock()
// Reload certificates that merely need to be updated in memory
for _, oldCert := range reloadQueue {
timeLeft := oldCert.NotAfter.Sub(time.Now().UTC())
log.Printf("[INFO] Certificate for %v expires in %v, but is already renewed in storage; reloading stored certificate",
oldCert.Names, timeLeft)
err = certCache.reloadManagedCertificate(oldCert)
if err != nil {
if allowPrompts {
return err // operator is present, so report error immediately
}
log.Printf("[ERROR] Loading renewed certificate: %v", err)
}
}
// Renewal queue
for _, oldCert := range renewQueue {
timeLeft := oldCert.NotAfter.Sub(time.Now().UTC())
log.Printf("[INFO] Certificate for %v expires in %v; attempting renewal", oldCert.Names, timeLeft)
// Get the name which we should use to renew this certificate;
// we only support managing certificates with one name per cert,
// so this should be easy. We can't rely on cert.Config.Hostname
// because it may be a wildcard value from the Caddyfile (e.g.
// *.something.com) which, as of Jan. 2017, is not supported by ACME.
// TODO: ^ ^ ^ (wildcards)
renewName := oldCert.Names[0]
// perform renewal
err := oldCert.configs[0].RenewCert(renewName, allowPrompts)
if err != nil {
if allowPrompts {
// Certificate renewal failed and the operator is present. See a discussion
// about this in issue 642. For a while, we only stopped if the certificate
// was expired, but in reality, there is no difference between reporting
// it now versus later, except that there's somebody present to deal with
// it right now. Follow-up: See issue 1680. Only fail in this case if the
// certificate is dangerously close to expiration.
timeLeft := oldCert.NotAfter.Sub(time.Now().UTC())
if timeLeft < RenewDurationBeforeAtStartup {
return err
}
}
log.Printf("[ERROR] %v", err)
if oldCert.configs[0].OnDemand {
// loaded dynamically, remove dynamically
deleteQueue = append(deleteQueue, oldCert)
}
continue
}
// successful renewal, so update in-memory cache by loading
// renewed certificate so it will be used with handshakes
err = certCache.reloadManagedCertificate(oldCert)
if err != nil {
if allowPrompts {
return err // operator is present, so report error immediately
}
log.Printf("[ERROR] %v", err)
}
}
// Deletion queue
for _, cert := range deleteQueue {
certCache.Lock()
// remove any pointers to this certificate from Configs
for _, cfg := range cert.configs {
for name, certKey := range cfg.Certificates {
if certKey == cert.Hash {
delete(cfg.Certificates, name)
}
}
}
// then delete the certificate from the cache
delete(certCache.cache, cert.Hash)
certCache.Unlock()
}
}
return nil
}
// UpdateOCSPStaples updates the OCSP stapling in all
// eligible, cached certificates.
//
// OCSP maintenance strives to abide the relevant points on
// Ryan Sleevi's recommendations for good OCSP support:
// https://gist.github.com/sleevi/5efe9ef98961ecfb4da8
func UpdateOCSPStaples() {
for _, inst := range caddy.Instances() {
inst.StorageMu.RLock()
certCache, ok := inst.Storage[CertCacheInstStorageKey].(*certificateCache)
inst.StorageMu.RUnlock()
if !ok || certCache == nil {
continue
}
// Create a temporary place to store updates
// until we release the potentially long-lived
// read lock and use a short-lived write lock
// on the certificate cache.
type ocspUpdate struct {
rawBytes []byte
parsed *ocsp.Response
}
updated := make(map[string]ocspUpdate)
certCache.RLock()
for certHash, cert := range certCache.cache {
// no point in updating OCSP for expired certificates
if time.Now().After(cert.NotAfter) {
continue
}
var lastNextUpdate time.Time
if cert.OCSP != nil {
lastNextUpdate = cert.OCSP.NextUpdate
if freshOCSP(cert.OCSP) {
continue // no need to update staple if ours is still fresh
}
}
err := stapleOCSP(&cert, nil)
if err != nil {
if cert.OCSP != nil {
// if there was no staple before, that's fine; otherwise we should log the error
log.Printf("[ERROR] Checking OCSP: %v", err)
}
continue
}
// By this point, we've obtained the latest OCSP response.
// If there was no staple before, or if the response is updated, make
// sure we apply the update to all names on the certificate.
if cert.OCSP != nil && (lastNextUpdate.IsZero() || lastNextUpdate != cert.OCSP.NextUpdate) {
log.Printf("[INFO] Advancing OCSP staple for %v from %s to %s",
cert.Names, lastNextUpdate, cert.OCSP.NextUpdate)
updated[certHash] = ocspUpdate{rawBytes: cert.Certificate.OCSPStaple, parsed: cert.OCSP}
}
}
certCache.RUnlock()
// These write locks should be brief since we have all the info we need now.
for certKey, update := range updated {
certCache.Lock()
cert := certCache.cache[certKey]
cert.OCSP = update.parsed
cert.Certificate.OCSPStaple = update.rawBytes
certCache.cache[certKey] = cert
certCache.Unlock()
}
}
}
// DeleteOldStapleFiles deletes cached OCSP staples that have expired.
// TODO: Should we do this for certificates too?
func DeleteOldStapleFiles() {
// TODO: Upgrade caddytls.Storage to support OCSP operations too
files, err := ioutil.ReadDir(ocspFolder)
if err != nil {
// maybe just hasn't been created yet; no big deal
return
}
for _, file := range files {
if file.IsDir() {
// weird, what's a folder doing inside the OCSP cache?
continue
}
stapleFile := filepath.Join(ocspFolder, file.Name())
ocspBytes, err := ioutil.ReadFile(stapleFile)
if err != nil {
continue
}
resp, err := ocsp.ParseResponse(ocspBytes, nil)
if err != nil {
// contents are invalid; delete it
err = os.Remove(stapleFile)
if err != nil {
log.Printf("[ERROR] Purging corrupt staple file %s: %v", stapleFile, err)
}
}
if time.Now().After(resp.NextUpdate) {
// response has expired; delete it
err = os.Remove(stapleFile)
if err != nil {
log.Printf("[ERROR] Purging expired staple file %s: %v", stapleFile, err)
}
}
}
}
// freshOCSP returns true if resp is still fresh,
// meaning that it is not expedient to get an
// updated response from the OCSP server.
func freshOCSP(resp *ocsp.Response) bool {
nextUpdate := resp.NextUpdate
// If there is an OCSP responder certificate, and it expires before the
// OCSP response, use its expiration date as the end of the OCSP
// response's validity period.
if resp.Certificate != nil && resp.Certificate.NotAfter.Before(nextUpdate) {
nextUpdate = resp.Certificate.NotAfter
}
// start checking OCSP staple about halfway through validity period for good measure
refreshTime := resp.ThisUpdate.Add(nextUpdate.Sub(resp.ThisUpdate) / 2)
return time.Now().Before(refreshTime)
}
var ocspFolder = filepath.Join(caddy.AssetsPath(), "ocsp")