-
-
Notifications
You must be signed in to change notification settings - Fork 586
/
checked_redis_source.go
159 lines (137 loc) · 5.95 KB
/
checked_redis_source.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
package redis
import (
"context"
"errors"
"reflect"
"sync"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/crypto/ocsp"
"github.com/letsencrypt/boulder/core"
"github.com/letsencrypt/boulder/db"
berrors "github.com/letsencrypt/boulder/errors"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/ocsp/responder"
"github.com/letsencrypt/boulder/sa"
sapb "github.com/letsencrypt/boulder/sa/proto"
)
// dbSelector is a limited subset of the db.WrappedMap interface to allow for
// easier mocking of mysql operations in tests.
type dbSelector interface {
SelectOne(ctx context.Context, holder interface{}, query string, args ...interface{}) error
}
// rocspSourceInterface expands on responder.Source by adding a private signAndSave method.
// This allows checkedRedisSource to trigger a live signing if the DB disagrees with Redis.
type rocspSourceInterface interface {
Response(ctx context.Context, req *ocsp.Request) (*responder.Response, error)
signAndSave(ctx context.Context, req *ocsp.Request, cause signAndSaveCause) (*responder.Response, error)
}
// checkedRedisSource implements the Source interface. It relies on two
// underlying datastores to provide its OCSP responses: a rocspSourceInterface
// (a Source that can also signAndSave new responses) to provide the responses
// themselves, and the database to double-check that those responses match the
// authoritative revocation status stored in the db.
// TODO(#6285): Inline the rocspSourceInterface into this type.
// TODO(#6295): Remove the dbMap after all deployments use the SA instead.
type checkedRedisSource struct {
base rocspSourceInterface
dbMap dbSelector
sac sapb.StorageAuthorityReadOnlyClient
counter *prometheus.CounterVec
log blog.Logger
}
// NewCheckedRedisSource builds a source that queries both the DB and Redis, and confirms
// the value in Redis matches the DB.
func NewCheckedRedisSource(base *redisSource, dbMap dbSelector, sac sapb.StorageAuthorityReadOnlyClient, stats prometheus.Registerer, log blog.Logger) (*checkedRedisSource, error) {
if base == nil {
return nil, errors.New("base was nil")
}
// We have to use reflect here because these arguments are interfaces, and
// thus checking for nil the normal way doesn't work reliably, because they
// may be non-nil interfaces whose inner value is still nil, i.e. "boxed nil".
// But using reflect here is okay, because we only expect this constructor to
// be called once per process.
if (reflect.TypeOf(sac) == nil || reflect.ValueOf(sac).IsNil()) &&
(reflect.TypeOf(dbMap) == nil || reflect.ValueOf(dbMap).IsNil()) {
return nil, errors.New("either SA gRPC or direct DB connection must be provided")
}
return newCheckedRedisSource(base, dbMap, sac, stats, log), nil
}
// newCheckedRedisSource is an internal-only constructor that takes a private interface as a parameter.
// We call this from tests and from NewCheckedRedisSource.
func newCheckedRedisSource(base rocspSourceInterface, dbMap dbSelector, sac sapb.StorageAuthorityReadOnlyClient, stats prometheus.Registerer, log blog.Logger) *checkedRedisSource {
counter := prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "checked_rocsp_responses",
Help: "Count of OCSP requests/responses from checkedRedisSource, by result",
}, []string{"result"})
stats.MustRegister(counter)
return &checkedRedisSource{
base: base,
dbMap: dbMap,
sac: sac,
counter: counter,
log: log,
}
}
// Response implements the responder.Source interface. It looks up the requested OCSP
// response in the redis cluster and looks up the corresponding status in the DB. If
// the status disagrees with what redis says, it signs a fresh response and serves it.
func (src *checkedRedisSource) Response(ctx context.Context, req *ocsp.Request) (*responder.Response, error) {
serialString := core.SerialToString(req.SerialNumber)
var wg sync.WaitGroup
wg.Add(2)
var dbStatus *sapb.RevocationStatus
var redisResult *responder.Response
var redisErr, dbErr error
go func() {
defer wg.Done()
if src.sac != nil {
dbStatus, dbErr = src.sac.GetRevocationStatus(ctx, &sapb.Serial{Serial: serialString})
} else {
dbStatus, dbErr = sa.SelectRevocationStatus(ctx, src.dbMap, serialString)
}
}()
go func() {
defer wg.Done()
redisResult, redisErr = src.base.Response(ctx, req)
}()
wg.Wait()
if dbErr != nil {
// If the DB says "not found", the certificate either doesn't exist or has
// expired and been removed from the DB. We don't need to check the Redis error.
if db.IsNoRows(dbErr) || errors.Is(dbErr, berrors.NotFound) {
src.counter.WithLabelValues("not_found").Inc()
return nil, responder.ErrNotFound
}
src.counter.WithLabelValues("db_error").Inc()
return nil, dbErr
}
if redisErr != nil {
src.counter.WithLabelValues("redis_error").Inc()
return nil, redisErr
}
// If the DB status matches the status returned from the Redis pipeline, all is good.
if agree(dbStatus, redisResult.Response) {
src.counter.WithLabelValues("success").Inc()
return redisResult, nil
}
// Otherwise, the DB is authoritative. Trigger a fresh signing.
freshResult, err := src.base.signAndSave(ctx, req, causeMismatch)
if err != nil {
src.counter.WithLabelValues("revocation_re_sign_error").Inc()
return nil, err
}
if agree(dbStatus, freshResult.Response) {
src.counter.WithLabelValues("revocation_re_sign_success").Inc()
return freshResult, nil
}
// This could happen for instance with replication lag, or if the
// RA was talking to a different DB.
src.counter.WithLabelValues("revocation_re_sign_mismatch").Inc()
return nil, errors.New("freshly signed status did not match DB")
}
// agree returns true if the contents of the redisResult ocsp.Response agree with what's in the DB.
func agree(dbStatus *sapb.RevocationStatus, redisResult *ocsp.Response) bool {
return dbStatus.Status == int64(redisResult.Status) &&
dbStatus.RevokedReason == int64(redisResult.RevocationReason) &&
dbStatus.RevokedDate.AsTime().Equal(redisResult.RevokedAt)
}