/
main.go
304 lines (259 loc) · 8.58 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
package notmain
import (
"bufio"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"strings"
"time"
"github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/db"
"github.com/letsencrypt/boulder/features"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/sa"
)
type idExporter struct {
log blog.Logger
dbMap *db.WrappedMap
clk clock.Clock
grace time.Duration
}
// resultEntry is a JSON marshalable exporter result entry.
type resultEntry struct {
// ID is exported to support marshaling to JSON.
ID int64 `json:"id"`
// Hostname is exported to support marshaling to JSON. Not all queries
// will fill this field, so it's JSON field tag marks at as
// omittable.
Hostname string `json:"hostname,omitempty"`
}
// reverseHostname converts (reversed) names sourced from the
// registrations table to standard hostnames.
func (r *resultEntry) reverseHostname() {
r.Hostname = sa.ReverseName(r.Hostname)
}
// idExporterResults is passed as a selectable 'holder' for the results
// of id-exporter database queries
type idExporterResults []*resultEntry
// marshalToJSON returns JSON as bytes for all elements of the inner `id`
// slice.
func (i *idExporterResults) marshalToJSON() ([]byte, error) {
data, err := json.Marshal(i)
if err != nil {
return nil, err
}
data = append(data, '\n')
return data, nil
}
// writeToFile writes the contents of the inner `ids` slice, as JSON, to
// a file
func (i *idExporterResults) writeToFile(outfile string) error {
data, err := i.marshalToJSON()
if err != nil {
return err
}
return os.WriteFile(outfile, data, 0644)
}
// findIDs gathers all registration IDs with unexpired certificates.
func (c idExporter) findIDs(ctx context.Context) (idExporterResults, error) {
var holder idExporterResults
_, err := c.dbMap.Select(
ctx,
&holder,
`SELECT DISTINCT r.id
FROM registrations AS r
INNER JOIN certificates AS c on c.registrationID = r.id
WHERE r.contact NOT IN ('[]', 'null')
AND c.expires >= :expireCutoff;`,
map[string]interface{}{
"expireCutoff": c.clk.Now().Add(-c.grace),
})
if err != nil {
c.log.AuditErrf("Error finding IDs: %s", err)
return nil, err
}
return holder, nil
}
// findIDsWithExampleHostnames gathers all registration IDs with
// unexpired certificates and a corresponding example hostname.
func (c idExporter) findIDsWithExampleHostnames(ctx context.Context) (idExporterResults, error) {
var holder idExporterResults
_, err := c.dbMap.Select(
ctx,
&holder,
`SELECT SQL_BIG_RESULT
cert.registrationID AS id,
name.reversedName AS hostname
FROM certificates AS cert
INNER JOIN issuedNames AS name ON name.serial = cert.serial
WHERE cert.expires >= :expireCutoff
GROUP BY cert.registrationID;`,
map[string]interface{}{
"expireCutoff": c.clk.Now().Add(-c.grace),
})
if err != nil {
c.log.AuditErrf("Error finding IDs and example hostnames: %s", err)
return nil, err
}
for _, result := range holder {
result.reverseHostname()
}
return holder, nil
}
// findIDsForHostnames gathers all registration IDs with unexpired
// certificates for each `hostnames` entry.
func (c idExporter) findIDsForHostnames(ctx context.Context, hostnames []string) (idExporterResults, error) {
var holder idExporterResults
for _, hostname := range hostnames {
// Pass the same list in each time, borp will happily just append to the slice
// instead of overwriting it each time
// https://github.com/letsencrypt/borp/blob/c87bd6443d59746a33aca77db34a60cfc344adb2/select.go#L349-L353
_, err := c.dbMap.Select(
ctx,
&holder,
`SELECT DISTINCT c.registrationID AS id
FROM certificates AS c
INNER JOIN issuedNames AS n ON c.serial = n.serial
WHERE c.expires >= :expireCutoff
AND n.reversedName = :reversedName;`,
map[string]interface{}{
"expireCutoff": c.clk.Now().Add(-c.grace),
"reversedName": sa.ReverseName(hostname),
},
)
if err != nil {
if db.IsNoRows(err) {
continue
}
return nil, err
}
}
return holder, nil
}
const usageIntro = `
Introduction:
The ID exporter exists to retrieve the IDs of all registered
users with currently unexpired certificates. This list of registration IDs can
then be given as input to the notification mailer to send bulk notifications.
The -grace parameter can be used to allow registrations with certificates that
have already expired to be included in the export. The argument is a Go duration
obeying the usual suffix rules (e.g. 24h).
Registration IDs are favoured over email addresses as the intermediate format in
order to ensure the most up to date contact information is used at the time of
notification. The notification mailer will resolve the ID to email(s) when the
mailing is underway, ensuring we use the correct address if a user has updated
their contact information between the time of export and the time of
notification.
By default, the ID exporter's output will be JSON of the form:
[
{ "id": 1 },
...
{ "id": n }
]
Operations that return a hostname will be JSON of the form:
[
{ "id": 1, "hostname": "example-1.com" },
...
{ "id": n, "hostname": "example-n.com" }
]
Examples:
Export all registration IDs with unexpired certificates to "regs.json":
id-exporter -config test/config/id-exporter.json -outfile regs.json
Export all registration IDs with certificates that are unexpired or expired
within the last two days to "regs.json":
id-exporter -config test/config/id-exporter.json -grace 48h -outfile
"regs.json"
Required arguments:
- config
- outfile`
// unmarshalHostnames unmarshals a hostnames file and ensures that the file
// contained at least one entry.
func unmarshalHostnames(filePath string) ([]string, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
var hostnames []string
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, " ") {
return nil, fmt.Errorf(
"line: %q contains more than one entry, entries must be separated by newlines", line)
}
hostnames = append(hostnames, line)
}
if len(hostnames) == 0 {
return nil, errors.New("provided file contains 0 hostnames")
}
return hostnames, nil
}
type Config struct {
ContactExporter struct {
DB cmd.DBConfig
cmd.PasswordConfig
Features features.Config
}
}
func main() {
outFile := flag.String("outfile", "", "File to output results JSON to.")
grace := flag.Duration("grace", 2*24*time.Hour, "Include results with certificates that expired in < grace ago.")
hostnamesFile := flag.String(
"hostnames", "", "Only include results with unexpired certificates that contain hostnames\nlisted (newline separated) in this file.")
withExampleHostnames := flag.Bool(
"with-example-hostnames", false, "Include an example hostname for each registration ID with an unexpired certificate.")
configFile := flag.String("config", "", "File containing a JSON config.")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "%s\n\n", usageIntro)
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
flag.PrintDefaults()
}
// Parse flags and check required.
flag.Parse()
if *outFile == "" || *configFile == "" {
flag.Usage()
os.Exit(1)
}
log := cmd.NewLogger(cmd.SyslogConfig{StdoutLevel: 7})
log.Info(cmd.VersionString())
// Load configuration file.
configData, err := os.ReadFile(*configFile)
cmd.FailOnError(err, fmt.Sprintf("Reading %q", *configFile))
// Unmarshal JSON config file.
var cfg Config
err = json.Unmarshal(configData, &cfg)
cmd.FailOnError(err, "Unmarshaling config")
features.Set(cfg.ContactExporter.Features)
dbMap, err := sa.InitWrappedDb(cfg.ContactExporter.DB, nil, log)
cmd.FailOnError(err, "While initializing dbMap")
exporter := idExporter{
log: log,
dbMap: dbMap,
clk: cmd.Clock(),
grace: *grace,
}
var results idExporterResults
if *hostnamesFile != "" {
hostnames, err := unmarshalHostnames(*hostnamesFile)
cmd.FailOnError(err, "Problem unmarshalling hostnames")
results, err = exporter.findIDsForHostnames(context.TODO(), hostnames)
cmd.FailOnError(err, "Could not find IDs for hostnames")
} else if *withExampleHostnames {
results, err = exporter.findIDsWithExampleHostnames(context.TODO())
cmd.FailOnError(err, "Could not find IDs with hostnames")
} else {
results, err = exporter.findIDs(context.TODO())
cmd.FailOnError(err, "Could not find IDs")
}
err = results.writeToFile(*outFile)
cmd.FailOnError(err, fmt.Sprintf("Could not write result to outfile %q", *outFile))
}
func init() {
cmd.RegisterCommand("id-exporter", main, &cmd.ConfigValidator{Config: &Config{}})
}