/
main.go
501 lines (441 loc) · 14.7 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
package main
import (
"bytes"
"encoding/csv"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"net/mail"
"os"
"sort"
"strconv"
"strings"
"text/template"
"time"
"github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/db"
"github.com/letsencrypt/boulder/features"
blog "github.com/letsencrypt/boulder/log"
bmail "github.com/letsencrypt/boulder/mail"
"github.com/letsencrypt/boulder/metrics"
"github.com/letsencrypt/boulder/policy"
"github.com/letsencrypt/boulder/sa"
)
type mailer struct {
clk clock.Clock
log blog.Logger
dbMap dbSelector
mailer bmail.Mailer
subject string
emailTemplate *template.Template
destinations []recipient
targetRange interval
sleepInterval time.Duration
}
// interval defines a range of email addresses to send to, alphabetically.
// The "start" field is inclusive and the "end" field is exclusive.
// To include everything, set "end" to "\xFF".
type interval struct {
start string
end string
}
type contactJSON struct {
ID int
Contact []byte
}
func (i *interval) ok() error {
if i.start > i.end {
return fmt.Errorf(
"interval start value (%s) is greater than end value (%s)",
i.start, i.end)
}
return nil
}
func (i *interval) includes(s string) bool {
return s >= i.start && s < i.end
}
func (m *mailer) ok() error {
// Make sure the checkpoint range is OK
if checkpointErr := m.targetRange.ok(); checkpointErr != nil {
return checkpointErr
}
// Do not allow a negative sleep interval
if m.sleepInterval < 0 {
return fmt.Errorf(
"sleep interval (%d) is < 0", m.sleepInterval)
}
return nil
}
func (m *mailer) printStatus(to string, cur, total int, start time.Time) {
// Should never happen
if total <= 0 || cur < 1 || cur > total {
m.log.AuditErrf("invalid cur (%d) or total (%d)", cur, total)
}
completion := (float32(cur) / float32(total)) * 100
now := m.clk.Now()
elapsed := now.Sub(start)
m.log.Infof("Sending to %q. Message %d of %d [%.2f%%]. Elapsed: %s",
to, cur, total, completion, elapsed)
}
func sortAddresses(input emailToRecipientMap) []string {
var addresses []string
for k := range input {
addresses = append(addresses, k)
}
sort.Strings(addresses)
return addresses
}
func (m *mailer) run() error {
if err := m.ok(); err != nil {
return err
}
m.log.Infof("Resolving %d destination addresses", len(m.destinations))
addressesToRecipients, err := m.resolveEmailAddresses()
if err != nil {
return err
}
if len(addressesToRecipients) == 0 {
return fmt.Errorf("zero recipients after looking up addresses?")
}
m.log.Infof("Resolved destination addresses. %d accounts became %d addresses.",
len(m.destinations), len(addressesToRecipients))
var biggest int
var biggestAddress string
for k, v := range addressesToRecipients {
if len(v) > biggest {
biggest = len(v)
biggestAddress = k
}
}
m.log.Infof("Most frequent address %q had %d associated lines", biggestAddress, biggest)
err = m.mailer.Connect()
if err != nil {
return err
}
defer func() {
_ = m.mailer.Close()
}()
startTime := m.clk.Now()
sortedAddresses := sortAddresses(addressesToRecipients)
numAddresses := len(addressesToRecipients)
var sent int
for i, address := range sortedAddresses {
if !m.targetRange.includes(address) {
m.log.Debugf("skipping %q: out of target range")
continue
}
if err := policy.ValidEmail(address); err != nil {
m.log.Infof("skipping %q: %s", address, err)
continue
}
recipients := addressesToRecipients[address]
m.printStatus(address, i+1, numAddresses, startTime)
var mailBody bytes.Buffer
err = m.emailTemplate.Execute(&mailBody, recipients)
if err != nil {
return err
}
if mailBody.Len() == 0 {
return fmt.Errorf("email body was empty after interpolation.")
}
err := m.mailer.SendMail([]string{address}, m.subject, mailBody.String())
if err != nil {
var recoverableSMTPErr bmail.RecoverableSMTPError
if errors.As(err, &recoverableSMTPErr) {
m.log.Errf("address %q was rejected by server: %s", address, err)
continue
}
return fmt.Errorf("sending mail %d of %d to %q: %s",
i, len(sortedAddresses), address, err)
}
sent++
m.clk.Sleep(m.sleepInterval)
}
if sent == 0 {
return fmt.Errorf("sent zero messages. Check recipients and configured interval")
}
return nil
}
// resolveEmailAddresses looks up the id of each recipient to find that
// account's email addresses, then adds that recipient to a map from address to
// recipient struct.
func (m *mailer) resolveEmailAddresses() (emailToRecipientMap, error) {
result := make(emailToRecipientMap, len(m.destinations))
for _, r := range m.destinations {
// Get the email address for the reg ID
emails, err := emailsForReg(r.id, m.dbMap)
if err != nil {
return nil, err
}
for _, email := range emails {
parsedEmail, err := mail.ParseAddress(email)
if err != nil {
m.log.Errf("unparsable email for reg ID %d : %q", r.id, email)
continue
}
addr := parsedEmail.Address
result[addr] = append(result[addr], r)
}
}
return result, nil
}
// Since the only thing we use from gorp is the SelectOne method on the
// gorp.DbMap object, we just define an interface with that method
// instead of importing all of gorp. This facilitates mock implementations for
// unit tests
type dbSelector interface {
SelectOne(holder interface{}, query string, args ...interface{}) error
}
// Finds the email addresses associated with a reg ID
func emailsForReg(id int, dbMap dbSelector) ([]string, error) {
var contact contactJSON
err := dbMap.SelectOne(&contact,
`SELECT id, contact
FROM registrations
WHERE contact != 'null' AND id = :id;`,
map[string]interface{}{
"id": id,
})
if err != nil {
if db.IsNoRows(err) {
return []string{}, nil
}
return nil, err
}
var contactFields []string
var addresses []string
err = json.Unmarshal(contact.Contact, &contactFields)
if err != nil {
return nil, err
}
for _, entry := range contactFields {
if strings.HasPrefix(entry, "mailto:") {
addresses = append(addresses, strings.TrimPrefix(entry, "mailto:"))
}
}
return addresses, nil
}
// recipient represents one line in the input CSV, containing an account and
// (optionally) some extra fields related to that account.
type recipient struct {
id int
Extra map[string]string
}
// emailToRecipientMap maps from an email address to a list of recipients with
// that email address.
type emailToRecipientMap map[string][]recipient
// readRecipientsList reads a CSV filename and parses that file into a list of
// recipient structs. It puts any columns after the first into a per-recipient
// map from column name -> value.
func readRecipientsList(filename string) ([]recipient, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
reader := csv.NewReader(f)
record, err := reader.Read()
if err != nil {
return nil, err
}
if len(record) == 0 {
return nil, fmt.Errorf("no entries in CSV")
}
if record[0] != "id" {
return nil, fmt.Errorf("first field of CSV input must be an ID.")
}
var columnNames []string
for _, v := range record[1:] {
columnNames = append(columnNames, strings.TrimSpace(v))
}
results := []recipient{}
for {
record, err := reader.Read()
if err == io.EOF {
if len(results) == 0 {
return nil, fmt.Errorf("no entries after the header in CSV")
}
return results, nil
}
if err != nil {
return nil, err
}
if len(record) == 0 {
return nil, fmt.Errorf("empty line in CSV")
}
if len(record) != len(columnNames)+1 {
return nil, fmt.Errorf("Number of columns in CSV line didn't match header columns."+
" Got %d, expected %d. Line: %v", len(record), len(columnNames)+1, record)
}
id, err := strconv.Atoi(record[0])
if err != nil {
return nil, err
}
recip := recipient{
id: id,
Extra: make(map[string]string),
}
for i, v := range record[1:] {
recip.Extra[columnNames[i]] = v
}
results = append(results, recip)
}
}
const usageIntro = `
Introduction:
The notification mailer exists to send a message to the contact associated
with a list of registration IDs. The attributes of the message (from address,
subject, and message content) are provided by the command line arguments. The
message content is provided as a path to a template file via the -body argument.
Provide a list of recipient user ids in a CSV file passed with the -recipientList
flag. The CSV file must have "id" as the first column and may have additional
fields to be interpolated into the email template:
id, lastIssuance
1234, "from example.com 2018-12-01"
5678, "from example.net 2018-12-13"
The additional fields will be interpolated with Golang templating, e.g.:
Your last issuance on each account was:
{{ range . }} {{ .Extra.lastIssuance }}
{{ end }}
To help the operator gain confidence in the mailing run before committing fully
three safety features are supported: dry runs, intervals and a sleep between emails.
The -dryRun=true flag will use a mock mailer that prints message content to
stdout instead of performing an SMTP transaction with a real mailserver. This
can be used when the initial parameters are being tweaked to ensure no real
emails are sent. Using -dryRun=false will send real email.
Intervals supported via the -start and -end arguments. Only email addresses that
are alphabetically between the -start and -end strings will be sent. This can be used
to break up sending into batches, or more likely to resume sending if a batch is killed,
without resending messages that have already been sent. The -start flag is inclusive and
the -end flag is exclusive.
Notify-mailer de-duplicates email addresses and groups together the resulting recipient
structs, so a person who has multiple accounts using the same address will only receive
one email.
During mailing the -sleep argument is used to space out individual messages.
This can be used to ensure that the mailing happens at a steady pace with ample
opportunity for the operator to terminate early in the event of error. The
-sleep flag honours durations with a unit suffix (e.g. 1m for 1 minute, 10s for
10 seconds, etc). Using -sleep=0 will disable the sleep and send at full speed.
Examples:
Send an email with subject "Hello!" from the email "hello@goodbye.com" with
the contents read from "test_msg_body.txt" to every email associated with the
registration IDs listed in "test_reg_recipients.json", sleeping 10 seconds
between each message:
notify-mailer -config test/config/notify-mailer.json -body
cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com
-recipientList cmd/notify-mailer/testdata/test_msg_recipients.csv -subject "Hello!"
-sleep 10s -dryRun=false
Do the same, but only to example@example.com:
notify-mailer -config test/config/notify-mailer.json
-body cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com
-recipientList cmd/notify-mailer/testdata/test_msg_recipients.csv -subject "Hello!"
-start example@example.com -end example@example.comX
Send the message starting with example@example.com and emailing every address that's
alphabetically higher:
notify-mailer -config test/config/notify-mailer.json
-body cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com
-recipientList cmd/notify-mailer/testdata/test_msg_recipients.csv -subject "Hello!"
-start example@example.com
Required arguments:
- body
- config
- from
- subject
- recipientList`
func main() {
from := flag.String("from", "", "From header for emails. Must be a bare email address.")
subject := flag.String("subject", "", "Subject of emails")
recipientListFile := flag.String("recipientList", "", "File containing a CSV list of registration IDs and extra info.")
bodyFile := flag.String("body", "", "File containing the email body in Golang template format.")
dryRun := flag.Bool("dryRun", true, "Whether to do a dry run.")
sleep := flag.Duration("sleep", 500*time.Millisecond, "How long to sleep between emails.")
start := flag.String("start", "", "Alphabetically lowest email address to include.")
end := flag.String("end", "\xFF", "Alphabetically highest email address (exclusive).")
reconnBase := flag.Duration("reconnectBase", 1*time.Second, "Base sleep duration between reconnect attempts")
reconnMax := flag.Duration("reconnectMax", 5*60*time.Second, "Max sleep duration between reconnect attempts after exponential backoff")
type config struct {
NotifyMailer struct {
DB cmd.DBConfig
cmd.PasswordConfig
cmd.SMTPConfig
Features map[string]bool
}
Syslog cmd.SyslogConfig
}
configFile := flag.String("config", "", "File containing a JSON config.")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "%s\n\n", usageIntro)
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
if *from == "" || *subject == "" || *bodyFile == "" || *configFile == "" ||
*recipientListFile == "" {
flag.Usage()
os.Exit(1)
}
configData, err := ioutil.ReadFile(*configFile)
cmd.FailOnError(err, fmt.Sprintf("Reading %q", *configFile))
var cfg config
err = json.Unmarshal(configData, &cfg)
cmd.FailOnError(err, "Unmarshaling config")
err = features.Set(cfg.NotifyMailer.Features)
cmd.FailOnError(err, "Failed to set feature flags")
log := cmd.NewLogger(cfg.Syslog)
defer log.AuditPanic()
dbURL, err := cfg.NotifyMailer.DB.URL()
cmd.FailOnError(err, "Couldn't load DB URL")
dbSettings := sa.DbSettings{
MaxOpenConns: 10,
}
dbMap, err := sa.NewDbMap(dbURL, dbSettings)
cmd.FailOnError(err, "Could not connect to database")
// Load email body
body, err := ioutil.ReadFile(*bodyFile)
cmd.FailOnError(err, fmt.Sprintf("Reading %q", *bodyFile))
template, err := template.New("email").Parse(string(body))
cmd.FailOnError(err, fmt.Sprintf("Parsing template %q", *bodyFile))
address, err := mail.ParseAddress(*from)
cmd.FailOnError(err, fmt.Sprintf("Parsing %q", *from))
recipients, err := readRecipientsList(*recipientListFile)
cmd.FailOnError(err, fmt.Sprintf("Reading %q", *recipientListFile))
targetRange := interval{
start: *start,
end: *end,
}
var mailClient bmail.Mailer
if *dryRun {
log.Infof("Doing a dry run.")
mailClient = bmail.NewDryRun(*address, log)
} else {
smtpPassword, err := cfg.NotifyMailer.PasswordConfig.Pass()
cmd.FailOnError(err, "Failed to load SMTP password")
mailClient = bmail.New(
cfg.NotifyMailer.Server,
cfg.NotifyMailer.Port,
cfg.NotifyMailer.Username,
smtpPassword,
nil,
*address,
log,
metrics.NoopRegisterer,
*reconnBase,
*reconnMax)
}
m := mailer{
clk: cmd.Clock(),
log: log,
dbMap: dbMap,
mailer: mailClient,
subject: *subject,
destinations: recipients,
emailTemplate: template,
targetRange: targetRange,
sleepInterval: *sleep,
}
err = m.run()
cmd.FailOnError(err, "mailer.send returned error")
}