Skip to content

Add authz2 expired-authz-purger tool #4226

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rolandshoemaker merged 4 commits into from
May 30, 2019
Merged

Add authz2 expired-authz-purger tool #4226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1fb1d5f
Add authz2 expired-authz-purger tool
rolandshoemaker May 23, 2019
1e58f58
Review feedback
rolandshoemaker May 24, 2019
ca42a42
Actually add the expires index migration
rolandshoemaker May 24, 2019
768b768
Quotify
rolandshoemaker May 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions cmd/expired-authz-purger2/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"expiredAuthzPurger2": {
"syslog": {
"stdoutLevel": 6
},
"dbConnectFile": "test/secrets/purger_dburl",
"maxDBConns": 10,
"gracePeriod": "168h",
"batchSize": 1000,
"debugAddr": ":8015"
}
}
104 changes: 104 additions & 0 deletions cmd/expired-authz-purger2/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package main

import (
"encoding/json"
"flag"
"io/ioutil"
"time"

"gopkg.in/go-gorp/gorp.v2"

"github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/features"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/metrics"
"github.com/letsencrypt/boulder/sa"
"github.com/prometheus/client_golang/prometheus"
)

type config struct {
ExpiredAuthzPurger2 struct {
cmd.DBConfig
DebugAddr string
Syslog cmd.SyslogConfig
Features map[string]bool

GracePeriod cmd.ConfigDuration
BatchSize int
WaitDuration cmd.ConfigDuration
}
}

var deletedStat = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "eap2_authorizations_deleted",
Help: "Number of authorizations the EAP2 tool has deleted.",
},
)

func deleteExpired(clk clock.Clock, gracePeriod time.Duration, batchSize int, dbMap *gorp.DbMap) (int64, error) {
expires := clk.Now().Add(-gracePeriod)
res, err := dbMap.Exec(
"DELETE FROM authz2 WHERE expires <= :expires LIMIT :limit",
map[string]interface{}{
"expires": expires,
"limit": batchSize,
},
)
if err != nil {
return 0, err
}
return res.RowsAffected()
}

func main() {
singleRun := flag.Bool("single-run", false, "Exit after running first delete query instead of running indefinitely")
configPath := flag.String("config", "config.json", "Path to Boulder configuration file")
flag.Parse()

configJSON, err := ioutil.ReadFile(*configPath)
cmd.FailOnError(err, "Failed to read config file")
var c config
err = json.Unmarshal(configJSON, &c)
cmd.FailOnError(err, "Failed to parse config file")
err = features.Set(c.ExpiredAuthzPurger2.Features)
cmd.FailOnError(err, "Failed to set feature flags")

var logger blog.Logger
if c.ExpiredAuthzPurger2.DebugAddr != "" {
var scope metrics.Scope
scope, logger = cmd.StatsAndLogging(c.ExpiredAuthzPurger2.Syslog, c.ExpiredAuthzPurger2.DebugAddr)
scope.MustRegister(deletedStat)
} else {
logger = cmd.NewLogger(c.ExpiredAuthzPurger2.Syslog)
}
defer logger.AuditPanic()
logger.Info(cmd.VersionString())

clk := cmd.Clock()

dbURL, err := c.ExpiredAuthzPurger2.DBConfig.URL()
cmd.FailOnError(err, "Couldn't load DB URL")
dbMap, err := sa.NewDbMap(dbURL, c.ExpiredAuthzPurger2.DBConfig.MaxDBConns)
cmd.FailOnError(err, "Could not connect to database")

for {
deleted, err := deleteExpired(clk, c.ExpiredAuthzPurger2.GracePeriod.Duration, c.ExpiredAuthzPurger2.BatchSize, dbMap)
if err != nil {
logger.Errf("failed to purge expired authorizations: %s", err)
if !*singleRun {
continue
}
}
logger.Infof("deleted %d expired authorizations", deleted)
if *singleRun {
break
}
deletedStat.Add(float64(deleted))
if deleted == 0 {
// Nothing to do, sit around a while and wait
time.Sleep(c.ExpiredAuthzPurger2.WaitDuration.Duration)
}
}
}
8 changes: 8 additions & 0 deletions sa/_db-next/migrations/20190524120239_AddAuthz2ExpiresIndex.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@

-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied
CREATE INDEX `expires_idx` ON `authz2` (`expires`);

-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back
DROP INDEX `expires_idx` ON `authz2`;
18 changes: 13 additions & 5 deletions test/integration-test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,23 @@ def run_expired_authz_purger():
# (e.g. test_expired_authzs_404).

def expect(target_time, num, table):
out = get_future_output("./bin/expired-authz-purger --config cmd/expired-authz-purger/config.json", target_time)
if CONFIG_NEXT:
tool = "expired-authz-purger2"
out = get_future_output("./bin/expired-authz-purger2 --single-run --config cmd/expired-authz-purger2/config.json", target_time)
else:
tool = "expired-authz-purger"
out = get_future_output("./bin/expired-authz-purger --config cmd/expired-authz-purger/config.json", target_time)
if 'via FAKECLOCK' not in out:
raise Exception("expired-authz-purger was not built with `integration` build tag")
raise Exception("%s was not built with `integration` build tag" % (tool))
if num is None:
return
expected_output = 'Deleted a total of %d expired authorizations from %s' % (num, table)
if CONFIG_NEXT:
expected_output = 'deleted %d expired authorizations' % (num)
else:
expected_output = 'Deleted a total of %d expired authorizations from %s' % (num, table)
if expected_output not in out:
raise Exception("expired-authz-purger did not print '%s'. Output:\n%s" % (
expected_output, out))
raise Exception("%s did not print '%s'. Output:\n%s" % (
tool, expected_output, out))

now = datetime.datetime.utcnow()

Expand Down
1 change: 1 addition & 0 deletions test/sa_db_users.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ GRANT SELECT ON certificates TO 'cert_checker'@'localhost';
GRANT SELECT,DELETE ON pendingAuthorizations TO 'purger'@'localhost';
GRANT SELECT,DELETE ON authz TO 'purger'@'localhost';
GRANT SELECT,DELETE ON challenges TO 'purger'@'localhost';
GRANT SELECT,DELETE ON authz2 TO 'purger'@'localhost';

-- Test setup and teardown
GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';