diff --git a/cmd/expired-authz-purger2/config.json b/cmd/expired-authz-purger2/config.json new file mode 100644 index 00000000000..d0f7151edab --- /dev/null +++ b/cmd/expired-authz-purger2/config.json @@ -0,0 +1,12 @@ +{ + "expiredAuthzPurger2": { + "syslog": { + "stdoutLevel": 6 + }, + "dbConnectFile": "test/secrets/purger_dburl", + "maxDBConns": 10, + "gracePeriod": "168h", + "batchSize": 1000, + "debugAddr": ":8015" + } +} diff --git a/cmd/expired-authz-purger2/main.go b/cmd/expired-authz-purger2/main.go new file mode 100644 index 00000000000..b6fa261be2e --- /dev/null +++ b/cmd/expired-authz-purger2/main.go @@ -0,0 +1,104 @@ +package main + +import ( + "encoding/json" + "flag" + "io/ioutil" + "time" + + "gopkg.in/go-gorp/gorp.v2" + + "github.com/jmhodges/clock" + "github.com/letsencrypt/boulder/cmd" + "github.com/letsencrypt/boulder/features" + blog "github.com/letsencrypt/boulder/log" + "github.com/letsencrypt/boulder/metrics" + "github.com/letsencrypt/boulder/sa" + "github.com/prometheus/client_golang/prometheus" +) + +type config struct { + ExpiredAuthzPurger2 struct { + cmd.DBConfig + DebugAddr string + Syslog cmd.SyslogConfig + Features map[string]bool + + GracePeriod cmd.ConfigDuration + BatchSize int + WaitDuration cmd.ConfigDuration + } +} + +var deletedStat = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "eap2_authorizations_deleted", + Help: "Number of authorizations the EAP2 tool has deleted.", + }, +) + +func deleteExpired(clk clock.Clock, gracePeriod time.Duration, batchSize int, dbMap *gorp.DbMap) (int64, error) { + expires := clk.Now().Add(-gracePeriod) + res, err := dbMap.Exec( + "DELETE FROM authz2 WHERE expires <= :expires LIMIT :limit", + map[string]interface{}{ + "expires": expires, + "limit": batchSize, + }, + ) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +func main() { + singleRun := flag.Bool("single-run", false, "Exit after running first delete query instead of running indefinitely") + configPath := flag.String("config", "config.json", "Path to Boulder configuration file") + flag.Parse() + + configJSON, err := ioutil.ReadFile(*configPath) + cmd.FailOnError(err, "Failed to read config file") + var c config + err = json.Unmarshal(configJSON, &c) + cmd.FailOnError(err, "Failed to parse config file") + err = features.Set(c.ExpiredAuthzPurger2.Features) + cmd.FailOnError(err, "Failed to set feature flags") + + var logger blog.Logger + if c.ExpiredAuthzPurger2.DebugAddr != "" { + var scope metrics.Scope + scope, logger = cmd.StatsAndLogging(c.ExpiredAuthzPurger2.Syslog, c.ExpiredAuthzPurger2.DebugAddr) + scope.MustRegister(deletedStat) + } else { + logger = cmd.NewLogger(c.ExpiredAuthzPurger2.Syslog) + } + defer logger.AuditPanic() + logger.Info(cmd.VersionString()) + + clk := cmd.Clock() + + dbURL, err := c.ExpiredAuthzPurger2.DBConfig.URL() + cmd.FailOnError(err, "Couldn't load DB URL") + dbMap, err := sa.NewDbMap(dbURL, c.ExpiredAuthzPurger2.DBConfig.MaxDBConns) + cmd.FailOnError(err, "Could not connect to database") + + for { + deleted, err := deleteExpired(clk, c.ExpiredAuthzPurger2.GracePeriod.Duration, c.ExpiredAuthzPurger2.BatchSize, dbMap) + if err != nil { + logger.Errf("failed to purge expired authorizations: %s", err) + if !*singleRun { + continue + } + } + logger.Infof("deleted %d expired authorizations", deleted) + if *singleRun { + break + } + deletedStat.Add(float64(deleted)) + if deleted == 0 { + // Nothing to do, sit around a while and wait + time.Sleep(c.ExpiredAuthzPurger2.WaitDuration.Duration) + } + } +} diff --git a/sa/_db-next/migrations/20190524120239_AddAuthz2ExpiresIndex.sql b/sa/_db-next/migrations/20190524120239_AddAuthz2ExpiresIndex.sql new file mode 100644 index 00000000000..4e055bb85da --- /dev/null +++ b/sa/_db-next/migrations/20190524120239_AddAuthz2ExpiresIndex.sql @@ -0,0 +1,8 @@ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied +CREATE INDEX `expires_idx` ON `authz2` (`expires`); + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP INDEX `expires_idx` ON `authz2`; diff --git a/test/integration-test.py b/test/integration-test.py index 7d00256b6fb..a008ebfcbb0 100644 --- a/test/integration-test.py +++ b/test/integration-test.py @@ -71,15 +71,23 @@ def run_expired_authz_purger(): # (e.g. test_expired_authzs_404). def expect(target_time, num, table): - out = get_future_output("./bin/expired-authz-purger --config cmd/expired-authz-purger/config.json", target_time) + if CONFIG_NEXT: + tool = "expired-authz-purger2" + out = get_future_output("./bin/expired-authz-purger2 --single-run --config cmd/expired-authz-purger2/config.json", target_time) + else: + tool = "expired-authz-purger" + out = get_future_output("./bin/expired-authz-purger --config cmd/expired-authz-purger/config.json", target_time) if 'via FAKECLOCK' not in out: - raise Exception("expired-authz-purger was not built with `integration` build tag") + raise Exception("%s was not built with `integration` build tag" % (tool)) if num is None: return - expected_output = 'Deleted a total of %d expired authorizations from %s' % (num, table) + if CONFIG_NEXT: + expected_output = 'deleted %d expired authorizations' % (num) + else: + expected_output = 'Deleted a total of %d expired authorizations from %s' % (num, table) if expected_output not in out: - raise Exception("expired-authz-purger did not print '%s'. Output:\n%s" % ( - expected_output, out)) + raise Exception("%s did not print '%s'. Output:\n%s" % ( + tool, expected_output, out)) now = datetime.datetime.utcnow() diff --git a/test/sa_db_users.sql b/test/sa_db_users.sql index 8b0052e8951..69c748d93e2 100644 --- a/test/sa_db_users.sql +++ b/test/sa_db_users.sql @@ -71,6 +71,7 @@ GRANT SELECT ON certificates TO 'cert_checker'@'localhost'; GRANT SELECT,DELETE ON pendingAuthorizations TO 'purger'@'localhost'; GRANT SELECT,DELETE ON authz TO 'purger'@'localhost'; GRANT SELECT,DELETE ON challenges TO 'purger'@'localhost'; +GRANT SELECT,DELETE ON authz2 TO 'purger'@'localhost'; -- Test setup and teardown GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';