From da6b4c58b05f311c8b732f9c45bcffe6c2ac06e7 Mon Sep 17 00:00:00 2001 From: hc-github-team-secure-vault-core <82990506+hc-github-team-secure-vault-core@users.noreply.github.com> Date: Tue, 26 Apr 2022 09:05:25 -0400 Subject: [PATCH] Backport of fix raft tls key rotation panic when rotation time in past into release/1.8.x (#15174) * backport of commit eda2fd830f46a0ea44ce363b5e8daebbed8bf289 * backport of commit 2962018ed532a7c58949910cb7148580bafcb6b7 * backport of commit bc084cf23e539794cbd75877a516608c91aa8dd3 * backport of commit 7e3408c1ebce9030e584f6b215ec9373cd9e17cb * backport of commit 3d4ef0d89f791e9aeb2aab10c360b60545c1b945 * backport of commit 8c5cf17931b3d842fb72bf827a58f9491b59251b Co-authored-by: Chris Capurso <1036769+ccapurso@users.noreply.github.com> --- changelog/15156.txt | 3 +++ vault/raft.go | 33 ++++++++++++++++++++------------- 2 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 changelog/15156.txt diff --git a/changelog/15156.txt b/changelog/15156.txt new file mode 100644 index 0000000000000..8718f1b1a81cd --- /dev/null +++ b/changelog/15156.txt @@ -0,0 +1,3 @@ +```release-note:bug +rafft: fix Raft TLS key rotation panic that occurs if active key is more than 24 hours old +``` diff --git a/vault/raft.go b/vault/raft.go index 6268795bf0a5e..427af3bd21825 100644 --- a/vault/raft.go +++ b/vault/raft.go @@ -473,27 +473,33 @@ func (c *Core) raftTLSRotatePhased(ctx context.Context, logger hclog.Logger, raf return errors.New("no active raft TLS key found") } + getNextRotationTime := func(next time.Time) time.Time { + now := time.Now() + + // active key's CreatedTime + raftTLSRotationPeriod might be in + // the past (meaning it is ready to be rotated) which will cause + // NewTicker to panic when used with time.Until, prevent this by + // pushing out rotation time into very near future + if next.Before(now) { + return now.Add(1 * time.Minute) + } + + // push out to ensure proposed time does not elapse + return next.Add(10 * time.Second) + } + // Start the process in a go routine go func() { - nextRotationTime := activeKey.CreatedTime.Add(raftTLSRotationPeriod) + nextRotationTime := getNextRotationTime(activeKey.CreatedTime.Add(raftTLSRotationPeriod)) keyCheckInterval := time.NewTicker(1 * time.Minute) defer keyCheckInterval.Stop() - var backoff bool // ticker is used to prevent memory leak of using time.After in // for - select pattern. ticker := time.NewTicker(time.Until(nextRotationTime)) defer ticker.Stop() for { - // If we encountered and error we should try to create the key - // again. - if backoff { - nextRotationTime = time.Now().Add(10 * time.Second) - backoff = false - } - - ticker.Reset(time.Until(nextRotationTime)) select { case <-keyCheckInterval.C: err := checkCommitted() @@ -505,11 +511,12 @@ func (c *Core) raftTLSRotatePhased(ctx context.Context, logger hclog.Logger, raf next, err := rotateKeyring() if err != nil { logger.Error("failed to rotate TLS key", "error", err) - backoff = true - continue + nextRotationTime = time.Now().Add(10 * time.Second) + } else { + nextRotationTime = getNextRotationTime(next) } - nextRotationTime = next + ticker.Reset(time.Until(nextRotationTime)) case <-stopCh: return