vault: ensure ttl expired tokens are purge

If a token is scheduled for revocation expires before we revoke it,
ensure that it is marked as purged in raft and is only removed from
local vault state if the purge operation succeeds.

Prior to this change, we may remove the accessor from local state but
not purge it from Raft.  This causes unnecessary and churn in the next
leadership elections (and until 0.11.2 result in indefinite retries).

nomad/vault.go

@@ -1262,9 +1262,10 @@ func (v *vaultClient) revokeDaemon() {
 				toRevoke = maxVaultRevokeBatchSize
 			}
 			revoking := make([]*structs.VaultAccessor, 0, toRevoke)
+			ttlExpired := []*structs.VaultAccessor{}
 			for va, ttl := range v.revoking {
 				if now.After(ttl) {
-					delete(v.revoking, va)
+					ttlExpired = append(ttlExpired, va)
 				} else {
 					revoking = append(revoking, va)
 				}
@@ -1283,6 +1284,10 @@ func (v *vaultClient) revokeDaemon() {
 			// Unlock before a potentially expensive operation
 			v.revLock.Unlock()
 
+			// purge all explicitly revoked as well as ttl expired tokens
+			// and only remove them locally on purge success
+			revoking = append(revoking, ttlExpired...)
+
 			// Call the passed in token revocation function
 			if err := v.purgeFn(revoking); err != nil {
 				// Can continue since revocation is idempotent