Add more logging to TokenService on superseding doc (#68721)

This adds two new log messages to TokenService

1. After finding a superseding document, log (DEBUG) that it was
successfully found (previously we only logged not-found+retries)
2. If a document cannot be found in order to validate it, log (WARN)
this explicitly.

Backport of: #66970

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java

@@ -1183,6 +1183,8 @@ void decryptAndReturnSupersedingTokens(String refreshToken, RefreshTokenStatus r
                     public void onResponse(GetResponse response) {
                         if (response.isExists()) {
                             try {
+                                logger.debug("Found superseding document: index=[{}] id=[{}] primTerm=[{}] seqNo=[{}]",
+                                    response.getIndex(), response.getId(), response.getPrimaryTerm(), response.getSeqNo());
                                 listener.onResponse(
                                     new CreateTokenResult(prependVersionAndEncodeAccessToken(refreshTokenStatus.getVersion(),
                                         decryptedTokens[0]),
@@ -1662,7 +1664,18 @@ private void checkIfTokenIsValid(UserToken userToken, ActionListener<UserToken>
                                 }
                             }
                         } else {
-                            onFailure.accept(new IllegalStateException("token document is missing and must be present"));
+                            // This shouldn't happen (if we have a valid `UserToken` object, then should mean that we loaded it from the
+                            // index in a prior operation (e.g. #getUserTokenFromId), however this error can happen if either:
+                            // 1. The document was deleted just after we read it
+                            // 2. This Get used a different replica to the previous one, and they were out of sync.
+                            logger.warn(
+                                "Could not find token document (index=[{}] id=[{}]) in order to validate user token [{}] for [{}]",
+                                response.getIndex(),
+                                response.getId(),
+                                userToken.getId(),
+                                userToken.getAuthentication().getUser().principal());
+                            onFailure.accept(traceLog("validate token", userToken.getId(),
+                                new IllegalStateException("token document is missing and must be present")));
                         }
                     }, e -> {
                         // if the index or the shard is not there / available we assume that

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/UserToken.java

@@ -93,7 +93,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     /**
-     * Get the authentication
+     * Get the authentication (will not be null)
      */
     Authentication getAuthentication() {
         return authentication;