Replace MD5 use in rcache with SHA-256

The rcache implementation uses an unkeyed MD5 hash of the
authenticator to distinguish between different requests with equal
client principal, server principal, and microsecond time.  When the
OpenSSL crypto provider is used and the underlying OpenSSL library is
run in FIPS mode, the MD5 algorithm is disabled and
gss_accept_sec_context() results in an abort in rcache processing.

This change effectively implements a different rcache extension.
The new extension identifier is 'SHA256:' (instead of 'HASH:')
and the new has algorithm is SHA-256.

ticket: 8353 (new)

src/lib/krb5/rcache/rc_conv.c

@@ -38,39 +38,39 @@ krb5_auth_to_rep(krb5_context context, krb5_tkt_authent *auth, krb5_donot_replay
  * Generate a printable hash value for a message for use in a replay
  * record.  It is not necessary for this hash function to be
  * collision-proof (the only thing you can do with a second preimage
- * is produce a false replay error) but it is necessary for the
- * function to be consistent across implementations.  We do an unkeyed
- * MD5 hash of the message and convert it into uppercase hex
+ * is produce a false replay error) but for fine granularity replay detection
+ * it is necessary for the function to be consistent across implementations.
+ * When two implementations sharing a single replay cache don't agree on hash
+ * function, the code falls back to legacy replay detection based on
+ * (client, server, timestamp, usec) tuples.  We do an unkeyed
+ * SHA256 hash of the message and convert it into uppercase hex
  * representation.
  */
 krb5_error_code
 krb5_rc_hash_message(krb5_context context, const krb5_data *message,
                      char **out)
 {
     krb5_error_code retval;
-    krb5_checksum cksum;
+    uint8_t cksum[K5_SHA256_HASHLEN];
     char *hash, *ptr;
     unsigned int i;
 
     *out = NULL;
 
     /* Calculate the binary checksum. */
-    retval = krb5_c_make_checksum(context, CKSUMTYPE_RSA_MD5, 0, 0,
-                                  message, &cksum);
+    retval = k5_sha256(message, cksum);
     if (retval)
         return retval;
 
     /* Convert the checksum into printable form. */
-    hash = malloc(cksum.length * 2 + 1);
+    hash = malloc(K5_SHA256_HASHLEN * 2 + 1);
     if (!hash) {
-        krb5_free_checksum_contents(context, &cksum);
         return KRB5_RC_MALLOC;
     }
 
-    for (i = 0, ptr = hash; i < cksum.length; i++, ptr += 2)
-        snprintf(ptr, 3, "%02X", cksum.contents[i]);
+    for (i = 0, ptr = hash; i < K5_SHA256_HASHLEN; i++, ptr += 2)
+        snprintf(ptr, 3, "%02X", cksum[i]);
     *ptr = '\0';
     *out = hash;
-    krb5_free_checksum_contents(context, &cksum);
     return 0;
 }

src/lib/krb5/rcache/rc_dfl.c

@@ -386,7 +386,7 @@ parse_counted_string(char **strptr, char **result)
 /*
  * Hash extension records have the format:
  *  client = <empty string>
- *  server = HASH:<msghash> <clientlen>:<client> <serverlen>:<server>
+ *  server = SHA256:<msghash> <clientlen>:<client> <serverlen>:<server>
  * Spaces in the client and server string are represented with
  * with backslashes.  Client and server lengths are represented in
  * ASCII decimal (which is different from the 32-bit binary we use
@@ -403,11 +403,11 @@ check_hash_extension(krb5_donot_replay *rep)
     /* Check if this appears to match the hash extension format. */
     if (*rep->client)
         return 0;
-    if (strncmp(rep->server, "HASH:", 5) != 0)
+    if (strncmp(rep->server, "SHA256:", 7) != 0)
         return 0;
 
     /* Parse out the message hash. */
-    str = rep->server + 5;
+    str = rep->server + 7;
     end = strchr(str, ' ');
     if (!end)
         return 0;
@@ -659,7 +659,7 @@ krb5_rc_io_store(krb5_context context, struct dfl_data *t,
 
         /* Format the extension value so we know its length. */
         k5_buf_init_dynamic(&extbuf);
-        k5_buf_add_fmt(&extbuf, "HASH:%s %lu:%s %lu:%s", rep->msghash,
+        k5_buf_add_fmt(&extbuf, "SHA256:%s %lu:%s %lu:%s", rep->msghash,
                        (unsigned long)clientlen, rep->client,
                        (unsigned long)serverlen, rep->server);
         if (k5_buf_status(&extbuf) != 0)