Implement HMAC over SHA3 truncated variants (#2484)

This PR implements HMAC over truncated SHA3 variants as specified in
[NIST SP
800-224](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-224.ipd.pdf#section.3).
We do so without externalizing any SHA3/keccak internals, including its
context struct. This is done intentionally, as OpenSSL [has not yet
externalized its SHA3/keccak context
struct](https://github.com/openssl/openssl/blob/be7467f5a0aa098531597b95a71be6d7c2a463c7/include/internal/sha3.h#L40)
and we want to leave the door open to future interoperability. To work
around this in `hmac.h`'s `md_ctx_union`, we hard-code the context
struct size and add a compile-time assertion that it does not grow
larger than the hard-coded value.

SHA3 is unique in supported HMAC digests in that, due to differences
between its sponge construction and others' Merkle-Dåmgard
constructions, it does not support pre-computed keys. To accommodate
this difference, we refactor the relevant code generation macros and
relevant unit tests.

The HMAC-SHA3 feature gap was discovered in a somewhat roundabout way.
While preparing a [pull
request](python/cpython#135402) to add AWS-LC to
upstream CPython's CI, I discovered that CPython's `./configure`
script's compile probe failed to detect `libcrypto` support for linking
`hashlib`. The compile probe
[referenced](https://github.com/python/cpython/blob/59963e866a1bb8128a50cd53d1b13eeab03df06e/configure#L30869)
`NID_blake2b512`, which AWS-LC does not support. The consequence of this
was that CPython used its HACL implementations for `hashlib` instead of
linking AWS-LC. This did not affect our `ssl` integration, as AWS-LC
always uses its own hash functions for TLS.

Author: WillChilds-Klein

crypto/fipsmodule/hmac/hmac.c

@@ -96,10 +96,6 @@ struct hmac_methods_st {
   static int AWS_LC_TRAMPOLINE_##HASH_NAME##_Update(void *, const void *,     \
                                                     size_t);                  \
   static int AWS_LC_TRAMPOLINE_##HASH_NAME##_Final(uint8_t *, void *);        \
-  static int AWS_LC_TRAMPOLINE_##HASH_NAME##_Init_from_state(                 \
-      void *, const uint8_t *, uint64_t);                                     \
-  static int AWS_LC_TRAMPOLINE_##HASH_NAME##_get_state(void *, uint8_t *,     \
-                                                       uint64_t *);           \
   static int AWS_LC_TRAMPOLINE_##HASH_NAME##_Init(void *ctx) {                \
     return HASH_NAME##_Init((HASH_CTX *)ctx);                                 \
   }                                                                           \
@@ -110,6 +106,18 @@ struct hmac_methods_st {
   static int AWS_LC_TRAMPOLINE_##HASH_NAME##_Final(uint8_t *out, void *ctx) { \
     return HASH_NAME##_Final(out, (HASH_CTX *)ctx);                           \
   }                                                                           \
+  OPENSSL_STATIC_ASSERT(HASH_CBLOCK % 8 == 0,                                 \
+                        HASH_NAME##_has_blocksize_not_divisible_by_eight_t)   \
+  OPENSSL_STATIC_ASSERT(HASH_CBLOCK <= EVP_MAX_MD_BLOCK_SIZE,                 \
+                        HASH_NAME##_has_overlarge_blocksize_t)                \
+  OPENSSL_STATIC_ASSERT(sizeof(HASH_CTX) <= sizeof(union md_ctx_union),       \
+                        HASH_NAME##_has_overlarge_context_t)
+
+// For merkle-damgard constructions, we also define functions for importing and
+// exporting hash state for precomputed keys. These are not applicable to
+// Keccak/SHA3.
+#define MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(HASH_NAME, HASH_CTX, HASH_CBLOCK) \
+  MD_TRAMPOLINES_EXPLICIT(HASH_NAME, HASH_CTX, HASH_CBLOCK);                  \
   static int AWS_LC_TRAMPOLINE_##HASH_NAME##_Init_from_state(                 \
       void *ctx, const uint8_t *h, uint64_t n) {                              \
     return HASH_NAME##_Init_from_state((HASH_CTX *)ctx, h, n);                \
@@ -118,62 +126,75 @@ struct hmac_methods_st {
       void *ctx, uint8_t *out_h, uint64_t *out_n) {                           \
     return HASH_NAME##_get_state((HASH_CTX *)ctx, out_h, out_n);              \
   }                                                                           \
-  OPENSSL_STATIC_ASSERT(HASH_CBLOCK % 8 == 0,                                 \
-                        HASH_NAME##_has_blocksize_not_divisible_by_eight_t)   \
-  OPENSSL_STATIC_ASSERT(HASH_CBLOCK <= EVP_MAX_MD_BLOCK_SIZE,                 \
-                        HASH_NAME##_has_overlarge_blocksize_t)                \
   OPENSSL_STATIC_ASSERT(HMAC_##HASH_NAME##_PRECOMPUTED_KEY_SIZE ==            \
                             2 * HASH_NAME##_CHAINING_LENGTH,                  \
                         HASH_NAME##_has_incorrect_precomputed_key_size)       \
   OPENSSL_STATIC_ASSERT(HMAC_##HASH_NAME##_PRECOMPUTED_KEY_SIZE <=            \
                             HMAC_MAX_PRECOMPUTED_KEY_SIZE,                    \
                         HASH_NAME##_has_too_large_precomputed_key_size)       \
-  OPENSSL_STATIC_ASSERT(sizeof(HASH_CTX) <= sizeof(union md_ctx_union),       \
-                        HASH_NAME##_has_overlarge_context_t)
 
 // The maximum number of HMAC implementations
-#define HMAC_METHOD_MAX 8
-
-MD_TRAMPOLINES_EXPLICIT(MD5, MD5_CTX, MD5_CBLOCK)
-MD_TRAMPOLINES_EXPLICIT(SHA1, SHA_CTX, SHA_CBLOCK)
-MD_TRAMPOLINES_EXPLICIT(SHA224, SHA256_CTX, SHA256_CBLOCK)
-MD_TRAMPOLINES_EXPLICIT(SHA256, SHA256_CTX, SHA256_CBLOCK)
-MD_TRAMPOLINES_EXPLICIT(SHA384, SHA512_CTX, SHA512_CBLOCK)
-MD_TRAMPOLINES_EXPLICIT(SHA512, SHA512_CTX, SHA512_CBLOCK)
-MD_TRAMPOLINES_EXPLICIT(SHA512_224, SHA512_CTX, SHA512_CBLOCK)
-MD_TRAMPOLINES_EXPLICIT(SHA512_256, SHA512_CTX, SHA512_CBLOCK)
+#define HMAC_METHOD_MAX 12
+
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(MD5, MD5_CTX, MD5_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(SHA1, SHA_CTX, SHA_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(SHA224, SHA256_CTX, SHA256_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(SHA256, SHA256_CTX, SHA256_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(SHA384, SHA512_CTX, SHA512_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(SHA512, SHA512_CTX, SHA512_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(SHA512_224, SHA512_CTX, SHA512_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT_PRECOMPUTED(SHA512_256, SHA512_CTX, SHA512_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT(SHA3_224, KECCAK1600_CTX, SHA3_224_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT(SHA3_256, KECCAK1600_CTX, SHA3_256_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT(SHA3_384, KECCAK1600_CTX, SHA3_384_CBLOCK)
+MD_TRAMPOLINES_EXPLICIT(SHA3_512, KECCAK1600_CTX, SHA3_512_CBLOCK)
 
 struct hmac_method_array_st {
   HmacMethods methods[HMAC_METHOD_MAX];
 };
 
+// This macro does not set any values for precomputed keys for portable state,
+// and as such is suitable for use with Keccak/SHA3.
 #define DEFINE_IN_PLACE_METHODS(EVP_MD, HASH_NAME)  {                        \
     out->methods[idx].evp_md = EVP_MD;                                       \
-    out->methods[idx].chaining_length = HASH_NAME##_CHAINING_LENGTH;         \
     out->methods[idx].init = AWS_LC_TRAMPOLINE_##HASH_NAME##_Init;           \
     out->methods[idx].update = AWS_LC_TRAMPOLINE_##HASH_NAME##_Update;       \
     out->methods[idx].finalize = AWS_LC_TRAMPOLINE_##HASH_NAME##_Final;      \
-    out->methods[idx].init_from_state =                                      \
-        AWS_LC_TRAMPOLINE_##HASH_NAME##_Init_from_state;                     \
-    out->methods[idx].get_state = AWS_LC_TRAMPOLINE_##HASH_NAME##_get_state; \
+    out->methods[idx].chaining_length = 0UL;                                 \
+    out->methods[idx].init_from_state = NULL;                                \
+    out->methods[idx].get_state = NULL;                                      \
     idx++;                                                                   \
     assert(idx <= HMAC_METHOD_MAX);                                          \
   }
 
+// Use |idx-1| because DEFINE_IN_PLACE_METHODS has already incremented it.
+#define DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_MD, HASH_NAME)  {            \
+    DEFINE_IN_PLACE_METHODS(EVP_MD, HASH_NAME);                              \
+    assert(idx-1 >= 0);                                                      \
+    out->methods[idx-1].chaining_length = HASH_NAME##_CHAINING_LENGTH;       \
+    out->methods[idx-1].init_from_state =                                    \
+        AWS_LC_TRAMPOLINE_##HASH_NAME##_Init_from_state;                     \
+    out->methods[idx-1].get_state =                                          \
+        AWS_LC_TRAMPOLINE_##HASH_NAME##_get_state;                           \
+  }
+
 DEFINE_LOCAL_DATA(struct hmac_method_array_st, AWSLC_hmac_in_place_methods) {
   OPENSSL_memset((void*) out->methods, 0, sizeof(out->methods));
   int idx = 0;
   // Since we search these linearly it helps (just a bit) to put the most common ones first.
   // This isn't based on hard metrics and will not make a significant different on performance.
-  // FIXME: all hashes but SHA256 have been disabled to check first SHA256
-  DEFINE_IN_PLACE_METHODS(EVP_sha256(), SHA256);
-  DEFINE_IN_PLACE_METHODS(EVP_sha1(), SHA1);
-  DEFINE_IN_PLACE_METHODS(EVP_sha384(), SHA384);
-  DEFINE_IN_PLACE_METHODS(EVP_sha512(), SHA512);
-  DEFINE_IN_PLACE_METHODS(EVP_md5(), MD5);
-  DEFINE_IN_PLACE_METHODS(EVP_sha224(), SHA224);
-  DEFINE_IN_PLACE_METHODS(EVP_sha512_224(), SHA512_224);
-  DEFINE_IN_PLACE_METHODS(EVP_sha512_256(), SHA512_256);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_sha256(), SHA256);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_sha1(), SHA1);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_sha384(), SHA384);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_sha512(), SHA512);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_md5(), MD5);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_sha224(), SHA224);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_sha512_224(), SHA512_224);
+  DEFINE_IN_PLACE_METHODS_PRECOMPUTED(EVP_sha512_256(), SHA512_256);
+  DEFINE_IN_PLACE_METHODS(EVP_sha3_224(), SHA3_224);
+  DEFINE_IN_PLACE_METHODS(EVP_sha3_256(), SHA3_256);
+  DEFINE_IN_PLACE_METHODS(EVP_sha3_384(), SHA3_384);
+  DEFINE_IN_PLACE_METHODS(EVP_sha3_512(), SHA3_512);
 }
 
 static const HmacMethods *GetInPlaceMethods(const EVP_MD *evp_md) {
@@ -494,6 +515,11 @@ void HMAC_CTX_reset(HMAC_CTX *ctx) {
 }
 
 int HMAC_set_precomputed_key_export(HMAC_CTX *ctx) {
+  GUARD_PTR(ctx);
+  if (ctx->methods != NULL && ctx->methods->get_state == NULL) {
+    OPENSSL_PUT_ERROR(HMAC, HMAC_R_PRECOMPUTED_KEY_NOT_SUPPORTED_FOR_DIGEST);
+    return 0;
+  }
   if (HMAC_STATE_INIT_NO_DATA != ctx->state &&
       HMAC_STATE_PRECOMPUTED_KEY_EXPORT_READY != ctx->state) {
     // HMAC_set_precomputed_key_export can only be called after Hmac_Init_*
@@ -505,6 +531,13 @@ int HMAC_set_precomputed_key_export(HMAC_CTX *ctx) {
 }
 
 int HMAC_get_precomputed_key(HMAC_CTX *ctx, uint8_t *out, size_t *out_len) {
+  GUARD_PTR(ctx);
+  GUARD_PTR(ctx->methods);
+  if (ctx->methods->get_state == NULL) {
+    OPENSSL_PUT_ERROR(HMAC, HMAC_R_PRECOMPUTED_KEY_NOT_SUPPORTED_FOR_DIGEST);
+    return 0;
+  }
+
   if (HMAC_STATE_PRECOMPUTED_KEY_EXPORT_READY != ctx->state) {
     OPENSSL_PUT_ERROR(EVP, HMAC_R_SET_PRECOMPUTED_KEY_EXPORT_NOT_CALLED);
     return 0;
@@ -583,6 +616,10 @@ int HMAC_Init_from_precomputed_key(HMAC_CTX *ctx,
   }
 
   const HmacMethods *methods = ctx->methods;
+  if (ctx->methods->init_from_state == NULL) {
+    OPENSSL_PUT_ERROR(HMAC, HMAC_R_PRECOMPUTED_KEY_NOT_SUPPORTED_FOR_DIGEST);
+    return 0;
+  }
 
   const size_t chaining_length = methods->chaining_length;
   const size_t block_size = EVP_MD_block_size(methods->evp_md);

crypto/fipsmodule/service_indicator/service_indicator.c

@@ -414,6 +414,10 @@ void HMAC_verify_service_indicator(const EVP_MD *evp_md) {
     case NID_sha512:
     case NID_sha512_224:
     case NID_sha512_256:
+    case NID_sha3_224:
+    case NID_sha3_256:
+    case NID_sha3_384:
+    case NID_sha3_512:
       FIPS_service_indicator_update_state();
       break;
     default:

crypto/fipsmodule/sha/internal.h

@@ -17,6 +17,8 @@
 
 #include <openssl/base.h>
 
+#include <openssl/hmac.h>
+
 #include "../../internal.h"
 #include "../cpucap/internal.h"
 
@@ -43,18 +45,22 @@ extern "C" {
 #define KECCAK1600_WIDTH 1600
 
 #define SHA3_224_CAPACITY_BYTES 56
+#define SHA3_224_CBLOCK SHA3_BLOCKSIZE(SHA3_224_DIGEST_BITLENGTH)
 #define SHA3_224_DIGEST_BITLENGTH 224
 #define SHA3_224_DIGEST_LENGTH 28
 
 #define SHA3_256_CAPACITY_BYTES 64
+#define SHA3_256_CBLOCK SHA3_BLOCKSIZE(SHA3_256_DIGEST_BITLENGTH)
 #define SHA3_256_DIGEST_BITLENGTH 256
 #define SHA3_256_DIGEST_LENGTH 32
 
 #define SHA3_384_CAPACITY_BYTES 96
+#define SHA3_384_CBLOCK SHA3_BLOCKSIZE(SHA3_384_DIGEST_BITLENGTH)
 #define SHA3_384_DIGEST_BITLENGTH 384
 #define SHA3_384_DIGEST_LENGTH 48
 
 #define SHA3_512_CAPACITY_BYTES 128
+#define SHA3_512_CBLOCK SHA3_BLOCKSIZE(SHA3_512_DIGEST_BITLENGTH)
 #define SHA3_512_DIGEST_BITLENGTH 512
 #define SHA3_512_DIGEST_LENGTH 64
 
@@ -81,11 +87,11 @@ extern "C" {
 // so that |SHAKE_Squeeze| cannot be called anymore.
 #define KECCAK1600_STATE_FINAL      2 
 
-typedef struct keccak_st KECCAK1600_CTX;
+typedef struct keccak_ctx_st KECCAK1600_CTX;
 
 // The data buffer should have at least the maximum number of
 // block size bytes to fit any SHA3/SHAKE block length.
-struct keccak_st {
+struct keccak_ctx_st {
   uint64_t A[KECCAK1600_ROWS][KECCAK1600_ROWS];
   size_t block_size;                               // cached ctx->digest->block_size
   size_t md_size;                                  // output length, variable in XOF (SHAKE)
@@ -95,6 +101,15 @@ struct keccak_st {
   uint8_t state;                                   // denotes the keccak phase (absorb, squeeze, final)
 };
 
+// To avoid externalizing KECCAK1600_CTX, we hard-code the context size in
+// hmac.h's |md_ctx_union| and use a compile time check here to make sure
+// |KECCAK1600_CTX|'s size never exceeds that of |md_ctx_union|. This means
+// that whenever a new field is added to |keccak_ctx_st| we must also update
+// the hard-coded size of |sha3| in hmac.h's |md_ctx_union| with the new
+// value given by |sizeof(keccaak_ctx_st)|.
+OPENSSL_STATIC_ASSERT(sizeof(KECCAK1600_CTX) <= sizeof(union md_ctx_union),
+                      hmac_md_ctx_union_sha3_size_needs_update)
+
 // KECCAK1600 x4 batched context structure
 typedef KECCAK1600_CTX KECCAK1600_CTX_x4[4];
 
@@ -445,6 +460,46 @@ int SHA3_Update(KECCAK1600_CTX *ctx, const void *data, size_t len);
 // When call-discipline is maintained, this function never fails.
 int SHA3_Final(uint8_t *md, KECCAK1600_CTX *ctx);
 
+// SHA3_224_Init initialises |sha| and returns 1.
+int SHA3_224_Init(KECCAK1600_CTX *sha);
+
+// SHA3_224_Update adds |len| bytes from |data| to |sha| and returns 1.
+int SHA3_224_Update(KECCAK1600_CTX *sha, const void *data, size_t len);
+
+// SHA3_224_Final adds the final padding to |sha| and writes the resulting
+// digest to |out|. It returns one on success and zero on programmer error.
+int SHA3_224_Final(uint8_t out[SHA3_224_DIGEST_LENGTH], KECCAK1600_CTX *sha);
+
+// SHA3_256_Init initialises |sha| and returns 1.
+int SHA3_256_Init(KECCAK1600_CTX *sha);
+
+// SHA3_256_Update adds |len| bytes from |data| to |sha| and returns 1.
+int SHA3_256_Update(KECCAK1600_CTX *sha, const void *data, size_t len);
+
+// SHA3_256_Final adds the final padding to |sha| and writes the resulting
+// digest to |out|. It returns one on success and zero on programmer error.
+int SHA3_256_Final(uint8_t out[SHA3_256_DIGEST_LENGTH], KECCAK1600_CTX *sha);
+
+// SHA3_384_Init initialises |sha| and returns 1.
+int SHA3_384_Init(KECCAK1600_CTX *sha);
+
+// SHA3_384_Update adds |len| bytes from |data| to |sha| and returns 1.
+int SHA3_384_Update(KECCAK1600_CTX *sha, const void *data, size_t len);
+
+// SHA3_384_Final adds the final padding to |sha| and writes the resulting
+// digest to |out|. It returns one on success and zero on programmer error.
+int SHA3_384_Final(uint8_t out[SHA3_384_DIGEST_LENGTH], KECCAK1600_CTX *sha);
+
+// SHA3_512_Init initialises |sha| and returns 1.
+int SHA3_512_Init(KECCAK1600_CTX *sha);
+
+// SHA3_512_Update adds |len| bytes from |data| to |sha| and returns 1.
+int SHA3_512_Update(KECCAK1600_CTX *sha, const void *data, size_t len);
+
+// SHA3_512_Final adds the final padding to |sha| and writes the resulting
+// digest to |out|. It returns one on success and zero on programmer error.
+int SHA3_512_Final(uint8_t out[SHA3_512_DIGEST_LENGTH], KECCAK1600_CTX *sha);
+
 /*
  * SHAKE APIs implement SHAKE functionalities on top of FIPS202 API layer
  *

crypto/fipsmodule/sha/sha3.c

@@ -270,11 +270,67 @@ int SHA3_Final(uint8_t *md, KECCAK1600_CTX *ctx) {
 
   Keccak1600_Squeeze(ctx->A, md, ctx->md_size, ctx->block_size, ctx->state);
   ctx->state = KECCAK1600_STATE_FINAL;
-  
+
   FIPS_service_indicator_update_state();
   return 1;
 }
 
+int SHA3_224_Init(KECCAK1600_CTX *ctx) {
+    return SHA3_Init(ctx, SHA3_224_DIGEST_BITLENGTH);
+}
+
+int SHA3_224_Update(KECCAK1600_CTX *ctx, const void *data,
+                                     size_t len) {
+    return SHA3_Update(ctx, data, len);
+}
+
+int SHA3_224_Final(uint8_t out[SHA3_224_DIGEST_LENGTH],
+                                    KECCAK1600_CTX *ctx) {
+    return SHA3_Final(&out[0], ctx);
+}
+
+int SHA3_256_Init(KECCAK1600_CTX *ctx) {
+    return SHA3_Init(ctx, SHA3_256_DIGEST_BITLENGTH);
+}
+
+int SHA3_256_Update(KECCAK1600_CTX *ctx, const void *data,
+                                     size_t len) {
+    return SHA3_Update(ctx, data, len);
+}
+
+int SHA3_256_Final(uint8_t out[SHA3_256_DIGEST_LENGTH],
+                                    KECCAK1600_CTX *ctx) {
+    return SHA3_Final(&out[0], ctx);
+}
+
+int SHA3_384_Init(KECCAK1600_CTX *ctx) {
+    return SHA3_Init(ctx, SHA3_384_DIGEST_BITLENGTH);
+}
+
+int SHA3_384_Update(KECCAK1600_CTX *ctx, const void *data,
+                                     size_t len) {
+    return SHA3_Update(ctx, data, len);
+}
+
+int SHA3_384_Final(uint8_t out[SHA3_384_DIGEST_LENGTH],
+                                    KECCAK1600_CTX *ctx) {
+    return SHA3_Final(&out[0], ctx);
+}
+
+int SHA3_512_Init(KECCAK1600_CTX *ctx) {
+    return SHA3_Init(ctx, SHA3_512_DIGEST_BITLENGTH);
+}
+
+int SHA3_512_Update(KECCAK1600_CTX *ctx, const void *data,
+                                     size_t len) {
+    return SHA3_Update(ctx, data, len);
+}
+
+int SHA3_512_Final(uint8_t out[SHA3_512_DIGEST_LENGTH],
+                                    KECCAK1600_CTX *ctx) {
+    return SHA3_Final(&out[0], ctx);
+}
+
 /*
  * SHAKE APIs implement SHAKE functionalities on top of FIPS202 API layer
  */