Fix DH_check() excessive time with over sized modulus

The DH_check() function checks numerous aspects of the key or parameters
that have been supplied. Some of those checks use the supplied modulus
value even if it is excessively large.

There is already a maximum DH modulus size (10,000 bits) over which
OpenSSL will not generate or derive keys. DH_check() will however still
perform various tests for validity on such a large modulus. We introduce a
new maximum (32,768) over which DH_check() will just fail.

An application that calls DH_check() and supplies a key or parameters
obtained from an untrusted source could be vulnerable to a Denial of
Service attack.

The function DH_check() is itself called by a number of other OpenSSL
functions. An application calling any of those other functions may
similarly be affected. The other functions affected by this are
DH_check_ex() and EVP_PKEY_param_check().

CVE-2023-3446

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>

crypto/dh/dh.h

@@ -76,6 +76,9 @@
 # ifndef OPENSSL_DH_MAX_MODULUS_BITS
 #  define OPENSSL_DH_MAX_MODULUS_BITS    10000
 # endif
+# ifndef OPENSSL_DH_CHECK_MAX_MODULUS_BITS
+#  define OPENSSL_DH_CHECK_MAX_MODULUS_BITS  32768
+# endif
 
 # define DH_FLAG_CACHE_MONT_P     0x01
 
@@ -363,6 +366,7 @@ int DH_KDF_X9_42(unsigned char *out, size_t outlen,
  * The following lines are auto generated by the script mkerr.pl. Any changes
  * made after this point may be overwritten when the script is next run.
  */
+
 void ERR_load_DH_strings(void);
 
 /* Error codes for the DH functions. */
@@ -371,6 +375,7 @@ void ERR_load_DH_strings(void);
 # define DH_F_COMPUTE_KEY                                 102
 # define DH_F_DHPARAMS_PRINT_FP                           101
 # define DH_F_DH_BUILTIN_GENPARAMS                        106
+# define DH_F_DH_CHECK                                    126
 # define DH_F_DH_CMS_DECRYPT                              117
 # define DH_F_DH_CMS_SET_PEERKEY                          118
 # define DH_F_DH_CMS_SET_SHARED_INFO                      119
@@ -406,7 +411,7 @@ void ERR_load_DH_strings(void);
 # define DH_R_PEER_KEY_ERROR                              113
 # define DH_R_SHARED_INFO_ERROR                           114
 
-#ifdef  __cplusplus
+# ifdef  __cplusplus
 }
-#endif
+# endif
 #endif

crypto/dh/dh_check.c

@@ -78,6 +78,12 @@ int DH_check(const DH *dh, int *ret)
     BN_ULONG l;
     BIGNUM *t1 = NULL, *t2 = NULL;
 
+    /* Don't do any checks at all with an excessively large modulus */
+    if (BN_num_bits(dh->p) > OPENSSL_DH_CHECK_MAX_MODULUS_BITS) {
+        DHerr(DH_F_DH_CHECK, DH_R_MODULUS_TOO_LARGE);
+        return 0;
+    }
+
     *ret = 0;
     ctx = BN_CTX_new();
     if (ctx == NULL)

crypto/dh/dh_err.c

@@ -1,6 +1,6 @@
 /* crypto/dh/dh_err.c */
 /* ====================================================================
- * Copyright (c) 1999-2013 The OpenSSL Project.  All rights reserved.
+ * Copyright (c) 1999-2023 The OpenSSL Project.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -73,6 +73,7 @@ static ERR_STRING_DATA DH_str_functs[] = {
     {ERR_FUNC(DH_F_COMPUTE_KEY), "COMPUTE_KEY"},
     {ERR_FUNC(DH_F_DHPARAMS_PRINT_FP), "DHparams_print_fp"},
     {ERR_FUNC(DH_F_DH_BUILTIN_GENPARAMS), "DH_BUILTIN_GENPARAMS"},
+    {ERR_FUNC(DH_F_DH_CHECK), "DH_check"},
     {ERR_FUNC(DH_F_DH_CMS_DECRYPT), "DH_CMS_DECRYPT"},
     {ERR_FUNC(DH_F_DH_CMS_SET_PEERKEY), "DH_CMS_SET_PEERKEY"},
     {ERR_FUNC(DH_F_DH_CMS_SET_SHARED_INFO), "DH_CMS_SET_SHARED_INFO"},