Introduce limits to prevent malicious keys being able to

cause a denial of service.  (CVE-2006-2940)
[Steve Henson, Bodo Moeller]

crypto/dh/dh.h

@@ -73,6 +73,10 @@
 #include <openssl/bn.h>
 #endif
 
+#ifndef OPENSSL_DH_MAX_MODULUS_BITS
+# define OPENSSL_DH_MAX_MODULUS_BITS	10000
+#endif
+
 #define DH_FLAG_CACHE_MONT_P     0x01
 #define DH_FLAG_NO_EXP_CONSTTIME 0x02 /* new with 0.9.7h; the built-in DH
                                        * implementation now uses constant time
@@ -239,11 +243,12 @@ void ERR_load_DH_strings(void);
 
 /* Reason codes. */
 #define DH_R_BAD_GENERATOR				 101
-#define DH_R_BN_DECODE_ERROR				 103
+#define DH_R_BN_DECODE_ERROR				 109
 #define DH_R_BN_ERROR					 106
 #define DH_R_DECODE_ERROR				 104
 #define DH_R_INVALID_PUBKEY				 102
 #define DH_R_KEYS_NOT_SET				 108
+#define DH_R_MODULUS_TOO_LARGE				 103
 #define DH_R_NO_PARAMETERS_SET				 107
 #define DH_R_NO_PRIVATE_VALUE				 100
 #define DH_R_PARAMETER_ENCODING_ERROR			 105

crypto/dh/dh_err.c

@@ -95,6 +95,7 @@ static ERR_STRING_DATA DH_str_reasons[]=
 {ERR_REASON(DH_R_DECODE_ERROR)           ,"decode error"},
 {ERR_REASON(DH_R_INVALID_PUBKEY)         ,"invalid public key"},
 {ERR_REASON(DH_R_KEYS_NOT_SET)           ,"keys not set"},
+{ERR_REASON(DH_R_MODULUS_TOO_LARGE)      ,"modulus too large"},
 {ERR_REASON(DH_R_NO_PARAMETERS_SET)      ,"no parameters set"},
 {ERR_REASON(DH_R_NO_PRIVATE_VALUE)       ,"no private value"},
 {ERR_REASON(DH_R_PARAMETER_ENCODING_ERROR),"parameter encoding error"},

crypto/dh/dh_key.c

@@ -179,6 +179,12 @@ static int compute_key(unsigned char *key, const BIGNUM *pub_key, DH *dh)
 	int ret= -1;
         int check_result;
 
+	if (BN_num_bits(dh->p) > OPENSSL_DH_MAX_MODULUS_BITS)
+		{
+		DHerr(DH_F_COMPUTE_KEY,DH_R_MODULUS_TOO_LARGE);
+		goto err;
+		}
+
 	ctx = BN_CTX_new();
 	if (ctx == NULL) goto err;
 	BN_CTX_start(ctx);

crypto/dsa/dsa.h

@@ -84,6 +84,10 @@
 #endif
 #endif
 
+#ifndef OPENSSL_DSA_MAX_MODULUS_BITS
+# define OPENSSL_DSA_MAX_MODULUS_BITS	10000
+#endif
+
 #define DSA_FLAG_CACHE_MONT_P	0x01
 #define DSA_FLAG_NO_EXP_CONSTTIME       0x02 /* new with 0.9.7h; the built-in DSA
                                               * implementation now uses constant time
@@ -284,12 +288,14 @@ void ERR_load_DSA_strings(void);
 #define DSA_F_SIG_CB					 114
 
 /* Reason codes. */
-#define DSA_R_BN_DECODE_ERROR				 102
-#define DSA_R_BN_ERROR					 103
+#define DSA_R_BAD_Q_VALUE				 102
+#define DSA_R_BN_DECODE_ERROR				 108
+#define DSA_R_BN_ERROR					 109
 #define DSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE		 100
 #define DSA_R_DECODE_ERROR				 104
 #define DSA_R_INVALID_DIGEST_TYPE			 106
 #define DSA_R_MISSING_PARAMETERS			 101
+#define DSA_R_MODULUS_TOO_LARGE				 103
 #define DSA_R_NO_PARAMETERS_SET				 107
 #define DSA_R_PARAMETER_ENCODING_ERROR			 105
 

crypto/dsa/dsa_err.c

@@ -97,12 +97,14 @@ static ERR_STRING_DATA DSA_str_functs[]=
 
 static ERR_STRING_DATA DSA_str_reasons[]=
 	{
+{ERR_REASON(DSA_R_BAD_Q_VALUE)           ,"bad q value"},
 {ERR_REASON(DSA_R_BN_DECODE_ERROR)       ,"bn decode error"},
 {ERR_REASON(DSA_R_BN_ERROR)              ,"bn error"},
 {ERR_REASON(DSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE),"data too large for key size"},
 {ERR_REASON(DSA_R_DECODE_ERROR)          ,"decode error"},
 {ERR_REASON(DSA_R_INVALID_DIGEST_TYPE)   ,"invalid digest type"},
 {ERR_REASON(DSA_R_MISSING_PARAMETERS)    ,"missing parameters"},
+{ERR_REASON(DSA_R_MODULUS_TOO_LARGE)     ,"modulus too large"},
 {ERR_REASON(DSA_R_NO_PARAMETERS_SET)     ,"no parameters set"},
 {ERR_REASON(DSA_R_PARAMETER_ENCODING_ERROR),"parameter encoding error"},
 {0,NULL}

crypto/dsa/dsa_ossl.c

@@ -303,6 +303,18 @@ static int dsa_do_verify(const unsigned char *dgst, int dgst_len, DSA_SIG *sig,
 		return -1;
 		}
 
+	if (BN_num_bits(dsa->q) != 160)
+		{
+		DSAerr(DSA_F_DSA_DO_VERIFY,DSA_R_BAD_Q_VALUE);
+		return -1;
+		}
+
+	if (BN_num_bits(dsa->p) > OPENSSL_DSA_MAX_MODULUS_BITS)
+		{
+		DSAerr(DSA_F_DSA_DO_VERIFY,DSA_R_MODULUS_TOO_LARGE);
+		return -1;
+		}
+
 	BN_init(&u1);
 	BN_init(&u2);
 	BN_init(&t1);

crypto/ec/ec.h

@@ -96,6 +96,11 @@ extern "C" {
 # endif
 #endif
 
+
+#ifndef OPENSSL_ECC_MAX_FIELD_BITS
+# define OPENSSL_ECC_MAX_FIELD_BITS 661
+#endif
+
 /** Enum for the point conversion form as defined in X9.62 (ECDSA)
  *  for the encoding of a elliptic curve point (x,y) */
 typedef enum {
@@ -1049,22 +1054,25 @@ void ERR_load_EC_strings(void);
 #define EC_R_ASN1_UNKNOWN_FIELD				 116
 #define EC_R_BUFFER_TOO_SMALL				 100
 #define EC_R_D2I_ECPKPARAMETERS_FAILURE			 117
-#define EC_R_DECODE_ERROR				 137
+#define EC_R_DECODE_ERROR				 142
 #define EC_R_DISCRIMINANT_IS_ZERO			 118
 #define EC_R_EC_GROUP_NEW_BY_NAME_FAILURE		 119
+#define EC_R_FIELD_TOO_LARGE				 138
 #define EC_R_GROUP2PKPARAMETERS_FAILURE			 120
 #define EC_R_I2D_ECPKPARAMETERS_FAILURE			 121
 #define EC_R_INCOMPATIBLE_OBJECTS			 101
 #define EC_R_INVALID_ARGUMENT				 112
 #define EC_R_INVALID_COMPRESSED_POINT			 110
 #define EC_R_INVALID_COMPRESSION_BIT			 109
-#define EC_R_INVALID_CURVE				 132
+#define EC_R_INVALID_CURVE				 141
 #define EC_R_INVALID_DIGEST_TYPE			 138
 #define EC_R_INVALID_ENCODING				 102
 #define EC_R_INVALID_FIELD				 103
 #define EC_R_INVALID_FORM				 104
 #define EC_R_INVALID_GROUP_ORDER			 122
+#define EC_R_INVALID_PENTANOMIAL_BASIS			 132
 #define EC_R_INVALID_PRIVATE_KEY			 123
+#define EC_R_INVALID_TRINOMIAL_BASIS			 137
 #define EC_R_KEYS_NOT_SET				 140
 #define EC_R_MISSING_PARAMETERS				 124
 #define EC_R_MISSING_PRIVATE_KEY			 125

crypto/ec/ec_asn1.c

@@ -741,6 +741,7 @@ static EC_GROUP *ec_asn1_parameters2group(const ECPARAMETERS *params)
 	EC_GROUP		*ret = NULL;
 	BIGNUM			*p = NULL, *a = NULL, *b = NULL;
 	EC_POINT		*point=NULL;
+	long    		field_bits;
 
 	if (!params->fieldID || !params->fieldID->fieldType || 
 	    !params->fieldID->p.ptr)
@@ -779,6 +780,13 @@ static EC_GROUP *ec_asn1_parameters2group(const ECPARAMETERS *params)
 
 		char_two = params->fieldID->p.char_two;
 
+		field_bits = char_two->m;
+		if (field_bits > OPENSSL_ECC_MAX_FIELD_BITS)
+			{
+			ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_FIELD_TOO_LARGE);
+			goto err;
+			}
+
 		if ((p = BN_new()) == NULL)
 			{
 			ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, ERR_R_MALLOC_FAILURE);
@@ -799,6 +807,13 @@ static EC_GROUP *ec_asn1_parameters2group(const ECPARAMETERS *params)
 				}
 
 			tmp_long = ASN1_INTEGER_get(char_two->p.tpBasis);
+
+			if (!(char_two->m > tmp_long && tmp_long > 0))
+				{
+				ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_INVALID_TRINOMIAL_BASIS);
+				goto err;
+				}
+
 			/* create the polynomial */
 			if (!BN_set_bit(p, (int)char_two->m))
 				goto err;
@@ -817,6 +832,13 @@ static EC_GROUP *ec_asn1_parameters2group(const ECPARAMETERS *params)
 				ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_ASN1_ERROR);
 				goto err;
 				}
+
+			if (!(char_two->m > penta->k3 && penta->k3 > penta->k2 && penta->k2 > penta->k1 && penta->k1 > 0))
+				{
+				ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_INVALID_PENTANOMIAL_BASIS);
+				goto err;
+				}
+
 			/* create the polynomial */
 			if (!BN_set_bit(p, (int)char_two->m)) goto err;
 			if (!BN_set_bit(p, (int)penta->k1)) goto err;
@@ -853,6 +875,20 @@ static EC_GROUP *ec_asn1_parameters2group(const ECPARAMETERS *params)
 			ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, ERR_R_ASN1_LIB);
 			goto err;
 			}
+
+		if (BN_is_negative(p) || BN_is_zero(p))
+			{
+			ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_INVALID_FIELD);
+			goto err;
+			}
+
+		field_bits = BN_num_bits(p);
+		if (field_bits > OPENSSL_ECC_MAX_FIELD_BITS)
+			{
+			ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_FIELD_TOO_LARGE);
+			goto err;
+			}
+
 		/* create the EC_GROUP structure */
 		ret = EC_GROUP_new_curve_GFp(p, a, b, NULL);
 		}
@@ -910,6 +946,16 @@ static EC_GROUP *ec_asn1_parameters2group(const ECPARAMETERS *params)
 		ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, ERR_R_ASN1_LIB);
 		goto err;
 		}
+	if (BN_is_negative(a) || BN_is_zero(a))
+		{
+		ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_INVALID_GROUP_ORDER);
+		goto err;
+		}
+	if (BN_num_bits(a) > (int)field_bits + 1) /* Hasse bound */
+		{
+		ECerr(EC_F_EC_ASN1_PARAMETERS2GROUP, EC_R_INVALID_GROUP_ORDER);
+		goto err;
+		}
 
 	/* extract the cofactor (optional) */
 	if (params->cofactor == NULL)

crypto/ec/ec_err.c

@@ -204,6 +204,7 @@ static ERR_STRING_DATA EC_str_reasons[]=
 {ERR_REASON(EC_R_DECODE_ERROR)           ,"decode error"},
 {ERR_REASON(EC_R_DISCRIMINANT_IS_ZERO)   ,"discriminant is zero"},
 {ERR_REASON(EC_R_EC_GROUP_NEW_BY_NAME_FAILURE),"ec group new by name failure"},
+{ERR_REASON(EC_R_FIELD_TOO_LARGE)        ,"field too large"},
 {ERR_REASON(EC_R_GROUP2PKPARAMETERS_FAILURE),"group2pkparameters failure"},
 {ERR_REASON(EC_R_I2D_ECPKPARAMETERS_FAILURE),"i2d ecpkparameters failure"},
 {ERR_REASON(EC_R_INCOMPATIBLE_OBJECTS)   ,"incompatible objects"},
@@ -216,7 +217,9 @@ static ERR_STRING_DATA EC_str_reasons[]=
 {ERR_REASON(EC_R_INVALID_FIELD)          ,"invalid field"},
 {ERR_REASON(EC_R_INVALID_FORM)           ,"invalid form"},
 {ERR_REASON(EC_R_INVALID_GROUP_ORDER)    ,"invalid group order"},
+{ERR_REASON(EC_R_INVALID_PENTANOMIAL_BASIS),"invalid pentanomial basis"},
 {ERR_REASON(EC_R_INVALID_PRIVATE_KEY)    ,"invalid private key"},
+{ERR_REASON(EC_R_INVALID_TRINOMIAL_BASIS),"invalid trinomial basis"},
 {ERR_REASON(EC_R_KEYS_NOT_SET)           ,"keys not set"},
 {ERR_REASON(EC_R_MISSING_PARAMETERS)     ,"missing parameters"},
 {ERR_REASON(EC_R_MISSING_PRIVATE_KEY)    ,"missing private key"},

crypto/rsa/rsa.h

@@ -160,6 +160,17 @@ struct rsa_st
 	BN_BLINDING *mt_blinding;
 	};
 
+#ifndef OPENSSL_RSA_MAX_MODULUS_BITS
+# define OPENSSL_RSA_MAX_MODULUS_BITS	16384
+#endif
+
+#ifndef OPENSSL_RSA_SMALL_MODULUS_BITS
+# define OPENSSL_RSA_SMALL_MODULUS_BITS	3072
+#endif
+#ifndef OPENSSL_RSA_MAX_PUBEXP_BITS
+# define OPENSSL_RSA_MAX_PUBEXP_BITS	64 /* exponent limit enforced for "large" modulus only */
+#endif
+
 #define RSA_3	0x3L
 #define RSA_F4	0x10001L
 
@@ -452,6 +463,7 @@ void ERR_load_RSA_strings(void);
 #define RSA_R_IQMP_NOT_INVERSE_OF_Q			 126
 #define RSA_R_KEY_SIZE_TOO_SMALL			 120
 #define RSA_R_LAST_OCTET_INVALID			 134
+#define RSA_R_MODULUS_TOO_LARGE				 105
 #define RSA_R_NO_PUBLIC_EXPONENT			 140
 #define RSA_R_NULL_BEFORE_BLOCK_MISSING			 113
 #define RSA_R_N_DOES_NOT_EQUAL_P_Q			 127

crypto/rsa/rsa_eay.c

@@ -168,6 +168,28 @@ static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE);
+		return -1;
+		}
+
+	if (BN_ucmp(rsa->n, rsa->e) <= 0)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
+		return -1;
+		}
+
+	/* for large moduli, enforce exponent limit */
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
+		{
+		if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
+			{
+			RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
+			return -1;
+			}
+		}
+
 	if ((ctx=BN_CTX_new()) == NULL) goto err;
 	BN_CTX_start(ctx);
 	f = BN_CTX_get(ctx);
@@ -597,6 +619,28 @@ static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_MODULUS_TOO_LARGE);
+		return -1;
+		}
+
+	if (BN_ucmp(rsa->n, rsa->e) <= 0)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
+		return -1;
+		}
+
+	/* for large moduli, enforce exponent limit */
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
+		{
+		if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
+			{
+			RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
+			return -1;
+			}
+		}
+
 	if((ctx = BN_CTX_new()) == NULL) goto err;
 	BN_CTX_start(ctx);
 	f = BN_CTX_get(ctx);

crypto/rsa/rsa_err.c

@@ -155,6 +155,7 @@ static ERR_STRING_DATA RSA_str_reasons[]=
 {ERR_REASON(RSA_R_IQMP_NOT_INVERSE_OF_Q) ,"iqmp not inverse of q"},
 {ERR_REASON(RSA_R_KEY_SIZE_TOO_SMALL)    ,"key size too small"},
 {ERR_REASON(RSA_R_LAST_OCTET_INVALID)    ,"last octet invalid"},
+{ERR_REASON(RSA_R_MODULUS_TOO_LARGE)     ,"modulus too large"},
 {ERR_REASON(RSA_R_NO_PUBLIC_EXPONENT)    ,"no public exponent"},
 {ERR_REASON(RSA_R_NULL_BEFORE_BLOCK_MISSING),"null before block missing"},
 {ERR_REASON(RSA_R_N_DOES_NOT_EQUAL_P_Q)  ,"n does not equal p q"},