Issue warnings for large DSA and RSA keys

Issue a warning when generating DSA or RSA keys of size greater than
OPENSSL_DSA_MAX_MODULUS_BITS resp. OPENSSL_RSA_MAX_MODULUS_BITS.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matthias St. Pierre <Matthias.St.Pierre@ncp-e.com>
(Merged from openssl#6380)

apps/dsaparam.c

@@ -128,6 +128,12 @@ int dsaparam_main(int argc, char **argv)
         goto end;
 
     if (numbits > 0) {
+        if (numbits > OPENSSL_DSA_MAX_MODULUS_BITS)
+            BIO_printf(bio_err,
+                       "Warning: It is not recommended to use more than %d bit for DSA keys.\n"
+                       "         Your key size is %d! Larger key size may behave not as expected.\n",
+                       OPENSSL_DSA_MAX_MODULUS_BITS, numbits);
+
         cb = BN_GENCB_new();
         if (cb == NULL) {
             BIO_printf(bio_err, "Error allocating BN_GENCB object\n");

apps/gendsa.c

@@ -117,6 +117,13 @@ int gendsa_main(int argc, char **argv)
         goto end2;
 
     DSA_get0_pqg(dsa, &p, NULL, NULL);
+
+    if (BN_num_bits(p) > OPENSSL_DSA_MAX_MODULUS_BITS)
+        BIO_printf(bio_err,
+                   "Warning: It is not recommended to use more than %d bit for DSA keys.\n"
+                   "         Your key size is %d! Larger key size may behave not as expected.\n",
+                   OPENSSL_DSA_MAX_MODULUS_BITS, BN_num_bits(p));
+
     BIO_printf(bio_err, "Generating DSA key, %d bits\n", BN_num_bits(p));
     if (!DSA_generate_key(dsa))
         goto end;

apps/genrsa.c

@@ -123,6 +123,11 @@ int genrsa_main(int argc, char **argv)
     if (argc == 1) {
         if (!opt_int(argv[0], &num) || num <= 0)
             goto end;
+        if (num > OPENSSL_RSA_MAX_MODULUS_BITS)
+            BIO_printf(bio_err,
+                       "Warning: It is not recommended to use more than %d bit for RSA keys.\n"
+                       "         Your key size is %d! Larger key size may behave not as expected.\n",
+                       OPENSSL_RSA_MAX_MODULUS_BITS, num);
     } else if (argc > 0) {
         BIO_printf(bio_err, "Extra arguments given.\n");
         goto opthelp;

apps/req.c

@@ -517,6 +517,18 @@ int req_main(int argc, char **argv)
             goto end;
         }
 
+        if (pkey_type == EVP_PKEY_RSA && newkey > OPENSSL_RSA_MAX_MODULUS_BITS)
+            BIO_printf(bio_err,
+                       "Warning: It is not recommended to use more than %d bit for RSA keys.\n"
+                       "         Your key size is %ld! Larger key size may behave not as expected.\n",
+                       OPENSSL_RSA_MAX_MODULUS_BITS, newkey);
+
+        if (pkey_type == EVP_PKEY_DSA && newkey > OPENSSL_DSA_MAX_MODULUS_BITS)
+            BIO_printf(bio_err,
+                       "Warning: It is not recommended to use more than %d bit for DSA keys.\n"
+                       "         Your key size is %ld! Larger key size may behave not as expected.\n",
+                       OPENSSL_DSA_MAX_MODULUS_BITS, newkey);
+
         if (genctx == NULL) {
             genctx = set_keygen_ctx(NULL, &pkey_type, &newkey,
                                     &keyalgstr, gen_eng);