Fix some memory leaks in the openssl app

In some error cases the normal cleanup did not
happen, but instead an exit(1) which caused some
memory leaks, as reported in #22049.

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
(Merged from #22055)

(cherry picked from commit 8c040c0)

apps/dgst.c

@@ -320,6 +320,8 @@ int dgst_main(int argc, char **argv)
         sigkey = app_keygen(mac_ctx, mac_name, 0, 0 /* not verbose */);
         /* Verbose output would make external-tests gost-engine fail */
         EVP_PKEY_CTX_free(mac_ctx);
+        if (sigkey == NULL)
+            goto end;
     }
 
     if (hmac_key != NULL) {

apps/dhparam.c

@@ -222,6 +222,8 @@ int dhparam_main(int argc, char **argv)
         }
 
         tmppkey = app_paramgen(ctx, alg);
+        if (tmppkey == NULL)
+            goto end;
         EVP_PKEY_CTX_free(ctx);
         ctx = NULL;
         if (dsaparam) {

apps/dsaparam.c

@@ -218,6 +218,8 @@ int dsaparam_main(int argc, char **argv)
             goto end;
         }
         pkey = app_keygen(ctx, "DSA", numbits, verbose);
+        if (pkey == NULL)
+            goto end;
         assert(private);
         if (outformat == FORMAT_ASN1)
             i = i2d_PrivateKey_bio(out, pkey);

apps/gendsa.c

@@ -146,6 +146,8 @@ int gendsa_main(int argc, char **argv)
         goto end;
     }
     pkey = app_keygen(ctx, "DSA", nbits, verbose);
+    if (pkey == NULL)
+        goto end;
 
     assert(private);
     if (!PEM_write_bio_PrivateKey(out, pkey, enc, NULL, 0, NULL, passout)) {

apps/genpkey.c

@@ -183,6 +183,8 @@ int genpkey_main(int argc, char **argv)
 
     pkey = do_param ? app_paramgen(ctx, algname)
                     : app_keygen(ctx, algname, 0, 0 /* not verbose */);
+    if (pkey == NULL)
+        goto end;
 
     if (do_param) {
         rv = PEM_write_bio_Parameters(out, pkey);

apps/genrsa.c

@@ -203,6 +203,8 @@ int genrsa_main(int argc, char **argv)
         goto end;
     }
     pkey = app_keygen(ctx, "RSA", num, verbose);
+    if (pkey == NULL)
+        goto end;
 
     if (verbose) {
         BIGNUM *e = NULL;

apps/lib/apps.c

@@ -3351,8 +3351,8 @@ EVP_PKEY *app_keygen(EVP_PKEY_CTX *ctx, const char *alg, int bits, int verbose)
         BIO_printf(bio_err, "Warning: generating random key material may take a long time\n"
                    "if the system has a poor entropy source\n");
     if (EVP_PKEY_keygen(ctx, &res) <= 0)
-        app_bail_out("%s: Error generating %s key\n", opt_getprog(),
-                     alg != NULL ? alg : "asymmetric");
+        BIO_printf(bio_err, "%s: Error generating %s key\n", opt_getprog(),
+                   alg != NULL ? alg : "asymmetric");
     return res;
 }
 
@@ -3364,8 +3364,8 @@ EVP_PKEY *app_paramgen(EVP_PKEY_CTX *ctx, const char *alg)
         BIO_printf(bio_err, "Warning: generating random key parameters may take a long time\n"
                    "if the system has a poor entropy source\n");
     if (EVP_PKEY_paramgen(ctx, &res) <= 0)
-        app_bail_out("%s: Generating %s key parameters failed\n",
-                     opt_getprog(), alg != NULL ? alg : "asymmetric");
+        BIO_printf(bio_err, "%s: Generating %s key parameters failed\n",
+                   opt_getprog(), alg != NULL ? alg : "asymmetric");
     return res;
 }
 

apps/req.c

@@ -685,6 +685,8 @@ int req_main(int argc, char **argv)
         EVP_PKEY_CTX_set_app_data(genctx, bio_err);
 
         pkey = app_keygen(genctx, keyalgstr, newkey_len, verbose);
+        if (pkey == NULL)
+            goto end;
 
         EVP_PKEY_CTX_free(genctx);
         genctx = NULL;