Limit RSA-OAEP related functions to RSA keys only

Make EVP_PKEY_CTX_set_rsa_oaep_md() and
EVP_PKEY_CTX_get_rsa_oaep_md_name() only work for RSA keys.

Since these calls use "digest" as a OSSL_PARAM, they should not
work for other key types.

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from #20319)

(cherry picked from commit 0c3eb31)

crypto/rsa/rsa_lib.c

@@ -1001,6 +1001,10 @@ int EVP_PKEY_CTX_set_rsa_pss_keygen_md_name(EVP_PKEY_CTX *ctx,
  */
 int EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX *ctx, const EVP_MD *md)
 {
+    /* If key type not RSA return error */
+    if (!EVP_PKEY_CTX_is_a(ctx, "RSA"))
+        return -1;
+
     return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT,
                              EVP_PKEY_CTRL_RSA_OAEP_MD, 0, (void *)(md));
 }
@@ -1028,6 +1032,10 @@ int EVP_PKEY_CTX_get_rsa_oaep_md_name(EVP_PKEY_CTX *ctx, char *name,
  */
 int EVP_PKEY_CTX_get_rsa_oaep_md(EVP_PKEY_CTX *ctx, const EVP_MD **md)
 {
+    /* If key type not RSA return error */
+    if (!EVP_PKEY_CTX_is_a(ctx, "RSA"))
+        return -1;
+
     return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT,
                              EVP_PKEY_CTRL_GET_RSA_OAEP_MD, 0, (void *)md);
 }