rgw: under fips & openssl 3.x allow md5 iusage in select rgw ops

openssl 3.x (ex:RHEL9) requires a different override mechanism for MD5 usage under FIPS
for non-cryptographic putposes than openssl 1.x (RHEL8)

fixes: https://tracker.ceph.com/issues/58332

Signed-off-by: Mark Kogan <mkogan@redhat.com>

src/common/ceph_crypto.cc

@@ -196,14 +196,25 @@ ssl::OpenSSLDigest::OpenSSLDigest(const EVP_MD * _type)
 
 ssl::OpenSSLDigest::~OpenSSLDigest() {
   EVP_MD_CTX_destroy(mpContext);
+  if (mpType_FIPS) {
+    EVP_MD_free(mpType_FIPS);
+  }
 }
 
 void ssl::OpenSSLDigest::Restart() {
-  EVP_DigestInit_ex(mpContext, mpType, NULL);
+  if (mpType_FIPS) {
+    EVP_DigestInit_ex(mpContext, mpType_FIPS, NULL);
+  } else {
+    EVP_DigestInit_ex(mpContext, mpType, NULL);
+  }
 }
 
 void ssl::OpenSSLDigest::SetFlags(int flags) {
-  EVP_MD_CTX_set_flags(mpContext, flags);
+  if (flags == EVP_MD_CTX_FLAG_NON_FIPS_ALLOW && OpenSSL_version_num() >= 0x30000000L && mpType == EVP_md5() && !mpType_FIPS) {
+    mpType_FIPS = EVP_MD_fetch(NULL, "MD5", "fips=no");
+  } else {
+    EVP_MD_CTX_set_flags(mpContext, flags);
+  }
   this->Restart();
 }
 

src/common/ceph_crypto.h

@@ -54,6 +54,7 @@ namespace TOPNSPC::crypto {
       private:
 	EVP_MD_CTX *mpContext;
 	const EVP_MD *mpType;
+        EVP_MD *mpType_FIPS = nullptr;
       public:
 	OpenSSLDigest (const EVP_MD *_type);
 	~OpenSSLDigest ();