MDEV-7695 MariaDB - ssl - fips: can not connect with --ssl-cipher=DHE-RSA-AES256-SHA - handshake failure

Change 512bit DH key to 1024bit to meet FIPS requirements

Author: vuvova

vio/viosslfactories.c

@@ -21,33 +21,33 @@
 static my_bool     ssl_algorithms_added    = FALSE;
 static my_bool     ssl_error_strings_loaded= FALSE;
 
-static unsigned char dh512_p[]=
-{
-  0xDA,0x58,0x3C,0x16,0xD9,0x85,0x22,0x89,0xD0,0xE4,0xAF,0x75,
-  0x6F,0x4C,0xCA,0x92,0xDD,0x4B,0xE5,0x33,0xB8,0x04,0xFB,0x0F,
-  0xED,0x94,0xEF,0x9C,0x8A,0x44,0x03,0xED,0x57,0x46,0x50,0xD3,
-  0x69,0x99,0xDB,0x29,0xD7,0x76,0x27,0x6B,0xA2,0xD3,0xD4,0x12,
-  0xE2,0x18,0xF4,0xDD,0x1E,0x08,0x4C,0xF6,0xD8,0x00,0x3E,0x7C,
-  0x47,0x74,0xE8,0x33,
-};
-
-static unsigned char dh512_g[]={
-  0x02,
-};
-
-static DH *get_dh512(void)
+/* the function below was generated with "openssl dhparam -2 -C 1024" */
+static
+DH *get_dh1024()
 {
+  static unsigned char dh1024_p[]={
+    0xEC,0x46,0x7E,0xF9,0x4E,0x10,0x29,0xDC,0x44,0x97,0x71,0xFD,
+    0x71,0xC6,0x9F,0x0D,0xD1,0x09,0xF6,0x58,0x6F,0xAD,0xCA,0xF4,
+    0x37,0xD5,0xC3,0xBD,0xC3,0x9A,0x51,0x66,0x2C,0x58,0xBD,0x02,
+    0xBD,0xBA,0xBA,0xFC,0xE7,0x0E,0x5A,0xE5,0x97,0x81,0xC3,0xF3,
+    0x28,0x2D,0xAD,0x00,0x91,0xEF,0xF8,0xF0,0x5D,0xE9,0xE7,0x18,
+    0xE2,0xAD,0xC4,0x70,0xC5,0x3C,0x12,0x8A,0x80,0x6A,0x9F,0x3B,
+    0x00,0xA2,0x8F,0xA9,0x26,0xB0,0x0E,0x7F,0xED,0xF6,0xC2,0x03,
+    0x81,0xB5,0xC5,0x41,0xD0,0x00,0x2B,0x21,0xD4,0x4B,0x74,0xA6,
+    0xD7,0x1A,0x0E,0x82,0xC8,0xEE,0xD4,0xB1,0x6F,0xB4,0x79,0x01,
+    0x8A,0xF1,0x12,0xD7,0x3C,0xFD,0xCB,0x9B,0xAE,0x1C,0xA9,0x0F,
+    0x3D,0x0F,0xF8,0xD6,0x7D,0xDE,0xD6,0x0B,
+  };
+  static unsigned char dh1024_g[]={
+    0x02,
+  };
   DH *dh;
-  if ((dh=DH_new()))
-  {
-    dh->p=BN_bin2bn(dh512_p,sizeof(dh512_p),NULL);
-    dh->g=BN_bin2bn(dh512_g,sizeof(dh512_g),NULL);
-    if (! dh->p || ! dh->g)
-    {
-      DH_free(dh);
-      dh=0;
-    }
-  }
+
+  if ((dh=DH_new()) == NULL) return(NULL);
+  dh->p=BN_bin2bn(dh1024_p,sizeof(dh1024_p),NULL);
+  dh->g=BN_bin2bn(dh1024_g,sizeof(dh1024_g),NULL);
+  if ((dh->p == NULL) || (dh->g == NULL))
+  { DH_free(dh); return(NULL); }
   return(dh);
 }
 
@@ -259,7 +259,7 @@ new_VioSSLFd(const char *key_file, const char *cert_file,
   }
 
   /* DH stuff */
-  dh=get_dh512();
+  dh=get_dh1024();
   SSL_CTX_set_tmp_dh(ssl_fd->ssl_context, dh);
   DH_free(dh);