MDEV-8482 mysql-test - main.func_encrypt fails if FIPS=1

* check for openssl errors in DES_ENCRYPT/DES_DECRYPT * disable the test when DES doesn't work * also disable main.func_des_encrypt
MariaDB · Apr 24, 2016 · 797cadc · 797cadc
1 parent 906f97d
commit 797cadc
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 6 deletions.
diff --git a/mysql-test/include/have_des.inc b/mysql-test/include/have_des.inc
@@ -0,0 +1,6 @@
+# in the FIPS mode, OpenSSL disables DES and other weak algorithms
+source include/have_ssl_crypto_functs.inc;
+
+if (`select des_encrypt("a", "b") IS NULL`) {
+  skip DES is disabled (fips mode?);
+}
diff --git a/mysql-test/t/func_des_encrypt.test b/mysql-test/t/func_des_encrypt.test
@@ -1,4 +1,4 @@
--- source include/have_ssl_crypto_functs.inc
+-- source include/have_des.inc
 
 # This test can't be in func_encrypt.test, because it requires
 # --des-key-file to not be set.

diff --git a/mysql-test/t/func_encrypt.test b/mysql-test/t/func_encrypt.test
@@ -1,4 +1,4 @@
--- source include/have_ssl_crypto_functs.inc
+-- source include/have_des.inc
 
 --disable_warnings
 drop table if exists t1;

diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
@@ -828,9 +828,10 @@ String *Item_func_des_encrypt::val_str(String *str)
 
     /* We make good 24-byte (168 bit) key from given plaintext key with MD5 */
     bzero((char*) &ivec,sizeof(ivec));
-    EVP_BytesToKey(EVP_des_ede3_cbc(),EVP_md5(),NULL,
+    if (!EVP_BytesToKey(EVP_des_ede3_cbc(),EVP_md5(),NULL,
 		   (uchar*) keystr->ptr(), (int) keystr->length(),
-		   1, (uchar*) &keyblock,ivec);
+		   1, (uchar*) &keyblock,ivec))
+      goto error;
     DES_set_key_unchecked(&keyblock.key1,&keyschedule.ks1);
     DES_set_key_unchecked(&keyblock.key2,&keyschedule.ks2);
     DES_set_key_unchecked(&keyblock.key3,&keyschedule.ks3);
@@ -921,9 +922,10 @@ String *Item_func_des_decrypt::val_str(String *str)
       goto error;
 
     bzero((char*) &ivec,sizeof(ivec));
-    EVP_BytesToKey(EVP_des_ede3_cbc(),EVP_md5(),NULL,
+    if (!EVP_BytesToKey(EVP_des_ede3_cbc(),EVP_md5(),NULL,
 		   (uchar*) keystr->ptr(),(int) keystr->length(),
-		   1,(uchar*) &keyblock,ivec);
+		   1,(uchar*) &keyblock,ivec))
+      goto error;
     // Here we set all 64-bit keys (56 effective) one by one
     DES_set_key_unchecked(&keyblock.key1,&keyschedule.ks1);
     DES_set_key_unchecked(&keyblock.key2,&keyschedule.ks2);