cleanup: check_openssl_compatibility()

CRYPTO_set_mem_functions() works only until the first allocation is done:
* remove the second CRYPTO_set_mem_functions() call
* check whether the first CRYPTO_set_mem_functions() call worked
* stricter memory checks (==1, not >1, etc)
* as coc_malloc cannot be removed, make the counter a bit cheaper
* only do the check for OpenSSL 1.1 (because of OpenSSL 1.0 bug)

mysys_ssl/openssl.c

@@ -17,32 +17,30 @@
 #include <my_global.h>
 #include <ssl_compat.h>
 
-#ifdef HAVE_YASSL
+/*
+  The check is only done for OpenSSL 1.1.x.
+  It could run for OpenSSL 1.0.x but it doesn't make much sense
+  and it hits this bug:
+  https://bugs.launchpad.net/ubuntu/+source/openssl/+bug/1594748
+*/
 
+#ifndef HAVE_OPENSSL11
 int check_openssl_compatibility()
 {
   return 0;
 }
 #else
 #include <openssl/evp.h>
 
-#ifdef HAVE_OPENSSL11
-typedef void *(*CRYPTO_malloc_t)(size_t, const char *, int);
-#endif
-
-#ifdef HAVE_OPENSSL10
-typedef void *(*CRYPTO_malloc_t)(size_t);
-#define CRYPTO_malloc   malloc
-#define CRYPTO_realloc  realloc
-#define CRYPTO_free     free
-#endif
-
-static uint allocated_size, allocated_count;
+static uint testing, alloc_size, alloc_count;
 
-static void *coc_malloc(size_t size)
+static void *coc_malloc(size_t size, const char *, int)
 {
-  allocated_size+= size;
-  allocated_count++;
+  if (unlikely(testing))
+  {
+    alloc_size+= size;
+    alloc_count++;
+  }
   return malloc(size);
 }
 
@@ -51,21 +49,23 @@ int check_openssl_compatibility()
   EVP_CIPHER_CTX *evp_ctx;
   EVP_MD_CTX     *md5_ctx;
 
-  CRYPTO_set_mem_functions((CRYPTO_malloc_t)coc_malloc, CRYPTO_realloc, CRYPTO_free);
+  if (!CRYPTO_set_mem_functions(coc_malloc, CRYPTO_realloc, CRYPTO_free))
+    return 1;
 
-  allocated_size= allocated_count= 0;
+  testing= 1;
+  alloc_size= alloc_count= 0;
   evp_ctx= EVP_CIPHER_CTX_new();
   EVP_CIPHER_CTX_free(evp_ctx);
-  if (allocated_count > 1 || allocated_size > EVP_CIPHER_CTX_SIZE)
+  if (alloc_count != 1 || !alloc_size || alloc_size > EVP_CIPHER_CTX_SIZE)
     return 1;
 
-  allocated_size= allocated_count= 0;
+  alloc_size= alloc_count= 0;
   md5_ctx= EVP_MD_CTX_create();
   EVP_MD_CTX_destroy(md5_ctx);
-  if (allocated_count > 1 || allocated_size > EVP_MD_CTX_SIZE)
+  if (alloc_count != 1 || !alloc_size || alloc_size > EVP_MD_CTX_SIZE)
     return 1;
 
-  CRYPTO_set_mem_functions(CRYPTO_malloc, CRYPTO_realloc, CRYPTO_free);
+  testing= 0;
   return 0;
 }
 #endif