Enhance openssl compatibility. Fixes #676

Patch by: michaelortmann
Fixes: #676

    We were using old functions and if they were not available we defined them to use the new ones. This patch reverses this logic and automagically gets rid of the compiler warning.
    We were printing MD5 and SHA1 fingerprints. This patch changes this, and we ill print SHA-256 and SHA1.
    I also enhanced the error handling of function X509_digest().
    Last but not least, i enhanced variable definition unsigned i -> unsigned int i and other variable definitions.
    Fixed a well hidden bug. eggdrops aclocal.m4 /config.h and tls.c linked and used the wrong functions. Example: hex_to_string is OPENSSL_buf2hexstr and not OPENSSL_hexstr2buf. Eggdrop didn't fail, but there may be consequential errors.
   Addressed Geo's OCD

aclocal.m4

@@ -1616,17 +1616,17 @@ AC_DEFUN([EGG_TLS_DETECT],
         break
       ]])
     fi
-    AC_CHECK_FUNC(hex_to_string, ,
-      AC_CHECK_FUNC(OPENSSL_hexstr2buf,
-          AC_DEFINE([hex_to_string], [OPENSSL_hexstr2buf], [Define this to OPENSSL_hexstr2buf when using OpenSSL 1.1.0+])
+    AC_CHECK_FUNC(OPENSSL_buf2hexstr, ,
+      AC_CHECK_FUNC(hex_to_string,
+          AC_DEFINE([OPENSSL_buf2hexstr], [hex_to_string], [Define this to hex_to_string when using OpenSSL < 1.1.0])
         , [[
           havessllib="no"
           break
       ]])
     )
-    AC_CHECK_FUNC(string_to_hex, ,
-      AC_CHECK_FUNC(OPENSSL_buf2hexstr,
-          AC_DEFINE([string_to_hex], [OPENSSL_buf2hexstr], [Define this to OPENSSL_buf2hexstr when using OpenSSL 1.1.0+])
+    AC_CHECK_FUNC(OPENSSL_hexstr2buf, ,
+      AC_CHECK_FUNC(string_to_hex,
+          AC_DEFINE([OPENSSL_hexstr2buf], [string_to_hex], [Define this to string_to_hex when using OpenSSL < 1.1.0])
         , [[
           havessllib="no"
           break

src/tls.c

@@ -201,8 +201,8 @@ char *ssl_fpconv(char *in, char *out)
   if (!in)
     return NULL;
 
-  if ((md5 = string_to_hex(in, &len))) {
-    fp = hex_to_string(md5, len);
+  if ((md5 = OPENSSL_hexstr2buf(in, &len))) {
+    fp = OPENSSL_buf2hexstr(md5, len);
     if (fp) {
       out = user_realloc(out, strlen(fp) + 1);
       strcpy(out, fp);
@@ -241,7 +241,7 @@ static X509 *ssl_getcert(int sock)
 char *ssl_getfp(int sock)
 {
   char *p;
-  unsigned i;
+  unsigned int i;
   X509 *cert;
   static char fp[64];
   unsigned char md[EVP_MAX_MD_SIZE];
@@ -250,7 +250,7 @@ char *ssl_getfp(int sock)
     return NULL;
   if (!X509_digest(cert, EVP_sha1(), md, &i))
     return NULL;
-  if (!(p = hex_to_string(md, i)))
+  if (!(p = OPENSSL_buf2hexstr(md, i)))
     return NULL;
   strlcpy(fp, p, sizeof fp);
   OPENSSL_free(p);
@@ -500,7 +500,7 @@ static char *ssl_printnum(ASN1_INTEGER *i)
 /* Show the user all relevant information about a certificate: subject,
  * issuer, validity dates and fingerprints.
  */
-static void ssl_showcert(X509 *cert, int loglev)
+static void ssl_showcert(X509 *cert, const int loglev)
 {
   char *buf, *from, *to;
   X509_NAME *name;
@@ -522,19 +522,18 @@ static void ssl_showcert(X509 *cert, int loglev)
     putlog(loglev, "*", "TLS: cannot get issuer name from certificate!");
 
   /* Fingerprints */
-  X509_digest(cert, EVP_md5(), md, &len); /* MD5 hash */
-  if (len <= sizeof(md)) {
-    buf = hex_to_string(md, len);
-    putlog(loglev, "*", "TLS: certificate MD5 Fingerprint: %s", buf);
+  if (X509_digest(cert, EVP_sha1(), md, &len)) {
+    buf = OPENSSL_buf2hexstr(md, len);
+    putlog(loglev, "*", "TLS: certificate SHA1 Fingerprint: %s", buf);
     OPENSSL_free(buf);
   }
-  X509_digest(cert, EVP_sha1(), md, &len); /* SHA-1 hash */
-  if (len <= sizeof(md)) {
-    buf = hex_to_string(md, len);
-    putlog(loglev, "*", "TLS: certificate SHA1 Fingerprint: %s", buf);
+  if (X509_digest(cert, EVP_sha256(), md, &len)) {
+    buf = OPENSSL_buf2hexstr(md, len);
+    putlog(loglev, "*", "TLS: certificate SHA-256 Fingerprint: %s", buf);
     OPENSSL_free(buf);
   }
 
+
   /* Validity time */
   from = ssl_printtime(X509_get_notBefore(cert));
   to = ssl_printtime(X509_get_notAfter(cert));