digest: remove OpenSSL engine

The OpenSSL engine of Digest uses the low-level API of OpenSSL, whose
use has been discouraged for years for multiple reasons.

A long-standing issue on a FIPS-enabled system is that using ::Digest
results in crashing the Ruby process, because the low-level API lacks
the mechanism to report an error (the policy violation) and thus kills
the process as a last resort[1][2]. Also, the upcoming OpenSSL 3.0 will
deprecate it for future removal[3]. Compiling with
-Wdeprecated-declarations will start to emit warnings.

A proper fix for this is to make it use the EVP API instead. This is a
non-trivial work as it requires backwards-incompatible changes to the
framework interface of Digest::Base and rb_digest_metadata_t.

It is more than 15 years ago that the openssl library became part of the
standard library. It has implemented the exactly same functionality as
OpenSSL::Digest, in fact, as a subclass of Digest::Class. There is not
much point in having an identical code in the digest library. Let's
just get rid of OpenSSL within digest. This leaves the C implementations
and the CommonCrypto engine for Apple systems.

A patch is being prepared for the openssl library to provide ::Digest
constants for better performance[4].

[1] https://bugs.ruby-lang.org/issues/6946
[2] https://bugs.ruby-lang.org/issues/13681
[3] https://www.openssl.org/docs/OpenSSL300Design.html
[4] ruby/openssl#377

ext/digest/digest_conf.rb

@@ -1,57 +1,6 @@
 # frozen_string_literal: false
 
-# Copy from ext/openssl/extconf.rb
-def find_openssl_library
-  if $mswin || $mingw
-    # required for static OpenSSL libraries
-    have_library("gdi32") # OpenSSL <= 1.0.2 (for RAND_screen())
-    have_library("crypt32")
-  end
-
-  return false unless have_header("openssl/ssl.h")
-
-  ret = have_library("crypto", "CRYPTO_malloc") &&
-    have_library("ssl", "SSL_new")
-  return ret if ret
-
-  if $mswin
-    # OpenSSL >= 1.1.0: libcrypto.lib and libssl.lib.
-    if have_library("libcrypto", "CRYPTO_malloc") &&
-        have_library("libssl", "SSL_new")
-      return true
-    end
-
-    # OpenSSL <= 1.0.2: libeay32.lib and ssleay32.lib.
-    if have_library("libeay32", "CRYPTO_malloc") &&
-        have_library("ssleay32", "SSL_new")
-      return true
-    end
-
-    # LibreSSL: libcrypto-##.lib and libssl-##.lib, where ## is the ABI version
-    # number. We have to find the version number out by scanning libpath.
-    libpath = $LIBPATH.dup
-    libpath |= ENV["LIB"].split(File::PATH_SEPARATOR)
-    libpath.map! { |d| d.tr(File::ALT_SEPARATOR, File::SEPARATOR) }
-
-    ret = [
-      ["crypto", "CRYPTO_malloc"],
-      ["ssl", "SSL_new"]
-    ].all? do |base, func|
-      result = false
-      libs = ["lib#{base}-[0-9][0-9]", "lib#{base}-[0-9][0-9][0-9]"]
-      libs = Dir.glob(libs.map{|l| libpath.map{|d| File.join(d, l + ".*")}}.flatten).map{|path| File.basename(path, ".*")}.uniq
-      libs.each do |lib|
-        result = have_library(lib, func)
-        break if result
-      end
-      result
-    end
-    return ret if ret
-  end
-  return false
-end
-
-def digest_conf(name, hdr = name, funcs = nil, types = nil)
+def digest_conf(name)
   unless with_config("bundled-#{name}")
     cc = with_config("common-digest")
     if cc == true or /\b#{name}\b/ =~ cc
@@ -62,21 +11,6 @@ def digest_conf(name, hdr = name, funcs = nil, types = nil)
         return :commondigest
       end
     end
-
-    dir_config("openssl")
-    pkg_config("openssl")
-    if find_openssl_library
-      funcs ||= name.upcase
-      funcs = Array(funcs)
-      types ||= funcs
-      hdr = "openssl/#{hdr}.h"
-      if funcs.all? {|func| have_func("#{func}_Transform", hdr)} &&
-         types.all? {|type| have_type("#{type}_CTX", hdr)}
-        $defs << "-D#{name.upcase}_USE_OPENSSL"
-        $headers << "#{name}ossl.h"
-        return :ossl
-      end
-    end
   end
   $objs << "#{name}.#{$OBJEXT}"
   return

ext/digest/md5/depend

@@ -329,5 +329,4 @@ md5init.o: $(srcdir)/../defs.h
 md5init.o: $(srcdir)/../digest.h
 md5init.o: md5.h
 md5init.o: md5init.c
-md5init.o: md5ossl.h
 # AUTOGENERATED DEPENDENCIES END

ext/digest/md5/md5init.c

@@ -3,9 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(MD5_USE_OPENSSL)
-#include "md5ossl.h"
-#elif defined(MD5_USE_COMMONDIGEST)
+#if defined(MD5_USE_COMMONDIGEST)
 #include "md5cc.h"
 #else
 #include "md5.h"

ext/digest/rmd160/depend

@@ -329,5 +329,4 @@ rmd160init.o: $(srcdir)/../defs.h
 rmd160init.o: $(srcdir)/../digest.h
 rmd160init.o: rmd160.h
 rmd160init.o: rmd160init.c
-rmd160init.o: rmd160ossl.h
 # AUTOGENERATED DEPENDENCIES END

ext/digest/rmd160/extconf.rb

@@ -10,7 +10,7 @@
 
 $objs = [ "rmd160init.#{$OBJEXT}" ]
 
-digest_conf("rmd160", "ripemd", "RIPEMD160")
+digest_conf("rmd160")
 
 have_header("sys/cdefs.h")
 

ext/digest/rmd160/rmd160init.c

@@ -3,11 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(RMD160_USE_OPENSSL)
-#include "rmd160ossl.h"
-#else
 #include "rmd160.h"
-#endif
 
 static const rb_digest_metadata_t rmd160 = {
     RUBY_DIGEST_API_VERSION,

ext/digest/sha1/depend

@@ -329,5 +329,4 @@ sha1init.o: $(srcdir)/../defs.h
 sha1init.o: $(srcdir)/../digest.h
 sha1init.o: sha1.h
 sha1init.o: sha1init.c
-sha1init.o: sha1ossl.h
 # AUTOGENERATED DEPENDENCIES END

ext/digest/sha1/extconf.rb

@@ -10,7 +10,7 @@
 
 $objs = [ "sha1init.#{$OBJEXT}" ]
 
-digest_conf("sha1", "sha", nil, %w[SHA])
+digest_conf("sha1")
 
 have_header("sys/cdefs.h")
 

ext/digest/sha1/sha1init.c

@@ -3,9 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(SHA1_USE_OPENSSL)
-#include "sha1ossl.h"
-#elif defined(SHA1_USE_COMMONDIGEST)
+#if defined(SHA1_USE_COMMONDIGEST)
 #include "sha1cc.h"
 #else
 #include "sha1.h"

ext/digest/sha2/depend

@@ -328,5 +328,4 @@ sha2init.o: $(hdrdir)/ruby/subst.h
 sha2init.o: $(srcdir)/../digest.h
 sha2init.o: sha2.h
 sha2init.o: sha2init.c
-sha2init.o: sha2ossl.h
 # AUTOGENERATED DEPENDENCIES END

ext/digest/sha2/extconf.rb

@@ -10,7 +10,7 @@
 
 $objs = [ "sha2init.#{$OBJEXT}" ]
 
-unless digest_conf("sha2", "sha", %w[SHA256 SHA512])
+unless digest_conf("sha2")
   have_type("u_int8_t")
 end
 

ext/digest/sha2/sha2init.c

@@ -3,9 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(SHA2_USE_OPENSSL)
-#include "sha2ossl.h"
-#elif defined(SHA2_USE_COMMONDIGEST)
+#if defined(SHA2_USE_COMMONDIGEST)
 #include "sha2cc.h"
 #else
 #include "sha2.h"