From 2e601c284c9b61c286aa031d91e5198c17b44f00 Mon Sep 17 00:00:00 2001
From: Kazuki Yamaguchi <k@rhe.jp>
Date: Thu, 28 May 2020 00:53:41 +0900
Subject: [PATCH] digest: remove OpenSSL engine

The OpenSSL engine of Digest uses the low-level API of OpenSSL, whose
use has been discouraged for years for multiple reasons.

A long-standing issue on a FIPS-enabled system is that using ::Digest
results in crashing the Ruby process, because the low-level API lacks
the mechanism to report an error (the policy violation) and thus kills
the process as a last resort[1][2]. Also, the upcoming OpenSSL 3.0 will
deprecate it for future removal[3]. Compiling with
-Wdeprecated-declarations will start to emit warnings.

A proper fix for this is to make it use the EVP API instead. This is a
non-trivial work as it requires backwards-incompatible changes to the
framework interface of Digest::Base and rb_digest_metadata_t.

It is more than 15 years ago that the openssl library became part of the
standard library. It has implemented the exactly same functionality as
OpenSSL::Digest, in fact, as a subclass of Digest::Class. There is not
much point in having an identical code in the digest library. Let's
just get rid of OpenSSL within digest. This leaves the C implementations
and the CommonCrypto engine for Apple systems.

A patch is being prepared for the openssl library to provide ::Digest
constants for better performance[4].

[1] https://bugs.ruby-lang.org/issues/6946
[2] https://bugs.ruby-lang.org/issues/13681
[3] https://www.openssl.org/docs/OpenSSL300Design.html
[4] https://github.com/ruby/openssl/pull/377
---
 ext/digest/digest_conf.rb      | 68 +---------------------------------
 ext/digest/md5/depend          |  1 -
 ext/digest/md5/md5init.c       |  4 +-
 ext/digest/md5/md5ossl.h       | 15 --------
 ext/digest/rmd160/depend       |  1 -
 ext/digest/rmd160/extconf.rb   |  2 +-
 ext/digest/rmd160/rmd160init.c |  4 --
 ext/digest/rmd160/rmd160ossl.h | 20 ----------
 ext/digest/sha1/depend         |  1 -
 ext/digest/sha1/extconf.rb     |  2 +-
 ext/digest/sha1/sha1init.c     |  4 +-
 ext/digest/sha1/sha1ossl.h     | 22 -----------
 ext/digest/sha2/depend         |  1 -
 ext/digest/sha2/extconf.rb     |  2 +-
 ext/digest/sha2/sha2init.c     |  4 +-
 ext/digest/sha2/sha2ossl.h     | 27 --------------
 16 files changed, 7 insertions(+), 171 deletions(-)
 delete mode 100644 ext/digest/md5/md5ossl.h
 delete mode 100644 ext/digest/rmd160/rmd160ossl.h
 delete mode 100644 ext/digest/sha1/sha1ossl.h
 delete mode 100644 ext/digest/sha2/sha2ossl.h

diff --git a/ext/digest/digest_conf.rb b/ext/digest/digest_conf.rb
index 1af401a06b0c7a..1b929d87322577 100644
--- a/ext/digest/digest_conf.rb
+++ b/ext/digest/digest_conf.rb
@@ -1,57 +1,6 @@
 # frozen_string_literal: false
 
-# Copy from ext/openssl/extconf.rb
-def find_openssl_library
-  if $mswin || $mingw
-    # required for static OpenSSL libraries
-    have_library("gdi32") # OpenSSL <= 1.0.2 (for RAND_screen())
-    have_library("crypt32")
-  end
-
-  return false unless have_header("openssl/ssl.h")
-
-  ret = have_library("crypto", "CRYPTO_malloc") &&
-    have_library("ssl", "SSL_new")
-  return ret if ret
-
-  if $mswin
-    # OpenSSL >= 1.1.0: libcrypto.lib and libssl.lib.
-    if have_library("libcrypto", "CRYPTO_malloc") &&
-        have_library("libssl", "SSL_new")
-      return true
-    end
-
-    # OpenSSL <= 1.0.2: libeay32.lib and ssleay32.lib.
-    if have_library("libeay32", "CRYPTO_malloc") &&
-        have_library("ssleay32", "SSL_new")
-      return true
-    end
-
-    # LibreSSL: libcrypto-##.lib and libssl-##.lib, where ## is the ABI version
-    # number. We have to find the version number out by scanning libpath.
-    libpath = $LIBPATH.dup
-    libpath |= ENV["LIB"].split(File::PATH_SEPARATOR)
-    libpath.map! { |d| d.tr(File::ALT_SEPARATOR, File::SEPARATOR) }
-
-    ret = [
-      ["crypto", "CRYPTO_malloc"],
-      ["ssl", "SSL_new"]
-    ].all? do |base, func|
-      result = false
-      libs = ["lib#{base}-[0-9][0-9]", "lib#{base}-[0-9][0-9][0-9]"]
-      libs = Dir.glob(libs.map{|l| libpath.map{|d| File.join(d, l + ".*")}}.flatten).map{|path| File.basename(path, ".*")}.uniq
-      libs.each do |lib|
-        result = have_library(lib, func)
-        break if result
-      end
-      result
-    end
-    return ret if ret
-  end
-  return false
-end
-
-def digest_conf(name, hdr = name, funcs = nil, types = nil)
+def digest_conf(name)
   unless with_config("bundled-#{name}")
     cc = with_config("common-digest")
     if cc == true or /\b#{name}\b/ =~ cc
@@ -62,21 +11,6 @@ def digest_conf(name, hdr = name, funcs = nil, types = nil)
         return :commondigest
       end
     end
-
-    dir_config("openssl")
-    pkg_config("openssl")
-    if find_openssl_library
-      funcs ||= name.upcase
-      funcs = Array(funcs)
-      types ||= funcs
-      hdr = "openssl/#{hdr}.h"
-      if funcs.all? {|func| have_func("#{func}_Transform", hdr)} &&
-         types.all? {|type| have_type("#{type}_CTX", hdr)}
-        $defs << "-D#{name.upcase}_USE_OPENSSL"
-        $headers << "#{name}ossl.h"
-        return :ossl
-      end
-    end
   end
   $objs << "#{name}.#{$OBJEXT}"
   return
diff --git a/ext/digest/md5/depend b/ext/digest/md5/depend
index 9c3f6e55e0e932..987ebcc12df130 100644
--- a/ext/digest/md5/depend
+++ b/ext/digest/md5/depend
@@ -329,5 +329,4 @@ md5init.o: $(srcdir)/../defs.h
 md5init.o: $(srcdir)/../digest.h
 md5init.o: md5.h
 md5init.o: md5init.c
-md5init.o: md5ossl.h
 # AUTOGENERATED DEPENDENCIES END
diff --git a/ext/digest/md5/md5init.c b/ext/digest/md5/md5init.c
index dafd38a29c32d7..688a955b693f44 100644
--- a/ext/digest/md5/md5init.c
+++ b/ext/digest/md5/md5init.c
@@ -3,9 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(MD5_USE_OPENSSL)
-#include "md5ossl.h"
-#elif defined(MD5_USE_COMMONDIGEST)
+#if defined(MD5_USE_COMMONDIGEST)
 #include "md5cc.h"
 #else
 #include "md5.h"
diff --git a/ext/digest/md5/md5ossl.h b/ext/digest/md5/md5ossl.h
deleted file mode 100644
index 94aa7ae77b0488..00000000000000
--- a/ext/digest/md5/md5ossl.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* $Id$ */
-
-#ifndef MD5OSSL_H_INCLUDED
-#define MD5OSSL_H_INCLUDED
-
-#include <stddef.h>
-#include <openssl/md5.h>
-
-#define MD5_BLOCK_LENGTH	MD5_CBLOCK
-
-static DEFINE_FINISH_FUNC_FROM_FINAL(MD5)
-#undef MD5_Finish
-#define MD5_Finish rb_digest_MD5_finish
-
-#endif
diff --git a/ext/digest/rmd160/depend b/ext/digest/rmd160/depend
index 424aa43b0c3c6c..ce5dcdb871ff4f 100644
--- a/ext/digest/rmd160/depend
+++ b/ext/digest/rmd160/depend
@@ -329,5 +329,4 @@ rmd160init.o: $(srcdir)/../defs.h
 rmd160init.o: $(srcdir)/../digest.h
 rmd160init.o: rmd160.h
 rmd160init.o: rmd160init.c
-rmd160init.o: rmd160ossl.h
 # AUTOGENERATED DEPENDENCIES END
diff --git a/ext/digest/rmd160/extconf.rb b/ext/digest/rmd160/extconf.rb
index a02ba561692c17..ffa70ee803f0df 100644
--- a/ext/digest/rmd160/extconf.rb
+++ b/ext/digest/rmd160/extconf.rb
@@ -10,7 +10,7 @@
 
 $objs = [ "rmd160init.#{$OBJEXT}" ]
 
-digest_conf("rmd160", "ripemd", "RIPEMD160")
+digest_conf("rmd160")
 
 have_header("sys/cdefs.h")
 
diff --git a/ext/digest/rmd160/rmd160init.c b/ext/digest/rmd160/rmd160init.c
index a2c0a023c02f2e..2c44d83df610fe 100644
--- a/ext/digest/rmd160/rmd160init.c
+++ b/ext/digest/rmd160/rmd160init.c
@@ -3,11 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(RMD160_USE_OPENSSL)
-#include "rmd160ossl.h"
-#else
 #include "rmd160.h"
-#endif
 
 static const rb_digest_metadata_t rmd160 = {
     RUBY_DIGEST_API_VERSION,
diff --git a/ext/digest/rmd160/rmd160ossl.h b/ext/digest/rmd160/rmd160ossl.h
deleted file mode 100644
index e6bf5ea8d0b538..00000000000000
--- a/ext/digest/rmd160/rmd160ossl.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/* $Id$ */
-
-#ifndef RMD160OSSL_H_INCLUDED
-#define RMD160OSSL_H_INCLUDED
-
-#include <stddef.h>
-#include <openssl/ripemd.h>
-
-#define RMD160_CTX	RIPEMD160_CTX
-
-#define RMD160_Init	RIPEMD160_Init
-#define RMD160_Update	RIPEMD160_Update
-
-#define RMD160_BLOCK_LENGTH		RIPEMD160_CBLOCK
-#define RMD160_DIGEST_LENGTH		RIPEMD160_DIGEST_LENGTH
-
-static DEFINE_FINISH_FUNC_FROM_FINAL(RIPEMD160)
-#define RMD160_Finish rb_digest_RIPEMD160_finish
-
-#endif
diff --git a/ext/digest/sha1/depend b/ext/digest/sha1/depend
index f11e9e31f73866..3daf84a1375d99 100644
--- a/ext/digest/sha1/depend
+++ b/ext/digest/sha1/depend
@@ -329,5 +329,4 @@ sha1init.o: $(srcdir)/../defs.h
 sha1init.o: $(srcdir)/../digest.h
 sha1init.o: sha1.h
 sha1init.o: sha1init.c
-sha1init.o: sha1ossl.h
 # AUTOGENERATED DEPENDENCIES END
diff --git a/ext/digest/sha1/extconf.rb b/ext/digest/sha1/extconf.rb
index 0ff4158bef3d26..1e94ba3ef92873 100644
--- a/ext/digest/sha1/extconf.rb
+++ b/ext/digest/sha1/extconf.rb
@@ -10,7 +10,7 @@
 
 $objs = [ "sha1init.#{$OBJEXT}" ]
 
-digest_conf("sha1", "sha", nil, %w[SHA])
+digest_conf("sha1")
 
 have_header("sys/cdefs.h")
 
diff --git a/ext/digest/sha1/sha1init.c b/ext/digest/sha1/sha1init.c
index 3adf424b1d1d9f..1d201c51fca3ec 100644
--- a/ext/digest/sha1/sha1init.c
+++ b/ext/digest/sha1/sha1init.c
@@ -3,9 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(SHA1_USE_OPENSSL)
-#include "sha1ossl.h"
-#elif defined(SHA1_USE_COMMONDIGEST)
+#if defined(SHA1_USE_COMMONDIGEST)
 #include "sha1cc.h"
 #else
 #include "sha1.h"
diff --git a/ext/digest/sha1/sha1ossl.h b/ext/digest/sha1/sha1ossl.h
deleted file mode 100644
index 599efe9a2fa1a3..00000000000000
--- a/ext/digest/sha1/sha1ossl.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* $Id$ */
-
-#ifndef SHA1OSSL_H_INCLUDED
-#define SHA1OSSL_H_INCLUDED
-
-#include <stddef.h>
-#include <openssl/sha.h>
-
-#define SHA1_CTX	SHA_CTX
-
-#ifdef SHA_BLOCK_LENGTH
-#define SHA1_BLOCK_LENGTH	SHA_BLOCK_LENGTH
-#else
-#define SHA1_BLOCK_LENGTH	SHA_CBLOCK
-#endif
-#define SHA1_DIGEST_LENGTH	SHA_DIGEST_LENGTH
-
-static DEFINE_FINISH_FUNC_FROM_FINAL(SHA1)
-#undef SHA1_Finish
-#define SHA1_Finish rb_digest_SHA1_finish
-
-#endif
diff --git a/ext/digest/sha2/depend b/ext/digest/sha2/depend
index 7aa445d8a78ba7..d8d265c1ef79e0 100644
--- a/ext/digest/sha2/depend
+++ b/ext/digest/sha2/depend
@@ -328,5 +328,4 @@ sha2init.o: $(hdrdir)/ruby/subst.h
 sha2init.o: $(srcdir)/../digest.h
 sha2init.o: sha2.h
 sha2init.o: sha2init.c
-sha2init.o: sha2ossl.h
 # AUTOGENERATED DEPENDENCIES END
diff --git a/ext/digest/sha2/extconf.rb b/ext/digest/sha2/extconf.rb
index 5c7f76c7f372ba..8bf28111b68e3c 100644
--- a/ext/digest/sha2/extconf.rb
+++ b/ext/digest/sha2/extconf.rb
@@ -10,7 +10,7 @@
 
 $objs = [ "sha2init.#{$OBJEXT}" ]
 
-unless digest_conf("sha2", "sha", %w[SHA256 SHA512])
+unless digest_conf("sha2")
   have_type("u_int8_t")
 end
 
diff --git a/ext/digest/sha2/sha2init.c b/ext/digest/sha2/sha2init.c
index 7d211784a32647..aad7b655c880c5 100644
--- a/ext/digest/sha2/sha2init.c
+++ b/ext/digest/sha2/sha2init.c
@@ -3,9 +3,7 @@
 
 #include <ruby/ruby.h>
 #include "../digest.h"
-#if defined(SHA2_USE_OPENSSL)
-#include "sha2ossl.h"
-#elif defined(SHA2_USE_COMMONDIGEST)
+#if defined(SHA2_USE_COMMONDIGEST)
 #include "sha2cc.h"
 #else
 #include "sha2.h"
diff --git a/ext/digest/sha2/sha2ossl.h b/ext/digest/sha2/sha2ossl.h
deleted file mode 100644
index 8dd0530107ed32..00000000000000
--- a/ext/digest/sha2/sha2ossl.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef SHA2OSSL_H_INCLUDED
-#define SHA2OSSL_H_INCLUDED
-
-#include <stddef.h>
-#include <openssl/sha.h>
-
-#define SHA256_BLOCK_LENGTH	SHA256_CBLOCK
-#define SHA384_BLOCK_LENGTH	SHA512_CBLOCK
-#define SHA512_BLOCK_LENGTH	SHA512_CBLOCK
-
-#ifndef __DragonFly__
-#define SHA384_Final SHA512_Final
-#endif
-
-typedef SHA512_CTX SHA384_CTX;
-
-#undef SHA256_Finish
-#undef SHA384_Finish
-#undef SHA512_Finish
-#define SHA256_Finish rb_digest_SHA256_finish
-#define SHA384_Finish rb_digest_SHA384_finish
-#define SHA512_Finish rb_digest_SHA512_finish
-static DEFINE_FINISH_FUNC_FROM_FINAL(SHA256)
-static DEFINE_FINISH_FUNC_FROM_FINAL(SHA384)
-static DEFINE_FINISH_FUNC_FROM_FINAL(SHA512)
-
-#endif