[ruby/openssl] Pass through nil as digest when signing certificates

(ruby/openssl#761)

In order to sign certificates with Ed25519 keys, NULL must be passed
as md to X509_sign.  This NULL is then passed
(via ASN1_item_sign_ex) as type to EVP_DigestSignInit.  The
documentation[1] of EVP_DigestSignInit states that type must be NULL
for various key types, including Ed25519.

[1]: https://www.openssl.org/docs/manmaster/man3/EVP_DigestSignInit.html

ruby/openssl@b0fc100091

Author: gartens

ext/openssl/ossl_x509cert.c

@@ -539,7 +539,11 @@ ossl_x509_sign(VALUE self, VALUE key, VALUE digest)
     const EVP_MD *md;
 
     pkey = GetPrivPKeyPtr(key); /* NO NEED TO DUP */
-    md = ossl_evp_get_digestbyname(digest);
+    if (NIL_P(digest)) {
+        md = NULL; /* needed for some key types, e.g. Ed25519 */
+    } else {
+        md = ossl_evp_get_digestbyname(digest);
+    }
     GetX509(self, x509);
     if (!X509_sign(x509, pkey, md)) {
 	ossl_raise(eX509CertError, NULL);

test/openssl/test_x509cert.rb

@@ -222,6 +222,29 @@ def test_sign_and_verify_dsa_md5
     }
   end
 
+  def test_sign_and_verify_ed25519
+    # See test_ed25519 in test_pkey.rb
+
+    # Ed25519 is not FIPS-approved.
+    omit_on_fips
+
+    begin
+      ed25519  = OpenSSL::PKey::generate_key("ED25519")
+    rescue OpenSSL::PKey::PKeyError => e
+      # OpenSSL < 1.1.1
+      #
+      pend "Ed25519 is not implemented" unless openssl?(1, 1, 1)
+
+      raise e
+    end
+
+    # See ASN1_item_sign_ctx in ChangeLog for 3.8.1: https://github.com/libressl/portable/blob/master/ChangeLog
+    pend 'ASN1 signing with Ed25519 not yet working' unless openssl? or libressl?(3, 8, 1)
+
+    cert = issue_cert(@ca, ed25519, 1, [], nil, nil, digest: nil)
+    assert_equal(true, cert.verify(ed25519))
+  end
+
   def test_dsa_with_sha2
     cert = issue_cert(@ca, @dsa256, 1, [], nil, nil, digest: "sha256")
     assert_equal("dsa_with_SHA256", cert.signature_algorithm)