Gate ML-DSA tests on what they actually need

The signing tests were gated on the PQC handshake probe, which needs
SSLContext#groups=. On OpenSSL >= 3.5 with Ruby OpenSSL < 4.0 all 23 were
omitted even though ML-DSA keys work there, so the core of this PR went
unverified. They now gate on ML-DSA key support, and the five that build
a certificate gate on nil-digest X509 signing, which Ruby OpenSSL only
accepts from 3.3 on.

The algorithm assertions read the SubjectPublicKeyInfo instead of
PKey#inspect, whose type_name field only exists from Ruby OpenSSL 4.0.
The handshake probe also returned nil on its early exits, so its
memoization never took and it rebound a TCP socket per call.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: hsbt

test/rubygems/helper.rb

@@ -1692,6 +1692,16 @@ def omit_unless_support_pqc
     end
   end
 
+  def omit_unless_support_ml_dsa_key
+    omit "OpenSSL does not support ML-DSA" unless
+      Gem::PQCUtilities.support_ml_dsa_key?
+  end
+
+  def omit_unless_support_ml_dsa_cert
+    omit "Ruby OpenSSL cannot sign a certificate with an ML-DSA key" unless
+      Gem::PQCUtilities.support_ml_dsa_cert?
+  end
+
   def omit_if_support_ml_dsa_key
     omit "OpenSSL supports ML-DSA" if Gem::PQCUtilities.support_ml_dsa_key?
   end

test/rubygems/pqc_utilities.rb

@@ -52,6 +52,41 @@ def self.support_ml_dsa_key?
       end
   end
 
+  ##
+  # Returns whether the runtime can sign an X.509 certificate with an ML-DSA
+  # key. Ruby OpenSSL rejects the nil digest that needs before 3.3, so
+  # support_ml_dsa_key? alone does not cover certificate building.
+
+  def self.support_ml_dsa_cert?
+    return @support_ml_dsa_cert unless @support_ml_dsa_cert.nil?
+
+    @support_ml_dsa_cert =
+      begin
+        key = OpenSSL::PKey.generate_key("ML-DSA-65")
+        cert = OpenSSL::X509::Certificate.new
+        cert.subject = cert.issuer = OpenSSL::X509::Name.new([["CN", "probe"]])
+        cert.public_key = OpenSSL::PKey.read(key.public_to_pem)
+        cert.not_before = Time.now
+        cert.not_after = Time.now + 60
+        cert.sign(key, nil)
+        true
+      # NoMethodError: JRuby's Ruby OpenSSL lacks generate_key.
+      # TypeError: Ruby OpenSSL < 3.3 rejects a nil digest here.
+      rescue OpenSSL::PKey::PKeyError, OpenSSL::X509::CertificateError,
+             NoMethodError, TypeError
+        false
+      end
+  end
+
+  ##
+  # Returns the algorithm named in the SubjectPublicKeyInfo of +key+, such as
+  # "ML-DSA-65". OpenSSL::PKey::PKey#inspect only names the algorithm on Ruby
+  # OpenSSL >= 4.0, and #oid raises for the provider-backed keys ML-DSA uses.
+
+  def self.key_algorithm_name(key)
+    OpenSSL::ASN1.decode(key.public_to_der).value.first.value.first.ln
+  end
+
   # Probe an actual PQC handshake between a forced-PQC server and a
   # default-configured client, mirroring what the integration tests exercise.
   # Memoized so the probe runs at most once per process.
@@ -68,10 +103,10 @@ def self.probe_pqc_handshake
     ctx.key = Gem::PEMUtilities::MLDSA65_SSL_KEY
     # ctx.key is nil when unsupported ML-DSA-65 algorithm's file is read with
     # old OpenSSL versions.
-    return nil unless ctx.key
+    return false unless ctx.key
 
     # ctx.groups (OpenSSL::SSL::SSLContext#groups) requires Ruby OpenSSL >= 4.0.
-    return nil unless ctx.respond_to?(:groups=)
+    return false unless ctx.respond_to?(:groups=)
 
     ctx.groups = "X25519MLKEM768"
     ssl_server = OpenSSL::SSL::SSLServer.new(server, ctx)

test/rubygems/test_gem_commands_build_command.rb

@@ -609,7 +609,7 @@ def test_build_signed_gem
   def test_build_signed_gem_ml_dsa_65
     pend "openssl is missing" unless Gem::HAVE_OPENSSL && !Gem.java_platform?
 
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     trust_dir = Gem::Security.trust_dir
 

test/rubygems/test_gem_commands_cert_command.rb

@@ -158,7 +158,7 @@ def test_execute_build_key_algorithm_ec_key
   end
 
   def test_execute_build_key_algorithm_ml_dsa_65_key
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_cert
 
     passphrase = "Foo bar"
 
@@ -193,7 +193,8 @@ def test_execute_build_key_algorithm_ml_dsa_65_key
     assert_path_exist cert_path
     cert = OpenSSL::X509::Certificate.new(File.read(cert_path))
     assert cert.public_key.is_a? OpenSSL::PKey::PKey
-    assert_match(/type_name=ML-DSA-65/, cert.public_key.inspect)
+    assert_equal "ML-DSA-65",
+                 Gem::PQCUtilities.key_algorithm_name(cert.public_key)
   end
 
   def test_execute_build_key_algorithm_ml_dsa_65_key_without_ml_dsa_support
@@ -390,7 +391,7 @@ def test_execute_build_bad_key
   end
 
   def test_execute_build_ml_dsa_65_key
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_cert
 
     @cmd.handle_options %W[
       --build nobody@example.com
@@ -414,7 +415,7 @@ def test_execute_build_ml_dsa_65_key
   end
 
   def test_execute_build_encrypted_ml_dsa_65_key
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_cert
 
     @cmd.handle_options %W[
       --build nobody@example.com
@@ -448,7 +449,7 @@ def test_execute_certificate
   end
 
   def test_execute_certificate_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     use_ui @ui do
       @cmd.handle_options %W[--certificate #{ML_DSA_65_PUBLIC_CERT_FILE}]
@@ -512,7 +513,7 @@ def test_execute_encrypted_private_key
   end
 
   def test_execute_private_ml_dsa_65_key
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     use_ui @ui do
       @cmd.send :handle_options, %W[--private-key #{ML_DSA_65_PRIVATE_KEY_FILE}]
@@ -538,7 +539,7 @@ def test_execute_private_ml_dsa_65_key_without_ml_dsa_support
   end
 
   def test_execute_encrypted_private_ml_dsa_65_key
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     use_ui @ui do
       @cmd.send :handle_options,

test/rubygems/test_gem_security.rb

@@ -104,30 +104,30 @@ def test_class_create_key_ec
   end
 
   def test_class_create_key_ml_dsa_44
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key = Gem::Security.create_key "ml-dsa-44"
 
     assert_kind_of OpenSSL::PKey::PKey, key
-    assert_match(/type_name=ML-DSA-44/, key.inspect)
+    assert_equal "ML-DSA-44", Gem::PQCUtilities.key_algorithm_name(key)
   end
 
   def test_class_create_key_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key = Gem::Security.create_key "ml-dsa-65"
 
     assert_kind_of OpenSSL::PKey::PKey, key
-    assert_match(/type_name=ML-DSA-65/, key.inspect)
+    assert_equal "ML-DSA-65", Gem::PQCUtilities.key_algorithm_name(key)
   end
 
   def test_class_create_key_ml_dsa_87
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key = Gem::Security.create_key "ml-dsa-87"
 
     assert_kind_of OpenSSL::PKey::PKey, key
-    assert_match(/type_name=ML-DSA-87/, key.inspect)
+    assert_equal "ML-DSA-87", Gem::PQCUtilities.key_algorithm_name(key)
   end
 
   def test_class_create_key_ml_dsa_44_without_ml_dsa_support
@@ -207,7 +207,7 @@ def test_class_digest_required_raises_unsupported_algorithm
   end
 
   def test_class_digest_required_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     refute Gem::Security.digest_required?(Gem::Security.create_key("ml-dsa-65"))
   end
@@ -225,7 +225,7 @@ def test_class_get_public_key_ec
   end
 
   def test_class_get_public_key_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     pkey = Gem::Security.get_public_key(ML_DSA_65_PRIVATE_KEY)
 
@@ -334,7 +334,7 @@ def test_class_sign_AltName
   end
 
   def test_class_sign_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_cert
 
     assert_sign ML_DSA_65_PUBLIC_CERT, ML_DSA_65_PRIVATE_KEY
   end
@@ -362,7 +362,7 @@ def test_class_write_private_key
   end
 
   def test_class_write_private_key_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key = Gem::Security.create_key "ml-dsa-65"
 
@@ -394,7 +394,7 @@ def test_class_write_private_key_encrypted
   end
 
   def test_class_write_private_key_encrypted_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key = Gem::Security.create_key "ml-dsa-65"
 
@@ -433,7 +433,7 @@ def test_class_write_private_key_encrypted_cipher
   end
 
   def test_class_write_private_key_encrypted_cipher_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key = Gem::Security.create_key "ml-dsa-65"
 

test/rubygems/test_gem_security_policy.rb

@@ -54,7 +54,7 @@ def test_check_data
   end
 
   def test_check_data_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     data = digest "hello"
 
@@ -78,7 +78,7 @@ def test_check_data_invalid
   end
 
   def test_check_data_invalid_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     data = digest "hello"
 
@@ -249,7 +249,7 @@ def test_check_trust
   end
 
   def test_check_trust_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     Gem::Security.trust_dir.trust_cert ML_DSA_65_PUBLIC_CERT
 

test/rubygems/test_gem_security_signer.rb

@@ -70,7 +70,7 @@ def test_initialize_key_path
   end
 
   def test_initialize_key_path_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key_file = ML_DSA_65_PRIVATE_KEY_FILE
 
@@ -103,7 +103,7 @@ def test_initialize_encrypted_key_path
   end
 
   def test_initialize_encrypted_key_path_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     key_file = ML_DSA_65_ENCRYPTED_PRIVATE_KEY_FILE
 
@@ -153,7 +153,7 @@ def test_sign
   end
 
   def test_sign_ml_dsa_65
-    omit_unless_support_pqc
+    omit_unless_support_ml_dsa_key
 
     signer = Gem::Security::Signer.new ML_DSA_65_PRIVATE_KEY, [ML_DSA_65_PUBLIC_CERT]