pkey/ec: check private key validity with OpenSSL 3

The behavior of EVP_PKEY_public_check changed between OpenSSL 1.1.1
and 3.0 so that it no longer validates the private key. Instead, private
keys can be validated through EVP_PKEY_private_check and
EVP_PKEY_pairwise_check.

[ky: simplified condition to use either EVP_PKEY_check() or
EVP_PKEY_public_check().]

Author: bannable

ext/openssl/ossl_pkey_ec.c

@@ -483,16 +483,28 @@ static VALUE ossl_ec_key_check_key(VALUE self)
 #ifdef HAVE_EVP_PKEY_CHECK
     EVP_PKEY *pkey;
     EVP_PKEY_CTX *pctx;
-    int ret;
+    EC_KEY *ec;
 
     GetPKey(self, pkey);
+    GetEC(self, ec);
     pctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL);
     if (!pctx)
-        ossl_raise(eDHError, "EVP_PKEY_CTX_new");
-    ret = EVP_PKEY_public_check(pctx);
+        ossl_raise(eECError, "EVP_PKEY_CTX_new");
+
+    if (EC_KEY_get0_private_key(ec) != NULL) {
+        if (EVP_PKEY_check(pctx) != 1) {
+            EVP_PKEY_CTX_free(pctx);
+            ossl_raise(eECError, "EVP_PKEY_check");
+        }
+    }
+    else {
+        if (EVP_PKEY_public_check(pctx) != 1) {
+            EVP_PKEY_CTX_free(pctx);
+            ossl_raise(eECError, "EVP_PKEY_public_check");
+        }
+    }
+
     EVP_PKEY_CTX_free(pctx);
-    if (ret != 1)
-        ossl_raise(eECError, "EVP_PKEY_public_check");
 #else
     EC_KEY *ec;
 

test/openssl/fixtures/pkey/p256_too_large.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIP+TT0V8Fndsnacji9tyf6hmhHywcOWTee9XkiBeJoVloAoGCCqGSM49
+AwEHoUQDQgAEBkhhJIU/2/YdPSlY2I1k25xjK4trr5OXSgXvBC21PtY0HQ7lor7A
+jzT0giJITqmcd81fwGw5+96zLcdxTF1hVQ==
+-----END EC PRIVATE KEY-----

test/openssl/fixtures/pkey/p384_invalid.pem

@@ -0,0 +1,6 @@
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDDA1Tm0m7YhkfeVpFuarAJYVlHp2tQj+1fOBiLa10t9E8TiQO/hVfxB
+vGaVEQwOheWgBwYFK4EEACKhZANiAASyGqmryZGqdpsq5gEDIfNvgC3AwSJxiBCL
+XKHBTFRp+tCezLDOK/6V8KK/vVGBJlGFW6/I7ahyXprxS7xs7hPA9iz5YiuqXlu+
+lbrIpZOz7b73hyQQCkvbBO/Avg+hPAk=
+-----END EC PRIVATE KEY-----

test/openssl/test_pkey_ec.rb

@@ -90,6 +90,13 @@ def test_check_key
     assert_equal(true, key2.public?)
     assert_equal(true, key2.check_key)
 
+    # Behavior of EVP_PKEY_public_check changes between OpenSSL 1.1.1 and 3.0
+    key4 = Fixtures.pkey("p256_too_large")
+    assert_raise(OpenSSL::PKey::ECError) { key4.check_key }
+
+    key5 = Fixtures.pkey("p384_invalid")
+    assert_raise(OpenSSL::PKey::ECError) { key5.check_key }
+
     # EC#private_key= is deprecated in 3.0 and won't work on OpenSSL 3.0
     if !openssl?(3, 0, 0)
       key2.private_key += 1