fix(ext/node): decrypt encrypted private keys in publicEncrypt/privateDecrypt and ignore passphrase (#33770)

Enables `test-crypto-keygen-async-encrypted-private-key` in node_compat
suite.

Co-authored-by: divybot <divybot@users.noreply.github.com>
Co-authored-by: Divy Srivastava <me@littledivy.com>

Author: divybot

ext/node/polyfills/internal/crypto/cipher.ts

@@ -18,10 +18,12 @@ import {
   op_node_cipheriv_take,
   op_node_create_cipheriv,
   op_node_create_decipheriv,
+  op_node_create_private_key,
   op_node_decipheriv_auth_tag,
   op_node_decipheriv_decrypt,
   op_node_decipheriv_final,
   op_node_decipheriv_set_aad,
+  op_node_export_private_key_pem,
   op_node_export_secret_key,
   op_node_private_decrypt,
   op_node_private_encrypt,
@@ -755,14 +757,36 @@ export function prepareKey(key) {
     const data = key.export({ type: "pkcs8", format: "pem" });
     return { data: getArrayBufferOrView(data, "key") };
   } else if (typeof key == "object") {
-    const { key: data, encoding } = key;
+    const { key: data, encoding, passphrase, format, type } = key;
     if (isKeyObject(data)) {
       return prepareKey(data);
     }
     if (!isStringOrBuffer(data)) {
       throw new TypeError("Invalid key type");
     }
 
+    // If a passphrase is supplied with raw key material, decrypt the key via
+    // the native key handle and re-export as unencrypted PKCS#8 PEM so the
+    // downstream RSA ops can parse it.
+    if (passphrase != null) {
+      const keyFormat = format ?? (typeof data === "string" ? "pem" : "der");
+      const keyData = getArrayBufferOrView(data, "key", encoding);
+      const passphraseData = getArrayBufferOrView(passphrase, "passphrase");
+      const handle = op_node_create_private_key(
+        keyData,
+        keyFormat,
+        type ?? "",
+        passphraseData,
+      );
+      const pem = op_node_export_private_key_pem(
+        handle,
+        "pkcs8",
+        null,
+        null,
+      );
+      return { data: getArrayBufferOrView(pem, "key") };
+    }
+
     return { data: getArrayBufferOrView(data, "key", encoding) };
   }
 

ext/node_crypto/keys.rs

@@ -902,9 +902,18 @@ impl KeyObjectHandle {
       "der" => match typ {
         "pkcs8" => {
           if let Some(passphrase) = passphrase {
-            SecretDocument::from_pkcs8_encrypted_der(key, passphrase).map_err(
-              |_| AsymmetricPrivateKeyError::InvalidEncryptedPkcs8PrivateKey,
-            )?
+            if EncryptedPrivateKeyInfo::try_from(key).is_ok() {
+              SecretDocument::from_pkcs8_encrypted_der(key, passphrase)
+                .map_err(|_| {
+                  AsymmetricPrivateKeyError::InvalidEncryptedPkcs8PrivateKey
+                })?
+            } else {
+              // Node ignores the passphrase when the key isn't actually
+              // encrypted.
+              SecretDocument::from_pkcs8_der(key).map_err(|_| {
+                AsymmetricPrivateKeyError::InvalidPkcs8PrivateKey
+              })?
+            }
           } else if EncryptedPrivateKeyInfo::try_from(key).is_ok() {
             return Err(
               AsymmetricPrivateKeyError::EncryptedPkcs8DerRequiresPassphrase,

tests/node_compat/config.jsonc

@@ -513,6 +513,7 @@
     "parallel/test-crypto-keygen-async-elliptic-curve-jwk-ec.js": {},
     "parallel/test-crypto-keygen-async-elliptic-curve-jwk.js": {},
     "parallel/test-crypto-keygen-async-elliptic-curve-jwk-rsa.js": {},
+    "parallel/test-crypto-keygen-async-encrypted-private-key.js": {},
     "parallel/test-crypto-keygen-async-named-elliptic-curve.js": {},
     "parallel/test-crypto-keygen-async-rsa.js": {
       "ignore": true,