crypto: reject public keys properly

Fixes: nodejs#29904

PR-URL: nodejs#29913
Reviewed-By: Sam Roberts <vieuxtech@gmail.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>

lib/internal/crypto/keys.js

@@ -270,7 +270,10 @@ function prepareAsymmetricKey(key, ctx) {
          ...(ctx !== kCreatePrivate ? ['KeyObject'] : [])],
         key);
     }
-    return { data, ...parseKeyEncoding(key, undefined) };
+
+    const isPublic =
+      (ctx === kConsumePrivate || ctx === kCreatePrivate) ? false : undefined;
+    return { data, ...parseKeyEncoding(key, undefined, isPublic) };
   } else {
     throw new ERR_INVALID_ARG_TYPE(
       'key',

test/parallel/test-crypto-key-objects.js

@@ -200,6 +200,27 @@ const privateDsa = fixtures.readKey('dsa_private_encrypted_1025.pem',
     library: 'BIO routines',
     function: 'BIO_new_mem_buf',
   });
+
+  // This should not abort either: https://github.com/nodejs/node/issues/29904
+  assert.throws(() => {
+    createPrivateKey({ key: Buffer.alloc(0), format: 'der', type: 'spki' });
+  }, {
+    code: 'ERR_INVALID_OPT_VALUE',
+    message: 'The value "spki" is invalid for option "type"'
+  });
+
+  // Unlike SPKI, PKCS#1 is a valid encoding for private keys (and public keys),
+  // so it should be accepted by createPrivateKey, but OpenSSL won't parse it.
+  assert.throws(() => {
+    const key = createPublicKey(publicPem).export({
+      format: 'der',
+      type: 'pkcs1'
+    });
+    createPrivateKey({ key, format: 'der', type: 'pkcs1' });
+  }, {
+    message: /asn1 encoding/,
+    library: 'asn1 encoding routines'
+  });
 }
 
 [