8347596: Update HSS/LMS public key encoding

Reviewed-by: mullan
Backport-of: 0ee6ba9

Author: wangweij

src/java.base/share/classes/sun/security/provider/HSS.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -757,12 +757,8 @@ static class HSSPublicKey extends X509Key implements Serializable {
                                 Arrays.copyOfRange(keyArray, 4, keyArray.length),
                                 0, true);
                 algid = new AlgorithmId(ObjectIdentifier.of(KnownOIDs.HSSLMS));
-                byte[] derEncodedKeyarray =
-                        new DerOutputStream()
-                                .putOctetString(keyArray)
-                                .toByteArray();
                 this.setKey(new BitArray(
-                        8 * derEncodedKeyarray.length, derEncodedKeyarray));
+                        8 * keyArray.length, keyArray));
             }
         }
 
@@ -783,11 +779,17 @@ public String toString() {
         @Override
         protected void parseKeyBits() throws InvalidKeyException {
             byte[] keyArray = getKey().toByteArray();
-            if ((keyArray[0] != DerValue.tag_OctetString) || (keyArray[1] != keyArray.length -2)) {
-                throw new InvalidKeyException("Bad X509Key");
+            if (keyArray.length < 12) { // More length check in LMSPublicKey
+                throw new InvalidKeyException("LMS public key is too short");
+            }
+            if (keyArray[0] == DerValue.tag_OctetString
+                    && keyArray[1] == keyArray.length - 2) {
+                // pre-8347596 format that has an inner OCTET STRING.
+                keyArray = Arrays.copyOfRange(keyArray, 2, keyArray.length);
+                setKey(new BitArray(keyArray.length * 8, keyArray));
             }
-            L = LMSUtils.fourBytesToInt(keyArray, 2);
-            lmsPublicKey = new LMSPublicKey(keyArray, 6, true);
+            L = LMSUtils.fourBytesToInt(keyArray, 0);
+            lmsPublicKey = new LMSPublicKey(keyArray, 4, true);
         }
 
         @java.io.Serial

src/java.base/share/classes/sun/security/util/KeyUtil.java

@@ -403,7 +403,7 @@ public static String hashAlgFromHSS(PublicKey publicKey)
         try {
             DerValue val = new DerValue(publicKey.getEncoded());
             val.data.getDerValue();
-            byte[] rawKey = new DerValue(val.data.getBitString()).getOctetString();
+            byte[] rawKey = val.data.getBitString();
             // According to https://www.rfc-editor.org/rfc/rfc8554.html:
             // Section 6.1: HSS public key is u32str(L) || pub[0], where pub[0]
             // is the LMS public key for the top-level tree.

test/jdk/sun/security/provider/hss/TestHSS.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 8298127
+ * @bug 8298127 8347596
  * @library /test/lib
  * @summary tests for HSS/LMS provider
  * @modules java.base/sun.security.util
@@ -40,6 +40,7 @@
 import java.security.spec.X509EncodedKeySpec;
 import java.util.HexFormat;
 
+import jdk.test.lib.Asserts;
 import sun.security.util.*;
 
 import jdk.test.lib.util.SerializationUtils;
@@ -61,9 +62,7 @@ public static void main(String[] args) throws Exception {
             i++;
         }
 
-        if (!serializeTest()) {
-            throw new RuntimeException("serializeTest failed");
-        }
+        serializeTest();
 
         System.out.println("All tests passed");
     }
@@ -88,7 +87,7 @@ static boolean kat(TestCase t) throws Exception {
         }
     }
 
-    static boolean serializeTest() throws Exception {
+    static void serializeTest() throws Exception {
         final ObjectIdentifier oid;
         var pk = decode("""
                 00000002
@@ -106,7 +105,19 @@ static boolean serializeTest() throws Exception {
             throw new AssertionError(e);
         }
 
-        var keyBits = new DerOutputStream().putOctetString(pk).toByteArray();
+        // Encoding without inner OCTET STRING
+        var pk0 = makeKey(oid, pk);
+        // Encoding with inner OCTET STRING
+        var pk1 = makeKey(oid, new DerOutputStream().putOctetString(pk).toByteArray());
+        Asserts.assertEquals(pk0, pk1);
+
+        PublicKey pk2 = (PublicKey) SerializationUtils
+                .deserialize(SerializationUtils.serialize(pk1));
+        Asserts.assertEquals(pk1, pk2);
+    }
+
+    static PublicKey makeKey(ObjectIdentifier oid, byte[] keyBits)
+            throws Exception {
         var oidBytes = new DerOutputStream().write(DerValue.tag_Sequence,
                 new DerOutputStream().putOID(oid));
         var x509encoding = new DerOutputStream().write(DerValue.tag_Sequence,
@@ -115,11 +126,7 @@ static boolean serializeTest() throws Exception {
                 .toByteArray();
 
         var x509KeySpec = new X509EncodedKeySpec(x509encoding);
-        var pk1 = KeyFactory.getInstance(ALG).generatePublic(x509KeySpec);
-
-        PublicKey pk2 = (PublicKey) SerializationUtils
-                .deserialize(SerializationUtils.serialize(pk1));
-        return pk2.equals(pk1);
+        return KeyFactory.getInstance(ALG).generatePublic(x509KeySpec);
     }
 
     static boolean verify(byte[] pk, byte[] sig, byte[] msg) throws Exception {