8331320: ClassFile API OutOfMemoryError with certain class files

Reviewed-by: psandoz

src/java.base/share/classes/jdk/internal/classfile/impl/AbstractInstruction.java

@@ -262,6 +262,9 @@ public static final class BoundLookupSwitchInstruction
 
             this.afterPad = pos + 1 + ((4 - ((pos + 1 - code.codeStart) & 3)) & 3);
             this.npairs = code.classReader.readInt(afterPad + 4);
+            if (npairs < 0 || npairs > code.codeLength >> 3) {
+                throw new IllegalArgumentException("Invalid lookupswitch npairs value: " + npairs);
+            }
         }
 
         static int size(CodeImpl code, int codeStart, int pos) {
@@ -314,6 +317,9 @@ static int size(CodeImpl code, int codeStart, int pos) {
             int pad = ap - (pos + 1);
             int low = code.classReader.readInt(ap + 4);
             int high = code.classReader.readInt(ap + 8);
+            if (high < low || high - low > code.codeLength >> 2) {
+                throw new IllegalArgumentException("Invalid tableswitch values low: " + low + " high: " + high);
+            }
             int cnt = high - low + 1;
             return 1 + pad + 12 + cnt * 4;
         }

test/jdk/jdk/classfile/LimitsTest.java

@@ -23,16 +23,22 @@
 
 /*
  * @test
- * @bug 8320360 8330684
+ * @bug 8320360 8330684 8331320
  * @summary Testing ClassFile limits.
  * @run junit LimitsTest
  */
+import java.lang.classfile.Attributes;
+import java.lang.classfile.BufWriter;
 import java.lang.constant.ClassDesc;
 import java.lang.constant.ConstantDescs;
 import java.lang.constant.MethodTypeDesc;
 import java.lang.classfile.ClassFile;
+import java.lang.classfile.Opcode;
+import java.lang.classfile.attribute.CodeAttribute;
 import java.lang.classfile.constantpool.ConstantPoolException;
+import jdk.internal.classfile.impl.DirectMethodBuilder;
 import jdk.internal.classfile.impl.LabelContext;
+import jdk.internal.classfile.impl.UnboundAttribute;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -99,4 +105,51 @@ void testInvalidClassEntry() {
         assertThrows(ConstantPoolException.class, () -> ClassFile.of().parse(new byte[]{(byte)0xCA, (byte)0xFE, (byte)0xBA, (byte)0xBE,
             0, 0, 0, 0, 0, 2, ClassFile.TAG_METHODREF, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}).thisClass());
     }
+
+    @Test
+    void testInvalidLookupSwitch() {
+        assertThrows(IllegalArgumentException.class, () ->
+                ClassFile.of().parse(ClassFile.of().build(ClassDesc.of("LookupSwitchClass"), cb -> cb.withMethod(
+                "lookupSwitchMethod", MethodTypeDesc.of(ConstantDescs.CD_void), 0, mb ->
+                        ((DirectMethodBuilder)mb).writeAttribute(new UnboundAttribute.AdHocAttribute<CodeAttribute>(Attributes.CODE) {
+                                @Override
+                                public void writeBody(BufWriter b) {
+                                    b.writeU2(-1);//max stack
+                                    b.writeU2(-1);//max locals
+                                    b.writeInt(16);
+                                    b.writeU1(Opcode.NOP.bytecode());
+                                    b.writeU1(Opcode.NOP.bytecode());
+                                    b.writeU1(Opcode.NOP.bytecode());
+                                    b.writeU1(Opcode.NOP.bytecode());
+                                    b.writeU1(Opcode.LOOKUPSWITCH.bytecode());
+                                    b.writeU1(0); //padding
+                                    b.writeU2(0); //padding
+                                    b.writeInt(0); //default
+                                    b.writeInt(-2); //npairs to jump back and cause OOME if not checked
+                                    b.writeU2(0);//exception handlers
+                                    b.writeU2(0);//attributes
+                                }})))).methods().get(0).code().get().elementList());
+    }
+
+    @Test
+    void testInvalidTableSwitch() {
+        assertThrows(IllegalArgumentException.class, () ->
+                ClassFile.of().parse(ClassFile.of().build(ClassDesc.of("TableSwitchClass"), cb -> cb.withMethod(
+                "tableSwitchMethod", MethodTypeDesc.of(ConstantDescs.CD_void), 0, mb ->
+                        ((DirectMethodBuilder)mb).writeAttribute(new UnboundAttribute.AdHocAttribute<CodeAttribute>(Attributes.CODE) {
+                                @Override
+                                public void writeBody(BufWriter b) {
+                                    b.writeU2(-1);//max stack
+                                    b.writeU2(-1);//max locals
+                                    b.writeInt(16);
+                                    b.writeU1(Opcode.TABLESWITCH.bytecode());
+                                    b.writeU1(0); //padding
+                                    b.writeU2(0); //padding
+                                    b.writeInt(0); //default
+                                    b.writeInt(0); //low
+                                    b.writeInt(-5); //high to jump back and cause OOME if not checked
+                                    b.writeU2(0);//exception handlers
+                                    b.writeU2(0);//attributes
+                                }})))).methods().get(0).code().get().elementList());
+    }
 }