8314295: Enhance verification of verifier

Reviewed-by: dholmes, poonam, dlong

src/hotspot/share/classfile/verifier.cpp

@@ -2250,11 +2250,12 @@ void ClassVerifier::verify_switch(
           "low must be less than or equal to high in tableswitch");
       return;
     }
-    keys = high - low + 1;
-    if (keys < 0) {
+    int64_t keys64 = ((int64_t)high - low) + 1;
+    if (keys64 > 65535) {  // Max code length
       verify_error(ErrorContext::bad_code(bci), "too many keys in tableswitch");
       return;
     }
+    keys = (int)keys64;
     delta = 1;
   } else {
     keys = (int)Bytes::get_Java_u4(aligned_bcp + jintSize);

src/hotspot/share/interpreter/bytecodes.cpp

@@ -385,12 +385,18 @@ int Bytecodes::special_length_at(Bytecodes::Code code, address bcp, address end)
       if (end != nullptr && aligned_bcp + 3*jintSize >= end) {
         return -1; // don't read past end of code buffer
       }
+      // Promote calculation to signed 64 bits to do range checks, used by the verifier.
       jlong lo = (jint)Bytes::get_Java_u4(aligned_bcp + 1*jintSize);
       jlong hi = (jint)Bytes::get_Java_u4(aligned_bcp + 2*jintSize);
       jlong len = (aligned_bcp - bcp) + (3 + hi - lo + 1)*jintSize;
-      // only return len if it can be represented as a positive int;
-      // return -1 otherwise
-      return (len > 0 && len == (int)len) ? len : -1;
+      // Only return len if it can be represented as a positive int and lo <= hi.
+      // The caller checks for bytecode stream overflow.
+      if (lo <= hi && len == (int)len) {
+        assert(len > 0, "must be");
+        return (int)len;
+      } else {
+        return -1;
+      }
     }
 
   case _lookupswitch:      // fall through
@@ -402,9 +408,13 @@ int Bytecodes::special_length_at(Bytecodes::Code code, address bcp, address end)
       }
       jlong npairs = (jint)Bytes::get_Java_u4(aligned_bcp + jintSize);
       jlong len = (aligned_bcp - bcp) + (2 + 2*npairs)*jintSize;
-      // only return len if it can be represented as a positive int;
-      // return -1 otherwise
-      return (len > 0 && len == (int)len) ? len : -1;
+      // Only return len if it can be represented as a positive int and npairs >= 0.
+      if (npairs >= 0 && len == (int)len) {
+        assert(len > 0, "must be");
+        return (int)len;
+      } else {
+        return -1;
+      }
     }
   default:
     // Note: Length functions must return <=0 for invalid bytecodes.

src/java.base/share/native/libverify/check_code.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1994, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1994, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,6 +81,7 @@
 #include <assert.h>
 #include <limits.h>
 #include <stdlib.h>
+#include <stdint.h>
 
 #include "jni.h"
 #include "jni_util.h"
@@ -1195,7 +1196,7 @@ verify_opcode_operands(context_type *context, unsigned int inumber, int offset)
             }
         }
         if (opcode == JVM_OPC_tableswitch) {
-            keys = _ck_ntohl(lpc[2]) -  _ck_ntohl(lpc[1]) + 1;
+            keys = _ck_ntohl(lpc[2]) - _ck_ntohl(lpc[1]) + 1;
             delta = 1;
         } else {
             keys = _ck_ntohl(lpc[1]); /* number of pairs */
@@ -1677,11 +1678,13 @@ static int instruction_length(unsigned char *iptr, unsigned char *end)
     switch (instruction) {
         case JVM_OPC_tableswitch: {
             int *lpc = (int *)UCALIGN(iptr + 1);
-            int index;
             if (lpc + 2 >= (int *)end) {
                 return -1; /* do not read pass the end */
             }
-            index = _ck_ntohl(lpc[2]) - _ck_ntohl(lpc[1]);
+            int64_t low  = _ck_ntohl(lpc[1]);
+            int64_t high = _ck_ntohl(lpc[2]);
+            int64_t index = high - low;
+            // The value of low must be less than or equal to high - i.e. index >= 0
             if ((index < 0) || (index > 65535)) {
                 return -1;      /* illegal */
             } else {