8262472: Buffer overflow in UNICODE::as_utf8 for zero length output buffer

Reviewed-by: dholmes, iklam

Author: tstuefe

src/hotspot/share/utilities/utf8.cpp

@@ -447,6 +447,7 @@ char* UNICODE::as_utf8(const T* base, int& length) {
 }
 
 char* UNICODE::as_utf8(const jchar* base, int length, char* buf, int buflen) {
+  assert(buflen > 0, "zero length output buffer");
   u_char* p = (u_char*)buf;
   for (int index = 0; index < length; index++) {
     jchar c = base[index];
@@ -459,6 +460,7 @@ char* UNICODE::as_utf8(const jchar* base, int length, char* buf, int buflen) {
 }
 
 char* UNICODE::as_utf8(const jbyte* base, int length, char* buf, int buflen) {
+  assert(buflen > 0, "zero length output buffer");
   u_char* p = (u_char*)buf;
   for (int index = 0; index < length; index++) {
     jbyte c = base[index];

test/hotspot/gtest/utilities/test_utf8.cpp

@@ -25,7 +25,22 @@
 #include "utilities/utf8.hpp"
 #include "unittest.hpp"
 
-TEST(utf8, length) {
+static void stamp(char* p, size_t len) {
+  if (len > 0) {
+    ::memset(p, 'A', len);
+  }
+}
+
+static bool test_stamp(const char* p, size_t len) {
+  for (const char* q = p; q < p + len; q++) {
+    if (*q != 'A') {
+      return false;
+    }
+  }
+  return true;
+}
+
+TEST_VM(utf8, jchar_length) {
   char res[60];
   jchar str[20];
 
@@ -35,16 +50,56 @@ TEST(utf8, length) {
   str[19] = (jchar) '\0';
 
   // The resulting string in UTF-8 is 3*19 bytes long, but should be truncated
+  stamp(res, sizeof(res));
   UNICODE::as_utf8(str, 19, res, 10);
   ASSERT_EQ(strlen(res), (size_t) 9) << "string should be truncated here";
+  ASSERT_TRUE(test_stamp(res + 10, sizeof(res) - 10));
 
+  stamp(res, sizeof(res));
   UNICODE::as_utf8(str, 19, res, 18);
   ASSERT_EQ(strlen(res), (size_t) 15) << "string should be truncated here";
+  ASSERT_TRUE(test_stamp(res + 18, sizeof(res) - 18));
 
+  stamp(res, sizeof(res));
   UNICODE::as_utf8(str, 19, res, 20);
   ASSERT_EQ(strlen(res), (size_t) 18) << "string should be truncated here";
+  ASSERT_TRUE(test_stamp(res + 20, sizeof(res) - 20));
 
   // Test with an "unbounded" buffer
   UNICODE::as_utf8(str, 19, res, INT_MAX);
   ASSERT_EQ(strlen(res), (size_t) 3 * 19) << "string should end here";
+
+  // Test that we do not overflow the output buffer
+  for (int i = 1; i < 5; i ++) {
+    stamp(res, sizeof(res));
+    UNICODE::as_utf8(str, 19, res, i);
+    EXPECT_TRUE(test_stamp(res + i, sizeof(res) - i));
+  }
+
+}
+
+TEST_VM(utf8, jbyte_length) {
+  char res[60];
+  jbyte str[20];
+
+  for (int i = 0; i < 19; i++) {
+    str[i] = 0x42;
+  }
+  str[19] = '\0';
+
+  stamp(res, sizeof(res));
+  UNICODE::as_utf8(str, 19, res, 10);
+  ASSERT_EQ(strlen(res), (size_t) 9) << "string should be truncated here";
+  ASSERT_TRUE(test_stamp(res + 10, sizeof(res) - 10));
+
+  UNICODE::as_utf8(str, 19, res, INT_MAX);
+  ASSERT_EQ(strlen(res), (size_t) 19) << "string should end here";
+
+  // Test that we do not overflow the output buffer
+  for (int i = 1; i < 5; i ++) {
+    stamp(res, sizeof(res));
+    UNICODE::as_utf8(str, 19, res, i);
+    EXPECT_TRUE(test_stamp(res + i, sizeof(res) - i));
+  }
+
 }