8240197: Cannot start JVM when $JAVA_HOME includes CJK characters

Reviewed-by: iklam, stuefe, rschmelter

src/hotspot/os/windows/os_windows.cpp

@@ -4161,98 +4161,121 @@ static void file_attribute_data_to_stat(struct stat* sbuf, WIN32_FILE_ATTRIBUTE_
   }
 }
 
+static errno_t convert_to_unicode(char const* char_path, LPWSTR* unicode_path) {
+  // Get required buffer size to convert to Unicode
+  int unicode_path_len = MultiByteToWideChar(CP_THREAD_ACP,
+                                             MB_ERR_INVALID_CHARS,
+                                             char_path, -1,
+                                             NULL, 0);
+  if (unicode_path_len == 0) {
+    return EINVAL;
+  }
+
+  *unicode_path = NEW_C_HEAP_ARRAY(WCHAR, unicode_path_len, mtInternal);
+
+  int result = MultiByteToWideChar(CP_THREAD_ACP,
+                                   MB_ERR_INVALID_CHARS,
+                                   char_path, -1,
+                                   *unicode_path, unicode_path_len);
+  assert(result == unicode_path_len, "length already checked above");
+
+  return ERROR_SUCCESS;
+}
+
+static errno_t get_full_path(LPCWSTR unicode_path, LPWSTR* full_path) {
+  // Get required buffer size to convert to full path. The return
+  // value INCLUDES the terminating null character.
+  DWORD full_path_len = GetFullPathNameW(unicode_path, 0, NULL, NULL);
+  if (full_path_len == 0) {
+    return EINVAL;
+  }
+
+  *full_path = NEW_C_HEAP_ARRAY(WCHAR, full_path_len, mtInternal);
+
+  // When the buffer has sufficient size, the return value EXCLUDES the
+  // terminating null character
+  DWORD result = GetFullPathNameW(unicode_path, full_path_len, *full_path, NULL);
+  assert(result <= full_path_len, "length already checked above");
+
+  return ERROR_SUCCESS;
+}
+
+static void set_path_prefix(char* buf, LPWSTR* prefix, int* prefix_off, bool* needs_fullpath) {
+  *prefix_off = 0;
+  *needs_fullpath = true;
+
+  if (::isalpha(buf[0]) && !::IsDBCSLeadByte(buf[0]) && buf[1] == ':' && buf[2] == '\\') {
+    *prefix = L"\\\\?\\";
+  } else if (buf[0] == '\\' && buf[1] == '\\') {
+    if (buf[2] == '?' && buf[3] == '\\') {
+      *prefix = L"";
+      *needs_fullpath = false;
+    } else {
+      *prefix = L"\\\\?\\UNC";
+      *prefix_off = 1; // Overwrite the first char with the prefix, so \\share\path becomes \\?\UNC\share\path
+    }
+  } else {
+    *prefix = L"\\\\?\\";
+  }
+}
+
 // Returns the given path as an absolute wide path in unc format. The returned path is NULL
 // on error (with err being set accordingly) and should be freed via os::free() otherwise.
-// additional_space is the number of additionally allocated wchars after the terminating L'\0'.
-// This is based on pathToNTPath() in io_util_md.cpp, but omits the optimizations for
-// short paths.
+// additional_space is the size of space, in wchar_t, the function will additionally add to
+// the allocation of return buffer (such that the size of the returned buffer is at least
+// wcslen(buf) + 1 + additional_space).
 static wchar_t* wide_abs_unc_path(char const* path, errno_t & err, int additional_space = 0) {
   if ((path == NULL) || (path[0] == '\0')) {
     err = ENOENT;
     return NULL;
   }
 
-  size_t path_len = strlen(path);
   // Need to allocate at least room for 3 characters, since os::native_path transforms C: to C:.
-  char* buf = (char*) os::malloc(1 + MAX2((size_t) 3, path_len), mtInternal);
-  wchar_t* result = NULL;
+  size_t buf_len = 1 + MAX2((size_t)3, strlen(path));
+  char* buf = NEW_C_HEAP_ARRAY(char, buf_len, mtInternal);
+  strncpy(buf, path, buf_len);
+  os::native_path(buf);
+
+  LPWSTR prefix = NULL;
+  int prefix_off = 0;
+  bool needs_fullpath = true;
+  set_path_prefix(buf, &prefix, &prefix_off, &needs_fullpath);
+
+  LPWSTR unicode_path = NULL;
+  err = convert_to_unicode(buf, &unicode_path);
+  FREE_C_HEAP_ARRAY(char, buf);
+  if (err != ERROR_SUCCESS) {
+    return NULL;
+  }
 
-  if (buf == NULL) {
-    err = ENOMEM;
+  LPWSTR converted_path = NULL;
+  if (needs_fullpath) {
+    err = get_full_path(unicode_path, &converted_path);
   } else {
-    memcpy(buf, path, path_len + 1);
-    os::native_path(buf);
-
-    wchar_t* prefix;
-    int prefix_off = 0;
-    bool is_abs = true;
-    bool needs_fullpath = true;
-
-    if (::isalpha(buf[0]) && !::IsDBCSLeadByte(buf[0]) && buf[1] == ':' && buf[2] == '\\') {
-      prefix = L"\\\\?\\";
-    } else if (buf[0] == '\\' && buf[1] == '\\') {
-      if (buf[2] == '?' && buf[3] == '\\') {
-        prefix = L"";
-        needs_fullpath = false;
-      } else {
-        prefix = L"\\\\?\\UNC";
-        prefix_off = 1; // Overwrite the first char with the prefix, so \\share\path becomes \\?\UNC\share\path
-      }
-    } else {
-      is_abs = false;
-      prefix = L"\\\\?\\";
-    }
+    converted_path = unicode_path;
+  }
 
-    size_t buf_len = strlen(buf);
+  LPWSTR result = NULL;
+  if (converted_path != NULL) {
     size_t prefix_len = wcslen(prefix);
-    size_t full_path_size = is_abs ? 1 + buf_len : JVM_MAXPATHLEN;
-    size_t result_size = prefix_len + full_path_size - prefix_off;
-    result = (wchar_t*) os::malloc(sizeof(wchar_t) * (additional_space + result_size), mtInternal);
-
-    if (result == NULL) {
-      err = ENOMEM;
-    } else {
-      size_t converted_chars;
-      wchar_t* path_start = result + prefix_len - prefix_off;
-      err = ::mbstowcs_s(&converted_chars, path_start, buf_len + 1, buf, buf_len);
-
-      if ((err == ERROR_SUCCESS) && needs_fullpath) {
-        wchar_t* tmp = (wchar_t*) os::malloc(sizeof(wchar_t) * full_path_size, mtInternal);
-
-        if (tmp == NULL) {
-          err = ENOMEM;
-        } else {
-          if (!_wfullpath(tmp, path_start, full_path_size)) {
-            err = ENOENT;
-          } else {
-            ::memcpy(path_start, tmp, (1 + wcslen(tmp)) * sizeof(wchar_t));
-          }
-
-          os::free(tmp);
-        }
-      }
-
-      memcpy(result, prefix, sizeof(wchar_t) * prefix_len);
-
-      // Remove trailing pathsep (not for \\?\<DRIVE>:\, since it would make it relative)
-      size_t result_len = wcslen(result);
-
-      if (result[result_len - 1] == L'\\') {
-        if (!(::iswalpha(result[4]) && result[5] == L':' && result_len == 7)) {
-          result[result_len - 1] = L'\0';
-        }
-      }
+    size_t result_len = prefix_len - prefix_off + wcslen(converted_path) + additional_space + 1;
+    result = NEW_C_HEAP_ARRAY(WCHAR, result_len, mtInternal);
+    _snwprintf(result, result_len, L"%s%s", prefix, &converted_path[prefix_off]);
+
+    // Remove trailing pathsep (not for \\?\<DRIVE>:\, since it would make it relative)
+    result_len = wcslen(result);
+    if ((result[result_len - 1] == L'\\') &&
+        !(::iswalpha(result[4]) && result[5] == L':' && result_len == 7)) {
+      result[result_len - 1] = L'\0';
     }
   }
 
-  os::free(buf);
-
-  if (err != ERROR_SUCCESS) {
-    os::free(result);
-    result = NULL;
+  if (converted_path != unicode_path) {
+    FREE_C_HEAP_ARRAY(WCHAR, converted_path);
   }
+  FREE_C_HEAP_ARRAY(WCHAR, unicode_path);
 
-  return result;
+  return static_cast<wchar_t*>(result); // LPWSTR and wchat_t* are the same type on Windows.
 }
 
 int os::stat(const char *path, struct stat *sbuf) {