Skip to content

Commit

Permalink
Adds unicode support to Windows environment.
Browse files Browse the repository at this point in the history 
Currently the Windows environment uses the *A ANSI Win32 API calls for file system operations. This precludes the ability of consuming applications to pass paths with unicode characters. A detailed discussion is in #755. This PR swtiches the *A ANSI calls to the *W wide unicode string calls along with the conversion methods from standard UTF-8 strings to the UTF-16 multi-byte strings requires by the Win32 *W API functions.
  • Loading branch information
@sipsorcery
sipsorcery committed Dec 11, 2019
1 parent 58a89bb commit 37a9962
Show file tree
Hide file tree
Showing 2 changed files with 201 additions and 93 deletions.
98 changes: 70 additions & 28 deletions util/env_windows.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,8 +379,9 @@ class WindowsEnv : public Env {
*result = nullptr;
DWORD desired_access = GENERIC_READ;
DWORD share_mode = FILE_SHARE_READ;
ScopedHandle handle = ::CreateFileA(
filename.c_str(), desired_access, share_mode,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
/*hTemplateFile=*/nullptr);
if (!handle.is_valid()) {
Expand All @@ -396,8 +397,9 @@ class WindowsEnv : public Env {
*result = nullptr;
DWORD desired_access = GENERIC_READ;
DWORD share_mode = FILE_SHARE_READ;
auto wFilename = toUtf16(filename);
ScopedHandle handle =
::CreateFileA(filename.c_str(), desired_access, share_mode,
::CreateFileW(wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, OPEN_EXISTING,
FILE_ATTRIBUTE_READONLY,
/*hTemplateFile=*/nullptr);
Expand All @@ -417,11 +419,12 @@ class WindowsEnv : public Env {
}

ScopedHandle mapping =
::CreateFileMappingA(handle.get(),
/*security attributes=*/nullptr, PAGE_READONLY,
/*dwMaximumSizeHigh=*/0,
/*dwMaximumSizeLow=*/0,
/*lpName=*/nullptr);
::CreateFileMappingW(handle.get(),
/*security attributes=*/nullptr,
PAGE_READONLY,
/*dwMaximumSizeHigh=*/0,
/*dwMaximumSizeLow=*/0,
/*lpName=*/nullptr);
if (mapping.is_valid()) {
void* mmap_base = ::MapViewOfFile(mapping.get(), FILE_MAP_READ,
/*dwFileOffsetHigh=*/0,
Expand All @@ -442,8 +445,9 @@ class WindowsEnv : public Env {
WritableFile** result) override {
DWORD desired_access = GENERIC_WRITE;
DWORD share_mode = 0; // Exclusive access.
ScopedHandle handle = ::CreateFileA(
filename.c_str(), desired_access, share_mode,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL,
/*hTemplateFile=*/nullptr);
if (!handle.is_valid()) {
Expand All @@ -459,8 +463,9 @@ class WindowsEnv : public Env {
WritableFile** result) override {
DWORD desired_access = FILE_APPEND_DATA;
DWORD share_mode = 0; // Exclusive access.
ScopedHandle handle = ::CreateFileA(
filename.c_str(), desired_access, share_mode,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), desired_access, share_mode,
/*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
/*hTemplateFile=*/nullptr);
if (!handle.is_valid()) {
Expand All @@ -473,14 +478,16 @@ class WindowsEnv : public Env {
}

bool FileExists(const std::string& filename) override {
return GetFileAttributesA(filename.c_str()) != INVALID_FILE_ATTRIBUTES;
auto wFilename = toUtf16(filename);
return GetFileAttributesW(wFilename.c_str()) != INVALID_FILE_ATTRIBUTES;
}

Status GetChildren(const std::string& directory_path,
std::vector<std::string>* result) override {
const std::string find_pattern = directory_path + "\\*";
WIN32_FIND_DATAA find_data;
HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data);
WIN32_FIND_DATAW find_data;
auto wFind_pattern = toUtf16(find_pattern);
HANDLE dir_handle = ::FindFirstFileW(wFind_pattern.c_str(), &find_data);
if (dir_handle == INVALID_HANDLE_VALUE) {
DWORD last_error = ::GetLastError();
if (last_error == ERROR_FILE_NOT_FOUND) {
Expand All @@ -492,11 +499,12 @@ class WindowsEnv : public Env {
char base_name[_MAX_FNAME];
char ext[_MAX_EXT];

if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name,
auto find_data_filename = toUtf8(find_data.cFileName);
if (!_splitpath_s(find_data_filename.c_str(), nullptr, 0, nullptr, 0, base_name,
ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) {
result->emplace_back(std::string(base_name) + ext);
}
} while (::FindNextFileA(dir_handle, &find_data));
} while (::FindNextFileW(dir_handle, &find_data));
DWORD last_error = ::GetLastError();
::FindClose(dir_handle);
if (last_error != ERROR_NO_MORE_FILES) {
Expand All @@ -506,29 +514,33 @@ class WindowsEnv : public Env {
}

Status DeleteFile(const std::string& filename) override {
if (!::DeleteFileA(filename.c_str())) {
auto wFilename = toUtf16(filename);
if (!::DeleteFileW(wFilename.c_str())) {
return WindowsError(filename, ::GetLastError());
}
return Status::OK();
}

Status CreateDir(const std::string& dirname) override {
if (!::CreateDirectoryA(dirname.c_str(), nullptr)) {
auto wDirname = toUtf16(dirname);
if (!::CreateDirectoryW(wDirname.c_str(), nullptr)) {
return WindowsError(dirname, ::GetLastError());
}
return Status::OK();
}

Status DeleteDir(const std::string& dirname) override {
if (!::RemoveDirectoryA(dirname.c_str())) {
auto wDirname = toUtf16(dirname);
if (!::RemoveDirectoryW(wDirname.c_str())) {
return WindowsError(dirname, ::GetLastError());
}
return Status::OK();
}

Status GetFileSize(const std::string& filename, uint64_t* size) override {
WIN32_FILE_ATTRIBUTE_DATA file_attributes;
if (!::GetFileAttributesExA(filename.c_str(), GetFileExInfoStandard,
auto wFilename = toUtf16(filename);
if (!::GetFileAttributesExW(wFilename.c_str(), GetFileExInfoStandard,
&file_attributes)) {
return WindowsError(filename, ::GetLastError());
}
Expand All @@ -542,7 +554,9 @@ class WindowsEnv : public Env {
Status RenameFile(const std::string& from, const std::string& to) override {
// Try a simple move first. It will only succeed when |to| doesn't already
// exist.
if (::MoveFileA(from.c_str(), to.c_str())) {
auto wFrom = toUtf16(from);
auto wTo = toUtf16(to);
if (::MoveFileW(wFrom.c_str(), wTo.c_str())) {
return Status::OK();
}
DWORD move_error = ::GetLastError();
Expand All @@ -551,7 +565,7 @@ class WindowsEnv : public Env {
// succeed when |to| does exist. When writing to a network share, we may not
// be able to change the ACLs. Ignore ACL errors then
// (REPLACEFILE_IGNORE_MERGE_ERRORS).
if (::ReplaceFileA(to.c_str(), from.c_str(), /*lpBackupFileName=*/nullptr,
if (::ReplaceFileW(wTo.c_str(), wFrom.c_str(), /*lpBackupFileName=*/nullptr,
REPLACEFILE_IGNORE_MERGE_ERRORS,
/*lpExclude=*/nullptr, /*lpReserved=*/nullptr)) {
return Status::OK();
Expand All @@ -571,8 +585,9 @@ class WindowsEnv : public Env {
Status LockFile(const std::string& filename, FileLock** lock) override {
*lock = nullptr;
Status result;
ScopedHandle handle = ::CreateFileA(
filename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
auto wFilename = toUtf16(filename);
ScopedHandle handle = ::CreateFileW(
wFilename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
/*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
nullptr);
if (!handle.is_valid()) {
Expand Down Expand Up @@ -612,10 +627,11 @@ class WindowsEnv : public Env {
return Status::OK();
}

char tmp_path[MAX_PATH];
if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) {
wchar_t wtmp_path[MAX_PATH];
if (!GetTempPathW(ARRAYSIZE(wtmp_path), wtmp_path)) {
return WindowsError("GetTempPath", ::GetLastError());
}
std::string tmp_path = toUtf8(std::wstring(wtmp_path));
std::stringstream ss;
ss << tmp_path << "leveldbtest-" << std::this_thread::get_id();
*result = ss.str();
Expand All @@ -626,7 +642,8 @@ class WindowsEnv : public Env {
}

Status NewLogger(const std::string& filename, Logger** result) override {
std::FILE* fp = std::fopen(filename.c_str(), "w");
auto wFilename = toUtf16(filename);
std::FILE* fp = _wfopen(wFilename.c_str(), L"w");
if (fp == nullptr) {
*result = nullptr;
return WindowsError(filename, ::GetLastError());
Expand Down Expand Up @@ -682,6 +699,31 @@ class WindowsEnv : public Env {
GUARDED_BY(background_work_mutex_);

Limiter mmap_limiter_; // Thread-safe.

// Converts a Windows wide multi-byte UTF-16 string to a UTF-8 string.
// See http://utf8everywhere.org/#windows
std::string toUtf8(const std::wstring& wstr) {
if (wstr.empty()) return std::string();
int size_needed = WideCharToMultiByte(
CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
std::string strTo(size_needed, 0);
WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0],
size_needed, NULL, NULL);
return strTo;
}

// Converts a UTF-8 string to a Windows UTF-16 multi-byte wide character
// string.
// See http://utf8everywhere.org/#windows
std::wstring toUtf16(const std::string& str) {
if (str.empty()) return std::wstring();
int size_needed = MultiByteToWideChar(
CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0);
std::wstring strTo(size_needed, 0);
MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &strTo[0],
size_needed);
return strTo;
}
};

// Return the maximum number of concurrent mmaps.
Expand Down
Loading

0 comments on commit 37a9962

Please sign in to comment.