Skip to content

Commit

Permalink
Force UTF-8 as the process code page on Windows
Browse files Browse the repository at this point in the history
On Windows, add a new "utf8.manifest" application manifest to Orbit,
OrbitService and all tests so that we force UTF-8 as the application
code page. With this change, the "A" variants of Windows APIs now
support UTF-8, which is great because we don't have to change API calls
to their "W" variant, saving a UTF-8 to UTF-16 string conversion in user
code. This also makes std::filesystem::path accept UTF-8 strings by
default (withouth explicit std::filesystem::u8path() call), and the
"std::filesystem::path::string()" also now outputs proper UTF-8 strings
out of the box, whitout calling "filesystem::path::u8string()".

References:
- microsoft/STL#909
- https://docs.microsoft.com/en-us/windows/apps/design/globalizing/use-utf8-code-page
- https://cmake.org/cmake/help/v3.4/release/3.4.html#other

From CMake's doc:
"CMake learned to honor *.manifest source files with MSVC tools.
Manifest files named as sources of .exe and .dll targets will be merged
with linker-generated manifests and embedded in the binary."

Tests: Newly added "UnicodeFileExists" in FileTest.cpp works out of the
box without modification to our "FileExists" function. This means that
"filesystem::path::exists()" can also be called directly with UTF-8
strings.
  • Loading branch information
pierricgimmig committed Jun 28, 2022
1 parent 71e366f commit 63675f9
Show file tree
Hide file tree
Showing 6 changed files with 49 additions and 2 deletions.
5 changes: 5 additions & 0 deletions cmake/tests.cmake
Expand Up @@ -34,6 +34,11 @@ function(register_test TEST_TARGET)
$<TARGET_FILE_DIR:${TEST_TARGET}>/testdata/${TEST_TARGET})
target_link_libraries(${TEST_TARGET} PRIVATE OrbitBase)
endif()

if(WIN32)
# Use UTF-8 as the process code page.
target_sources(${TEST_TARGET} PRIVATE ${CMAKE_SOURCE_DIR}/src/WindowsManifest/utf8.manifest)
endif()
endfunction()

if(NOT TARGET GTest::GTest)
Expand Down
3 changes: 3 additions & 0 deletions src/Orbit/CMakeLists.txt
Expand Up @@ -33,6 +33,9 @@ if(WIN32)
"${CMAKE_CURRENT_SOURCE_DIR}/Orbit.rc.in" Orbit)
target_sources(Orbit PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/Orbit.rc)

# Use UTF-8 as the process code page.
target_sources(Orbit PRIVATE ${CMAKE_SOURCE_DIR}/src/WindowsManifest/utf8.manifest)

# We have to wait for the OrbitQtTests to be finished building because
# as part of this target also windeployqt is called which is also needed
# by the Orbit target.
Expand Down
23 changes: 22 additions & 1 deletion src/OrbitBase/FileTest.cpp
Expand Up @@ -10,6 +10,7 @@
#include "OrbitBase/ReadFileToString.h"
#include "OrbitBase/Result.h"
#include "OrbitBase/TemporaryFile.h"
#include "OrbitBase/WindowsVersion.h"
#include "OrbitBase/WriteStringToFile.h"
#include "Test/Path.h"
#include "TestUtils/TestUtils.h"
Expand Down Expand Up @@ -424,4 +425,24 @@ TEST(File, GetFileDateModified) {
EXPECT_LE(file_time_or_error.value() - now, absl::Seconds(1));
}

} // namespace orbit_base
TEST(File, UnicodeFileExists) {
constexpr const char* kUnicodeFileName = "UnicodeFileName🚀.txt";
std::filesystem::path file_path = orbit_test::GetTestdataDir() / kUnicodeFileName;

// Call "FileExists" with a path containing a Unicode character.
auto file_exists = FileExists(file_path);
ASSERT_THAT(file_exists, HasNoError());
EXPECT_TRUE(file_exists.value()) << "Could not find unicode file path " << file_path.u8string()
#ifdef __WIN32
<< " " << orbit_base::GetWindowsVersionAsString().value()
#endif
;

// Call "FileExists" with "file_path.string()" which should be a valid UTF-8 string.
std::string unicode_file_path_as_string = file_path.string();
file_exists = FileExists(unicode_file_path_as_string);
ASSERT_THAT(file_exists, HasNoError());
EXPECT_TRUE(file_exists.value());
}

} // namespace orbit_base
1 change: 1 addition & 0 deletions src/OrbitBase/testdata/UnicodeFileName🚀.txt
@@ -0,0 +1 @@
🚀
11 changes: 10 additions & 1 deletion src/Service/CMakeLists.txt
Expand Up @@ -37,7 +37,16 @@ target_link_libraries(OrbitServiceLib PUBLIC
endif()

project(OrbitService)
add_executable(OrbitService main.cpp)

add_executable(OrbitService)

target_sources(OrbitService PRIVATE main.cpp)

if(WIN32)
# Use UTF-8 as the process code page.
target_sources(OrbitService PRIVATE ${CMAKE_SOURCE_DIR}/src/WindowsManifest/utf8.manifest)
endif()

target_link_libraries(OrbitService PRIVATE OrbitServiceLib)

strip_symbols(OrbitService)
8 changes: 8 additions & 0 deletions src/WindowsManifest/utf8.manifest
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
<application>
<windowsSettings>
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
</windowsSettings>
</application>
</assembly>

0 comments on commit 63675f9

Please sign in to comment.