From 47700d6c2bcbae98c6be69bae7839522b4c6e942 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 26 Oct 2023 13:12:53 +0200 Subject: [PATCH 1/3] Refactor utf8_to_windows_encoding --- libmamba/include/mamba/core/util_os.hpp | 1 - libmamba/include/mamba/util/os_win.hpp | 6 ++- libmamba/src/core/environment.cpp | 6 +-- libmamba/src/core/util_os.cpp | 37 ---------------- libmamba/src/util/os_win.cpp | 55 ++++++++++++++++++++---- libmamba/tests/CMakeLists.txt | 1 + libmamba/tests/src/core/test_util_os.cpp | 6 --- libmamba/tests/src/util/test_os_win.cpp | 31 +++++++++++++ 8 files changed, 87 insertions(+), 56 deletions(-) create mode 100644 libmamba/tests/src/util/test_os_win.cpp diff --git a/libmamba/include/mamba/core/util_os.hpp b/libmamba/include/mamba/core/util_os.hpp index 7e1049a1d0..1d1ede521a 100644 --- a/libmamba/include/mamba/core/util_os.hpp +++ b/libmamba/include/mamba/core/util_os.hpp @@ -49,7 +49,6 @@ namespace mamba std::string to_utf8(const wchar_t* windows_unicode_text, size_t size); std::string to_utf8(const wchar_t* windows_unicode_text); std::string to_utf8(const std::wstring& windows_unicode_text); - std::wstring to_windows_unicode(const std::string_view utf8_text); #endif /* Test whether a given `std::ostream` object refers to a terminal. */ diff --git a/libmamba/include/mamba/util/os_win.hpp b/libmamba/include/mamba/util/os_win.hpp index 3088bcb91f..42bfb37abe 100644 --- a/libmamba/include/mamba/util/os_win.hpp +++ b/libmamba/include/mamba/util/os_win.hpp @@ -7,8 +7,10 @@ #ifndef MAMBA_UTIL_OS_WIN_HPP #define MAMBA_UTIL_OS_WIN_HPP -#include "mamba/fs/filesystem.hpp" +#include +#include +#include "mamba/fs/filesystem.hpp" namespace mamba::util { @@ -23,5 +25,7 @@ namespace mamba::util }; auto get_windows_known_user_folder(WindowsKnowUserFolder dir) -> fs::u8path; + + auto utf8_to_windows_encoding(const std::string_view utf8_text) -> std::wstring; } #endif diff --git a/libmamba/src/core/environment.cpp b/libmamba/src/core/environment.cpp index 632c3f744c..53e06be3c2 100644 --- a/libmamba/src/core/environment.cpp +++ b/libmamba/src/core/environment.cpp @@ -51,7 +51,7 @@ namespace mamba::env ); }; - const std::wstring unicode_key = to_windows_unicode(key); + const std::wstring unicode_key = util::utf8_to_windows_encoding(key); size_t required_size = 0; if (auto error_code = _wgetenv_s(&required_size, nullptr, 0, unicode_key.c_str()); error_code == 0) @@ -99,8 +99,8 @@ namespace mamba::env // functions are not thread-safe, this // is to prevent related issues. - const std::wstring unicode_key = to_windows_unicode(key); - const std::wstring unicode_value = to_windows_unicode(value); + const std::wstring unicode_key = util::utf8_to_windows_encoding(key); + const std::wstring unicode_value = util::utf8_to_windows_encoding(value); auto res = _wputenv_s(unicode_key.c_str(), unicode_value.c_str()); if (res != 0) { diff --git a/libmamba/src/core/util_os.cpp b/libmamba/src/core/util_os.cpp index d486c9575d..9ba3682198 100644 --- a/libmamba/src/core/util_os.cpp +++ b/libmamba/src/core/util_os.cpp @@ -538,43 +538,6 @@ namespace mamba return to_utf8(s.data(), s.size()); } - std::wstring to_windows_unicode(const std::string_view utf8_text) - { - std::wstring output; - if (!utf8_text.empty()) - { - assert(utf8_text.size() <= INT_MAX); - const int size = MultiByteToWideChar( - CP_UTF8, - 0, - utf8_text.data(), - utf8_text.size(), - nullptr, - 0 - ); - if (size <= 0) - { - unsigned long last_error = ::GetLastError(); - LOG_ERROR << "Failed to convert UTF-8 string to Windows Unicode (UTF-16)" - << std::system_category().message(static_cast(last_error)); - throw std::runtime_error("Failed to convert UTF-8 string to UTF-16"); - } - - output.resize(size); - int res_size = MultiByteToWideChar( - CP_UTF8, - 0, - utf8_text.data(), - utf8_text.size(), - output.data(), - output.size() - ); - assert(res_size == size); - } - - return output; - } - #endif /* From https://github.com/ikalnytskyi/termcolor diff --git a/libmamba/src/util/os_win.cpp b/libmamba/src/util/os_win.cpp index 4144137be8..300e208910 100644 --- a/libmamba/src/util/os_win.cpp +++ b/libmamba/src/util/os_win.cpp @@ -6,8 +6,11 @@ #ifdef _WIN32 +#include +#include #include +#include #include #include "mamba/util/os_win.hpp" @@ -22,17 +25,17 @@ namespace mamba::util switch (dir) { case (WindowsKnowUserFolder::Documents): - return FOLDERID_Documents; + return ::FOLDERID_Documents; case (WindowsKnowUserFolder::Profile): - return FOLDERID_Profile; + return ::FOLDERID_Profile; case (WindowsKnowUserFolder::Programs): - return FOLDERID_Programs; + return ::FOLDERID_Programs; case (WindowsKnowUserFolder::ProgramData): - return FOLDERID_ProgramData; + return ::FOLDERID_ProgramData; case (WindowsKnowUserFolder::LocalAppData): - return FOLDERID_LocalAppData; + return ::FOLDERID_LocalAppData; case (WindowsKnowUserFolder::RoamingAppData): - return FOLDERID_RoamingAppData; + return ::FOLDERID_RoamingAppData; } throw std::invalid_argument("Invalid enum"); } @@ -41,7 +44,7 @@ namespace mamba::util { ~COMwstr() { - CoTaskMemFree(str); + ::CoTaskMemFree(str); } wchar_t* str = nullptr; @@ -52,7 +55,7 @@ namespace mamba::util { auto localAppData = COMwstr{ nullptr }; - HRESULT const hres = SHGetKnownFolderPath( + const auto hres = SHGetKnownFolderPath( get_windows_known_user_folder_raw(dir), KF_FLAG_DONT_VERIFY, nullptr, @@ -66,6 +69,37 @@ namespace mamba::util return fs::u8path(localAppData.str); } + + auto utf8_to_windows_encoding(const std::string_view utf8_text) -> std::wstring + { + if (utf8_text.empty()) + { + return {}; + } + + assert(utf8_text.size() <= std::numeric_limits::max()); + const int size = ::MultiByteToWideChar(CP_UTF8, 0, utf8_text.data(), utf8_text.size(), nullptr, 0); + if (size <= 0) + { + throw std::runtime_error(fmt::format( + R"(Failed to convert UTF-8 string "{}" to UTF-16: )", + utf8_text, + std::system_category().message(static_cast(::GetLastError())) + )); + } + + auto output = std::wstring(static_cast(size), char(0)); + [[maybe_unused]] const int res_size = ::MultiByteToWideChar( + CP_UTF8, + 0, + utf8_text.data(), + utf8_text.size(), + output.data(), + output.size() + ); + assert(res_size == size); + return output; + } } #else // #ifdef _WIN32 @@ -91,6 +125,11 @@ namespace mamba::util { throw_not_implemented("get_windows_known_user_folder"); } + + auto utf8_to_windows_encoding(const std::string_view) -> std::wstring + { + throw_not_implemented("utf8_to_windows_unicode"); + } } #endif // #ifdef _WIN32 diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index 296e253776..ed5596fe5f 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -36,6 +36,7 @@ set(LIBMAMBA_TEST_SRCS src/util/test_flat_bool_expr_tree.cpp src/util/test_graph.cpp src/util/test_iterator.cpp + src/util/test_os_win.cpp src/util/test_path_manip.cpp src/util/test_tuple_hash.cpp src/util/test_url_manip.cpp diff --git a/libmamba/tests/src/core/test_util_os.cpp b/libmamba/tests/src/core/test_util_os.cpp index 63d6616cb9..47563fd4c8 100644 --- a/libmamba/tests/src/core/test_util_os.cpp +++ b/libmamba/tests/src/core/test_util_os.cpp @@ -25,12 +25,6 @@ TEST_SUITE("basic_unicode_conversion") auto result = mamba::to_utf8(text_utf16); CHECK_EQ(text_utf8, result); } - - TEST_CASE("to_windows_unicode") - { - auto result = mamba::to_windows_unicode(text_utf8); - CHECK_EQ(text_utf16, result); - } } TEST_SUITE("windows_path") diff --git a/libmamba/tests/src/util/test_os_win.cpp b/libmamba/tests/src/util/test_os_win.cpp new file mode 100644 index 0000000000..618d42574c --- /dev/null +++ b/libmamba/tests/src/util/test_os_win.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2023, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifdef _WIN32 + +#include + +#include + +#include "mamba/util/os_win.hpp" + +using namespace mamba::util; + +TEST_SUITE("uti::os_win") +{ + TEST_CASE("") + { + const std::wstring text_utf16 = L"Hello, I am Joël. 私のにほんごわへたです"; + const std::string text_utf8 = u8"Hello, I am Joël. 私のにほんごわへたです"; + + SUBCASE("utf8_to_windows_encoding") + { + CHECK_EQ(utf8_to_windows_encoding(text_utf8), text_utf16); + } + } +} + +#endif From 01163e6134619e2c5db044bbca5f17e53dd3f996 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 26 Oct 2023 14:05:20 +0200 Subject: [PATCH 2/3] Refactor windows_encoding_to_utf8 --- libmamba/include/mamba/core/util_os.hpp | 6 --- libmamba/include/mamba/util/os_win.hpp | 6 ++- libmamba/src/core/environment.cpp | 2 +- libmamba/src/core/shell_init.cpp | 10 +++-- libmamba/src/core/util_os.cpp | 54 ------------------------ libmamba/src/util/os_win.cpp | 49 ++++++++++++++++++++- libmamba/tests/src/core/test_util_os.cpp | 15 ------- libmamba/tests/src/util/test_os_win.cpp | 7 +++ micromamba/src/main.cpp | 4 +- 9 files changed, 69 insertions(+), 84 deletions(-) diff --git a/libmamba/include/mamba/core/util_os.hpp b/libmamba/include/mamba/core/util_os.hpp index 1d1ede521a..0ab7031420 100644 --- a/libmamba/include/mamba/core/util_os.hpp +++ b/libmamba/include/mamba/core/util_os.hpp @@ -45,12 +45,6 @@ namespace mamba void init_console(); void reset_console(); -#ifdef _WIN32 - std::string to_utf8(const wchar_t* windows_unicode_text, size_t size); - std::string to_utf8(const wchar_t* windows_unicode_text); - std::string to_utf8(const std::wstring& windows_unicode_text); -#endif - /* Test whether a given `std::ostream` object refers to a terminal. */ bool is_atty(const std::ostream& stream); diff --git a/libmamba/include/mamba/util/os_win.hpp b/libmamba/include/mamba/util/os_win.hpp index 42bfb37abe..6a31b52a9b 100644 --- a/libmamba/include/mamba/util/os_win.hpp +++ b/libmamba/include/mamba/util/os_win.hpp @@ -24,8 +24,10 @@ namespace mamba::util RoamingAppData, }; - auto get_windows_known_user_folder(WindowsKnowUserFolder dir) -> fs::u8path; + [[nodiscard]] auto get_windows_known_user_folder(WindowsKnowUserFolder dir) -> fs::u8path; - auto utf8_to_windows_encoding(const std::string_view utf8_text) -> std::wstring; + [[nodiscard]] auto utf8_to_windows_encoding(const std::string_view utf8_text) -> std::wstring; + + [[nodiscard]] auto windows_encoding_to_utf8(std::wstring_view) -> std::string; } #endif diff --git a/libmamba/src/core/environment.cpp b/libmamba/src/core/environment.cpp index 53e06be3c2..243389b0c3 100644 --- a/libmamba/src/core/environment.cpp +++ b/libmamba/src/core/environment.cpp @@ -68,7 +68,7 @@ namespace mamba::env { value.pop_back(); // Remove the `\0` that was written in, otherwise any future // concatenation will fail. - return mamba::to_utf8(value); + return util::windows_encoding_to_utf8(value); } else { diff --git a/libmamba/src/core/shell_init.cpp b/libmamba/src/core/shell_init.cpp index 983fbc1977..f06eb6aeb7 100644 --- a/libmamba/src/core/shell_init.cpp +++ b/libmamba/src/core/shell_init.cpp @@ -16,6 +16,8 @@ #include #ifdef _WIN32 #include + +#include "mamba/util/os_win.hpp" #endif #include "mamba/core/activation.hpp" @@ -33,14 +35,14 @@ namespace mamba { namespace { - static std::regex const MAMBA_INITIALIZE_RE_BLOCK("\n?# >>> mamba initialize >>>(?:\n|\r\n)?" + static const std::regex MAMBA_INITIALIZE_RE_BLOCK("\n?# >>> mamba initialize >>>(?:\n|\r\n)?" "([\\s\\S]*?)" "# <<< mamba initialize <<<(?:\n|\r\n)?"); - static std::regex const MAMBA_INITIALIZE_PS_RE_BLOCK("\n?#region mamba initialize(?:\n|\r\n)?" + static const std::regex MAMBA_INITIALIZE_PS_RE_BLOCK("\n?#region mamba initialize(?:\n|\r\n)?" "([\\s\\S]*?)" "#endregion(?:\n|\r\n)?"); - static std::wregex const + static const std::wregex MAMBA_CMDEXE_HOOK_REGEX(L"(\"[^\"]*?mamba[-_]hook\\.bat\")", std::regex_constants::icase); } @@ -126,7 +128,7 @@ namespace mamba fmt::print( out, "Setting cmd.exe AUTORUN to: {}", - fmt::styled(to_utf8(value), graphics.palette.success) + fmt::styled(util::windows_encoding_to_utf8(value), graphics.palette.success) ); winreg::RegKey key{ HKEY_CURRENT_USER, reg_path }; diff --git a/libmamba/src/core/util_os.cpp b/libmamba/src/core/util_os.cpp index 9ba3682198..b3f6a13e8f 100644 --- a/libmamba/src/core/util_os.cpp +++ b/libmamba/src/core/util_os.cpp @@ -486,60 +486,6 @@ namespace mamba #endif } -#ifdef _WIN32 - std::string to_utf8(const wchar_t* w, size_t s) - { - std::string output; - if (s != 0) - { - assert(s <= INT_MAX); - const int size = WideCharToMultiByte( - CP_UTF8, - 0, - w, - static_cast(s), - nullptr, - 0, - nullptr, - nullptr - ); - if (size <= 0) - { - unsigned long last_error = ::GetLastError(); - LOG_ERROR << "Failed to convert string to UTF-8 " - << std::system_category().message(static_cast(last_error)); - throw std::runtime_error("Failed to convert string to UTF-8"); - } - - output.resize(size); - int res_size = WideCharToMultiByte( - CP_UTF8, - 0, - w, - static_cast(s), - output.data(), - static_cast(size), - nullptr, - nullptr - ); - assert(res_size == size); - } - - return output; - } - - std::string to_utf8(const wchar_t* w) - { - return to_utf8(w, wcslen(w)); - } - - std::string to_utf8(const std::wstring& s) - { - return to_utf8(s.data(), s.size()); - } - -#endif - /* From https://github.com/ikalnytskyi/termcolor * * copyright: (c) 2013 by Ihor Kalnytskyi. diff --git a/libmamba/src/util/os_win.cpp b/libmamba/src/util/os_win.cpp index 300e208910..6aac1a0a1b 100644 --- a/libmamba/src/util/os_win.cpp +++ b/libmamba/src/util/os_win.cpp @@ -82,7 +82,7 @@ namespace mamba::util if (size <= 0) { throw std::runtime_error(fmt::format( - R"(Failed to convert UTF-8 string "{}" to UTF-16: )", + R"(Failed to convert UTF-8 string "{}" to UTF-16: {})", utf8_text, std::system_category().message(static_cast(::GetLastError())) )); @@ -100,6 +100,48 @@ namespace mamba::util assert(res_size == size); return output; } + + auto windows_encoding_to_utf8(std::wstring_view str) -> std::string + { + if (str.empty()) + { + return {}; + } + + assert(str.size() <= std::numeric_limits::max()); + const int size = ::WideCharToMultiByte( + CP_UTF8, + 0, + str.data(), + static_cast(str.size()), + nullptr, + 0, + nullptr, + nullptr + ); + if (size <= 0) + { + throw std::runtime_error(fmt::format( + R"(Failed to convert UTF-16 string to UTF-8: {})", + std::system_category().message(static_cast(::GetLastError())) + )); + } + + auto output = std::string(static_cast(size), char(0)); + [[maybe_unused]] const int res_size = ::WideCharToMultiByte( + CP_UTF8, + 0, + str.data(), + static_cast(str.size()), + output.data(), + static_cast(size), + nullptr, + nullptr + ); + assert(res_size == size); + + return output; + } } #else // #ifdef _WIN32 @@ -130,6 +172,11 @@ namespace mamba::util { throw_not_implemented("utf8_to_windows_unicode"); } + + auto windows_encoding_to_utf8(std::wstring_view) -> std::string + { + throw_not_implemented("windows_encoding_to_utf8"); + } } #endif // #ifdef _WIN32 diff --git a/libmamba/tests/src/core/test_util_os.cpp b/libmamba/tests/src/core/test_util_os.cpp index 47563fd4c8..6c4e080e4f 100644 --- a/libmamba/tests/src/core/test_util_os.cpp +++ b/libmamba/tests/src/core/test_util_os.cpp @@ -12,21 +12,6 @@ #ifdef _WIN32 -namespace -{ - const std::wstring text_utf16 = L"Hello, I am Joël. 私のにほんごわへたです"; - const std::string text_utf8 = u8"Hello, I am Joël. 私のにほんごわへたです"; -} - -TEST_SUITE("basic_unicode_conversion") -{ - TEST_CASE("to_utf8") - { - auto result = mamba::to_utf8(text_utf16); - CHECK_EQ(text_utf8, result); - } -} - TEST_SUITE("windows_path") { TEST_CASE("fix_win_path") diff --git a/libmamba/tests/src/util/test_os_win.cpp b/libmamba/tests/src/util/test_os_win.cpp index 618d42574c..85c035664b 100644 --- a/libmamba/tests/src/util/test_os_win.cpp +++ b/libmamba/tests/src/util/test_os_win.cpp @@ -23,8 +23,15 @@ TEST_SUITE("uti::os_win") SUBCASE("utf8_to_windows_encoding") { + CHECK_EQ(utf8_to_windows_encoding(""), L""); CHECK_EQ(utf8_to_windows_encoding(text_utf8), text_utf16); } + + SUBCASE("windows_encoding_to_utf8") + { + CHECK_EQ(windows_encoding_to_utf8(L""), ""); + CHECK_EQ(windows_encoding_to_utf8(text_utf16), text_utf8); + } } } diff --git a/micromamba/src/main.cpp b/micromamba/src/main.cpp index a4979f8989..17be8b21fa 100644 --- a/micromamba/src/main.cpp +++ b/micromamba/src/main.cpp @@ -11,6 +11,8 @@ #include // Incomplete header included last #include + +#include "mamba/util/os_win.hpp" #endif #include @@ -56,7 +58,7 @@ main(int argc, char** argv) std::vector utf8CharArgs; for (int i = 0; i < argc; i++) { - utf8Args.push_back(to_utf8(wargv[i])); + utf8Args.push_back(util::windows_encoding_to_utf8(wargv[i])); } for (int i = 0; i < argc; ++i) { From c9a759304b78bdcf6a820d190315b487db083e50 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 26 Oct 2023 14:09:11 +0200 Subject: [PATCH 3/3] Remove unused fix_win_path --- libmamba/include/mamba/core/util_os.hpp | 2 -- libmamba/src/core/util_os.cpp | 19 ---------------- libmamba/tests/CMakeLists.txt | 1 - libmamba/tests/src/core/test_util_os.cpp | 29 ------------------------ 4 files changed, 51 deletions(-) delete mode 100644 libmamba/tests/src/core/test_util_os.cpp diff --git a/libmamba/include/mamba/core/util_os.hpp b/libmamba/include/mamba/core/util_os.hpp index 0ab7031420..99ce4be27d 100644 --- a/libmamba/include/mamba/core/util_os.hpp +++ b/libmamba/include/mamba/core/util_os.hpp @@ -58,8 +58,6 @@ namespace mamba int get_console_height(); void codesign(const fs::u8path& path, bool verbose = false); - - std::string fix_win_path(const std::string& path); } #endif diff --git a/libmamba/src/core/util_os.cpp b/libmamba/src/core/util_os.cpp index b3f6a13e8f..ff99b7cb59 100644 --- a/libmamba/src/core/util_os.cpp +++ b/libmamba/src/core/util_os.cpp @@ -596,23 +596,4 @@ namespace mamba throw std::runtime_error(std::string("Could not codesign executable: ") + ec.message()); } } - - std::string fix_win_path(const std::string& path) - { -#ifdef _WIN32 - if (util::starts_with(path, "file:")) - { - std::regex re(R"(\\(?! ))"); - std::string res = std::regex_replace(path, re, R"(/)"); - util::replace_all(res, ":////", "://"); - return res; - } - else - { - return path; - } -#else - return path; -#endif - } } diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index ed5596fe5f..d21f9c3560 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -69,7 +69,6 @@ set(LIBMAMBA_TEST_SRCS src/core/test_validate.cpp src/core/test_virtual_packages.cpp src/core/test_util.cpp - src/core/test_util_os.cpp src/core/test_system_env.cpp src/core/test_env_lockfile.cpp src/core/test_execution.cpp diff --git a/libmamba/tests/src/core/test_util_os.cpp b/libmamba/tests/src/core/test_util_os.cpp deleted file mode 100644 index 6c4e080e4f..0000000000 --- a/libmamba/tests/src/core/test_util_os.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2023, QuantStack and Mamba Contributors -// -// Distributed under the terms of the BSD 3-Clause License. -// -// The full license is in the file LICENSE, distributed with this software. - -#include - -#include - -#include "mamba/core/util_os.hpp" - -#ifdef _WIN32 - -TEST_SUITE("windows_path") -{ - TEST_CASE("fix_win_path") - { - std::string test_str("file://\\unc\\path\\on\\win"); - auto out = mamba::fix_win_path(test_str); - CHECK_EQ(out, "file:///unc/path/on/win"); - auto out2 = mamba::fix_win_path("file://C:\\Program\\ (x74)\\Users\\hello\\ world"); - CHECK_EQ(out2, "file://C:/Program\\ (x74)/Users/hello\\ world"); - auto out3 = mamba::fix_win_path("file://\\\\Programs\\xyz"); - CHECK_EQ(out3, "file://Programs/xyz"); - } -} - -#endif