diff --git a/libs/module/DynamicLibrary.cpp b/libs/module/DynamicLibrary.cpp index ae81282b9b..ac3c1d616e 100644 --- a/libs/module/DynamicLibrary.cpp +++ b/libs/module/DynamicLibrary.cpp @@ -59,7 +59,7 @@ DynamicLibrary::FunctionPointer DynamicLibrary::findSymbol(const std::string& sy std::string DynamicLibrary::getName() const { - return string::to_utf8(_name); + return string::unicode_to_utf8(_name); } /** diff --git a/libs/string/encoding.h b/libs/string/encoding.h index a082678115..9bc316b73c 100644 --- a/libs/string/encoding.h +++ b/libs/string/encoding.h @@ -7,9 +7,71 @@ namespace string { -inline std::string to_utf8(const std::wstring& wstring) +inline std::string unicode_to_utf8(const std::wstring& wstring) { return std::wstring_convert, wchar_t>().to_bytes(wstring); } +inline std::wstring utf8_to_unicode(const std::string& utf8) +{ + return std::wstring_convert< std::codecvt_utf8 >().from_bytes(utf8); +} + +inline std::wstring mb_to_unicode(const std::string& str) +{ + std::wstring ret; + std::mbstate_t state = {}; + const char* src = str.data(); + + size_t len = std::mbsrtowcs(nullptr, &src, 0, &state); + + if (len != static_cast(-1)) + { + std::vector buff(len + 1); + + len = std::mbsrtowcs(buff.data(), &src, len, &state); + + if (len != static_cast(-1)) + { + ret.assign(buff.data(), len); + } + } + + return ret; +} + +inline std::string unicode_to_mb(const std::wstring& wstr) +{ + std::string ret; + std::mbstate_t state = {}; + + const wchar_t* src = wstr.data(); + + size_t len = std::wcsrtombs(nullptr, &src, 0, &state); + + if (len != static_cast(-1)) + { + std::vector buff(len + 1); + + len = std::wcsrtombs(buff.data(), &src, len, &state); + + if (len != static_cast(-1)) + { + ret.assign(buff.data(), len); + } + } + + return ret; +} + +inline std::string utf8_to_mb(const std::string& input) +{ + return unicode_to_mb(utf8_to_unicode(input)); +} + +inline std::string mb_to_utf8(const std::string& input) +{ + return unicode_to_utf8(mb_to_unicode(input)); +} + } diff --git a/libs/wxutil/IConv.h b/libs/wxutil/IConv.h index cb4c8d70cd..dd6bb9a967 100644 --- a/libs/wxutil/IConv.h +++ b/libs/wxutil/IConv.h @@ -1,7 +1,10 @@ #pragma once #include +#if 0 #include +#endif +#include "string/encoding.h" #include @@ -25,8 +28,12 @@ class IConv */ static std::string localeToUTF8(const std::string& input) { +#if 1 + return string::mb_to_utf8(input); +#else wxString inp(input); return inp.ToUTF8().data(); +#endif } /** @@ -35,7 +42,11 @@ class IConv */ static std::string localeFromUTF8(const std::string& input) { +#if 1 + return string::utf8_to_mb(input); +#else return wxString::FromUTF8(input.c_str()).ToStdString(); +#endif } }; diff --git a/radiant/ui/prefdialog/GameSetupPageTdm.cpp b/radiant/ui/prefdialog/GameSetupPageTdm.cpp index 0507b3ef05..e0c4ad384e 100644 --- a/radiant/ui/prefdialog/GameSetupPageTdm.cpp +++ b/radiant/ui/prefdialog/GameSetupPageTdm.cpp @@ -230,7 +230,7 @@ void GameSetupPageTdm::validateSettings() } catch (const std::system_error& ex) { - rWarning() << "[vfs] Skipping file " << string::to_utf8(path.filename().wstring()) << + rWarning() << "[vfs] Skipping file " << string::unicode_to_utf8(path.filename().wstring()) << " - possibly unsupported characters in filename? " << "(Exception: " << ex.what() << ")" << std::endl; } diff --git a/radiantcore/vfs/DirectoryArchive.cpp b/radiantcore/vfs/DirectoryArchive.cpp index 4e3d2e47f5..0515034afb 100644 --- a/radiantcore/vfs/DirectoryArchive.cpp +++ b/radiantcore/vfs/DirectoryArchive.cpp @@ -92,7 +92,7 @@ void DirectoryArchive::traverse(Visitor& visitor, const std::string& root) } catch (const std::system_error& ex) { - rWarning() << "[vfs] Skipping file " << string::to_utf8(candidate.filename().wstring()) << + rWarning() << "[vfs] Skipping file " << string::unicode_to_utf8(candidate.filename().wstring()) << " - possibly unsupported characters in filename? " << "(Exception: " << ex.what() << ")" << std::endl; } diff --git a/radiantcore/vfs/Doom3FileSystem.cpp b/radiantcore/vfs/Doom3FileSystem.cpp index 066baec73c..ac726c51b2 100644 --- a/radiantcore/vfs/Doom3FileSystem.cpp +++ b/radiantcore/vfs/Doom3FileSystem.cpp @@ -269,7 +269,7 @@ void Doom3FileSystem::initDirectory(const std::string& inputPath) } catch (std::system_error& ex) { - rWarning() << "[vfs] Skipping file " << string::to_utf8(file.filename().wstring()) << + rWarning() << "[vfs] Skipping file " << string::unicode_to_utf8(file.filename().wstring()) << " - possibly unsupported characters in filename? " << "(Exception: " << ex.what() << ")" << std::endl; } diff --git a/radiantcore/xmlregistry/XMLRegistry.cpp b/radiantcore/xmlregistry/XMLRegistry.cpp index 607fce53cd..417daef79c 100644 --- a/radiantcore/xmlregistry/XMLRegistry.cpp +++ b/radiantcore/xmlregistry/XMLRegistry.cpp @@ -9,7 +9,7 @@ #include "version.h" #include "string/string.h" -#include "wxutil/IConv.h" +#include "string/encoding.h" #include "module/StaticModule.h" namespace registry @@ -191,7 +191,7 @@ std::string XMLRegistry::get(const std::string& key) if (!nodeList.empty()) { // Convert the UTF-8 string back to locale and return - return wxutil::IConv::localeFromUTF8(nodeList[0].getAttributeValue("value")); + return string::utf8_to_mb(nodeList[0].getAttributeValue("value")); } return std::string(); @@ -203,7 +203,7 @@ void XMLRegistry::set(const std::string& key, const std::string& value) // Create or set the value in the user tree, the default tree stays untouched // Convert the string to UTF-8 before storing it into the RegistryTree - _userTree.set(key, wxutil::IConv::localeToUTF8(value)); + _userTree.set(key, string::mb_to_utf8(value)); _changesSinceLastSave++;