shared: replace ToNarrow/ToWide wuth utfcpp for runtime safety and portability

Author: chromium

.gitmodules

@@ -109,3 +109,6 @@
 [submodule "vendor/eastl"]
 	path = vendor/eastl
 	url = https://github.com/electronicarts/EASTL.git
+[submodule "vendor/utfcpp"]
+	path = vendor/utfcpp
+	url = https://github.com/nemtrif/utfcpp.git

code/client/shared/Utils.cpp

@@ -10,6 +10,7 @@
 #include <sstream>
 #include <iomanip>
 #include <mutex>
+#include <utf8.h>
 #include "Error.h"
 
 static STATIC InitFunctionBase* g_initFunctions;
@@ -349,12 +350,34 @@ bool UrlDecode(const std::string& in, std::string& out)
 
 std::string ToNarrow(const std::wstring& wide)
 {
-	static std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> g_converter;
-	return g_converter.to_bytes(wide);
+	// TODO: replace with something faster if needed
+	std::vector<char> outVec;
+	outVec.reserve(wide.size());
+
+#ifdef _WIN32
+	utf8::utf16to8(wide.begin(), wide.end(), std::back_inserter(outVec));
+#else
+	utf8::utf32to8(wide.begin(), wide.end(), std::back_inserter(outVec));
+#endif
+	
+	return std::string(outVec.begin(), outVec.end());
 }
 
 std::wstring ToWide(const std::string& narrow)
 {
-	static std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> g_converter;
-	return g_converter.from_bytes(narrow);
+	std::vector<uint8_t> cleanVec;
+	cleanVec.reserve(narrow.size());
+
+	utf8::replace_invalid(narrow.begin(), narrow.end(), std::back_inserter(cleanVec));
+
+	std::vector<wchar_t> outVec;
+	outVec.reserve(cleanVec.size());
+
+#ifdef _WIN32
+	utf8::utf8to16(cleanVec.begin(), cleanVec.end(), std::back_inserter(outVec));
+#else
+	utf8::utf8to32(cleanVec.begin(), cleanVec.end(), std::back_inserter(outVec));
+#endif
+
+	return std::wstring(outVec.begin(), outVec.end());
 }

code/client/shared/premake5.lua

@@ -13,6 +13,8 @@
 			else
 				add_dependencies { 'vendor:fmtlib' }
 			end
+			
+			add_dependencies { 'vendor:utfcpp' }
 
 			if os.is('windows') then
 				links { libc and 'fmtlib-crt' or 'fmtlib' }

code/vendor/config.lua

@@ -48,5 +48,5 @@ vendor_component 'openssl_crypto'
 vendor_component 'openssl_ssl'
 vendor_component 'minizip'
 vendor_component 'glm'
-
+vendor_component 'utfcpp'
 vendor_component 'eastl'

code/vendor/utfcpp.lua

@@ -0,0 +1,17 @@
+return {
+	include = function()
+		includedirs {
+			"../vendor/utfcpp/source/"
+		}
+	end,
+	
+	run = function()
+		targetname 'utfcpp_dummy'
+		language 'C'
+		kind 'StaticLib'
+		
+		files {
+			'vendor/dummy.c'
+		}
+	end
+}