Make sure that all strings in the IDO database are UTF8-encoded

fixes #10554

lib/base/utility.cpp

@@ -1654,3 +1654,39 @@ String Utility::GetPlatformArchitecture(void)
 	return UnameHelper('m');
 #endif /* _WIN32 */
 }
+
+String Utility::ValidateUTF8(const String& input)
+{
+	String output;
+	size_t length = input.GetLength();
+
+	for (size_t i = 0; i < length; i++) {
+		if ((input[i] & 0x80) == 0) {
+			output += input[i];
+			continue;
+		}
+
+		if ((input[i] & 0xE0) == 0xC0 && length > i + 1 &&
+		    (input[i + 1] & 0xC0) == 0x80) {
+			output += input[i];
+			output += input[i + 1];
+			i++;
+			continue;
+		}
+
+		if ((input[i] & 0xF0) == 0xE0 && length > i + 2 &&
+		    (input[i + 1] & 0xC0) == 0x80 && (input[i + 2] & 0xC0) == 0x80) {
+			output += input[i];
+			output += input[i + 1];
+			output += input[i + 2];
+			i += 2;
+			continue;
+		}
+
+		output += '\xEF';
+		output += '\xBF';
+		output += '\xBD';
+	}
+
+	return output;
+}

lib/base/utility.hpp

@@ -139,6 +139,8 @@ class I2_BASE_API Utility
 	static String GetPlatformVersion(void);
 	static String GetPlatformArchitecture(void);
 
+	static String ValidateUTF8(const String& input);
+
 private:
 	Utility(void);
 	static void CollectPaths(const String& path, std::vector<String>& paths);

lib/db_ido_mysql/idomysqlconnection.cpp

@@ -574,10 +574,12 @@ String IdoMysqlConnection::Escape(const String& s)
 {
 	AssertOnWorkQueue();
 
-	size_t length = s.GetLength();
-	char *to = new char[s.GetLength() * 2 + 1];
+	String utf8s = Utility::ValidateUTF8(s);
 
-	mysql_real_escape_string(&m_Connection, to, s.CStr(), length);
+	size_t length = utf8s.GetLength();
+	char *to = new char[utf8s.GetLength() * 2 + 1];
+
+	mysql_real_escape_string(&m_Connection, to, utf8s.CStr(), length);
 
 	String result = String(to);
 

lib/db_ido_pgsql/idopgsqlconnection.cpp

@@ -464,10 +464,12 @@ String IdoPgsqlConnection::Escape(const String& s)
 {
 	AssertOnWorkQueue();
 
-	size_t length = s.GetLength();
-	char *to = new char[s.GetLength() * 2 + 1];
+	String utf8s = Utility::ValidateUTF8(s);
 
-	PQescapeStringConn(m_Connection, to, s.CStr(), length, NULL);
+	size_t length = utf8s.GetLength();
+	char *to = new char[utf8s.GetLength() * 2 + 1];
+
+	PQescapeStringConn(m_Connection, to, utf8s.CStr(), length, NULL);
 
 	String result = String(to);