[fix] Added full-scale UTF-8 sanitization; no more crashes.

This resolves the inflamous "Bad UTF-8 continuation byte" error dialog. Fixes #237 #385 #664 #445 #364

Incorporated UTFCPP library: (utfcpp.sourceforge.net/; License: boost) and added UTF-8 sanitization to all text-file processing.

@InvalidUTF8character: YOU SHALL NOT PASS! :D

DEPENDENCIES.md

@@ -17,6 +17,7 @@ Rigs of Rods uses 3rd party libraries licensed under the following licenses:
 | boost                           | advanced C++ functions                               | Boost Software License 1.0 |
 | wxWidgets                       | cross-platform GUI system                            | wxWindows 3.1              |
 | CrashRpt                        | Crash reporting system                               | BSD New                    |
+| UTFCpp                          | UTF-8 validation/conversion                          | Boost Software License 1.0 | 
 
 
 

source/main/audio/SoundScriptManager.cpp

@@ -25,6 +25,7 @@ along with Rigs of Rods.  If not, see <http://www.gnu.org/licenses/>.
 #include "Settings.h"
 #include "Sound.h"
 #include "SoundManager.h"
+#include "Utils.h"
 
 // some gcc fixes
 #if OGRE_PLATFORM == OGRE_PLATFORM_LINUX
@@ -413,7 +414,7 @@ void SoundScriptManager::parseScript(DataStreamPtr& stream, const String& groupN
 
 	while(!stream->eof())
 	{
-		line = stream->getLine();
+		line = RoR::Utils::SanitizeUtf8String(stream->getLine());
 		// ignore comments & blanks
 		if (!(line.length() == 0 || line.substr(0,2) == "//"))
 		{

source/main/gameplay/SkinManager.cpp

@@ -20,6 +20,7 @@
 */
 
 #include "SkinManager.h"
+#include "Utils.h"
 
 using namespace Ogre;
 using namespace RoR;
@@ -69,7 +70,7 @@ void SkinManager::parseScript(DataStreamPtr& stream, const String& groupName)
 
 		while(!stream->eof())
 		{
-			line = stream->getLine();
+			line = RoR::Utils::SanitizeUtf8String(stream->getLine());
 
 			// Ignore blanks & comments
 			if (!line.length() || line.substr(0, 2) == "//")

source/main/gameplay/TorqueCurve.cpp

@@ -19,6 +19,7 @@ along with Rigs of Rods.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "TorqueCurve.h"
+#include "Utils.h"
 
 #include <Ogre.h>
 
@@ -69,7 +70,7 @@ int TorqueCurve::loadDefaultTorqueModels()
 
 	while (!ds->eof())
 	{
-		line=ds->getLine();
+		line = RoR::Utils::SanitizeUtf8String(ds->getLine());
 		StringUtil::trim(line);
 
 		if (line.empty() || line[0]==';')

source/main/gfx/Skidmark.cpp

@@ -24,6 +24,7 @@ along with Rigs of Rods.  If not, see <http://www.gnu.org/licenses/>.
 #include "BeamData.h"
 #include "IHeightFinder.h"
 #include "Settings.h"
+#include "Utils.h"
 
 using namespace Ogre;
 
@@ -65,7 +66,7 @@ int SkidmarkManager::loadDefaultModels()
 
 	while (!ds->eof())
 	{	
-		line=ds->getLine();
+		line = RoR::Utils::SanitizeUtf8String(ds->getLine());
 		StringUtil::trim(line);
 
 		if (line.empty() || line[0]==';')

source/main/network/IRCWrapper.cpp

@@ -306,29 +306,29 @@ int IRCWrapper::processAuthenticationResults(String &results)
 	// fatal error?
 	if (cfg.hasSetting("fatalError"))
 	{
-		ErrorUtils::ShowOgreWebError(cfg.getSetting("fatalErrorTitle"), cfg.getSetting("fatalError"), cfg.getSetting("fatalErrorURL"));
+		ErrorUtils::ShowOgreWebError(cfg.GetString("fatalErrorTitle"), cfg.GetString("fatalError"), cfg.GetString("fatalErrorURL"));
 		return 1;
 	}
 
 	// non-fatal error?
 	if (cfg.hasSetting("error"))
 	{
-		push(constructMessage(MT_ErrorAuth, 0, 0, cfg.getSetting("error").c_str()));
+		push(constructMessage(MT_ErrorAuth, 0, 0, cfg.GetString("error").c_str()));
 		return 1;
 	}
 
 	if (!cfg.hasSetting("serverName") || !cfg.hasSetting("serverPort"))
 		return 1;
 
-	serverName     = cfg.getSetting("serverName");
-	serverPort     = cfg.getSettingInt("serverPort");
-	serverPassword = cfg.getSetting("serverPassword");
-	nick           = cfg.getSetting("nick");
-	userName       = cfg.getSetting("userName");
-	realName       = cfg.getSetting("realName");
-	channel        = cfg.getSetting("channel");
-	reJoin         = cfg.getSettingBool("reJoin");
-	reConnect      = cfg.getSettingBool("reConnect");
+	serverName     = cfg.GetString("serverName");
+	serverPort     = cfg.GetInt   ("serverPort");
+	serverPassword = cfg.GetString("serverPassword");
+	nick           = cfg.GetString("nick");
+	userName       = cfg.GetString("userName");
+	realName       = cfg.GetString("realName");
+	channel        = cfg.GetString("channel");
+	reJoin         = cfg.GetBool  ("reJoin");
+	reConnect      = cfg.GetBool  ("reConnect");
 
 	// TODO:
 	//userAuth = cfg.getSetting("userAuth");

source/main/physics/CmdKeyInertia.cpp

@@ -18,6 +18,7 @@ You should have received a copy of the GNU General Public License
 along with Rigs of Rods.  If not, see <http://www.gnu.org/licenses/>.
 */
 #include "CmdKeyInertia.h"
+#include "Utils.h"
 
 #include <Ogre.h>
 
@@ -139,7 +140,7 @@ int CmdKeyInertia::loadDefaultInertiaModels()
 
 	while (!ds->eof())
 	{
-		line=ds->getLine();
+		line=RoR::Utils::SanitizeUtf8String(ds->getLine());
 		StringUtil::trim(line);
 
 		if (line.empty() || line[0]==';')

source/main/resources/CacheSystem.cpp

@@ -246,7 +246,6 @@ CacheSystem::CacheValidityState CacheSystem::IsCacheValid()
 {
 	String cfgfilename = getCacheConfigFilename(false);
 	ImprovedConfigFile cfg;
-	ConfigFile ff;
 	if (!resourceExistsInAllGroups(cfgfilename))
 	{
 		LOG("unable to load config file: "+cfgfilename);
@@ -256,8 +255,8 @@ CacheSystem::CacheValidityState CacheSystem::IsCacheValid()
 	String group = ResourceGroupManager::getSingleton().findGroupContainingResource(cfgfilename);
 	DataStreamPtr stream=ResourceGroupManager::getSingleton().openResource(cfgfilename, group);
 	cfg.load(stream, "\t:=", false);
-	String shaone = cfg.getSetting("shaone");
-	String cacheformat = cfg.getSetting("cacheformat");
+	String shaone = cfg.GetString("shaone");
+	String cacheformat = cfg.GetString("cacheformat");
 
 	if (shaone == "" || shaone != currentSHA1)
 	{

source/main/scripting/LocalStorage.cpp

@@ -122,7 +122,7 @@ std::string LocalStorage::get(std::string &key)
 {
 	std::string sec;
 	parseKey(key, sec);
-	return getSetting(key, sec);
+	return GetStringEx(key, sec);
 }
 
 void LocalStorage::set(std::string &key, const std::string &value)

source/main/terrain/TerrainGeometryManager.cpp

@@ -24,6 +24,7 @@ along with Rigs of Rods.  If not, see <http://www.gnu.org/licenses/>.
 #include "TerrainManager.h"
 #include "ShadowManager.h"
 #include "OgreTerrainPSSMMaterialGenerator.h"
+#include "Utils.h"
 
 using namespace Ogre;
 
@@ -115,7 +116,7 @@ Ogre::String TerrainGeometryManager::getPageHeightmap(int x, int z)
 
 	char buf[4096];
 	ds->readLine(buf, 4096);
-	return String(buf);
+	return RoR::Utils::SanitizeUtf8String(String(buf));
 }
 
 void TerrainGeometryManager::initTerrain()
@@ -312,11 +313,11 @@ void TerrainGeometryManager::loadLayers(int x, int z, Terrain *terrain)
 	if (ds.isNull())
 		return;
 
-	char line[4096];
-	ds->readLine(line, 4096);
-	String heightmapImage = String(line);
-	ds->readLine(line, 4096);
-	terrainLayers = PARSEINT(String(line));
+	char line_buf[4096];
+	ds->readLine(line_buf, 4096);
+	String heightmapImage = RoR::Utils::SanitizeUtf8String(String(line_buf));
+	ds->readLine(line_buf, 4096);
+	terrainLayers = PARSEINT(RoR::Utils::SanitizeUtf8String(String(line_buf)));
 
 	if (terrainLayers == 0)
 		return;
@@ -333,7 +334,8 @@ void TerrainGeometryManager::loadLayers(int x, int z, Terrain *terrain)
 
 	while (!ds->eof())
 	{
-		size_t ll = ds->readLine(line, 4096);
+		size_t ll = ds->readLine(line_buf, 4096);
+		std::string line = RoR::Utils::SanitizeUtf8String(std::string(line_buf));
 		if (ll==0 || line[0]=='/' || line[0]==';') continue;
 
 		StringVector args = StringUtil::split(String(line), ",");

source/main/terrain/TerrainManager.cpp

@@ -1,25 +1,25 @@
 /*
-This source file is part of Rigs of Rods
-Copyright 2005-2012 Pierre-Michel Ricordel
-Copyright 2007-2012 Thomas Fischer
+    This source file is part of Rigs of Rods
+    Copyright 2005-2012 Pierre-Michel Ricordel
+    Copyright 2007-2012 Thomas Fischer
+    Copyright 2013-2016 Petr Ohlidal
 
-For more information, see http://www.rigsofrods.com/
+    For more information, see http://www.rigsofrods.com/
 
-Rigs of Rods is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License version 3, as
-published by the Free Software Foundation.
+    Rigs of Rods is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License version 3, as
+    published by the Free Software Foundation.
 
-Rigs of Rods is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
+    Rigs of Rods is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+    GNU General Public License for more details.
 
-You should have received a copy of the GNU General Public License
-along with Rigs of Rods.  If not, see <http://www.gnu.org/licenses/>.
+    You should have received a copy of the GNU General Public License
+    along with Rigs of Rods. If not, see <http://www.gnu.org/licenses/>.
 */
-#include "TerrainManager.h"
 
-#include <Terrain/OgreTerrainPaging.h>
+#include "TerrainManager.h"
 
 #include "BeamData.h"
 #include "BeamFactory.h"
@@ -43,9 +43,11 @@ along with Rigs of Rods.  If not, see <http://www.gnu.org/licenses/>.
 #include "Utils.h"
 #include "Water.h"
 
+#include <Terrain/OgreTerrainPaging.h>
+
 using namespace Ogre;
 
-TerrainManager::TerrainManager() : 
+TerrainManager::TerrainManager() :
 	  m_terrain_config()
 	, character(0)
 	, collisions(0)
@@ -145,27 +147,27 @@ void TerrainManager::loadTerrainConfigBasics(Ogre::DataStreamPtr &ds)
 	m_terrain_config.load(ds, "\t:=", true);
 
 	// read in the settings
-	terrain_name = m_terrain_config.getSetting("Name", "General");
+	terrain_name = m_terrain_config.GetStringEx("Name", "General");
 	if (terrain_name.empty())
 	{
 		ErrorUtils::ShowError(_L("Terrain loading error"), _L("the terrain name cannot be empty"));
 		exit(125);
 	}
 
-	ogre_terrain_config_filename = m_terrain_config.getSetting("GeometryConfig", "General");
+	ogre_terrain_config_filename = m_terrain_config.GetStringEx("GeometryConfig", "General");
 	// otc = ogre terrain config
 	if (ogre_terrain_config_filename.find(".otc") == String::npos)
 	{
 		ErrorUtils::ShowError(_L("Terrain loading error"), _L("the new terrain mode only supports .otc configurations"));
 		exit(125);
 	}
 
-	ambient_color = StringConverter::parseColourValue(m_terrain_config.getSetting("AmbientColor", "General"), ColourValue::White);
-	category_id = StringConverter::parseInt(m_terrain_config.getSetting("CategoryID", "General"), 129);
-	guid = m_terrain_config.getSetting("GUID", "General");
-	start_position = StringConverter::parseVector3(m_terrain_config.getSetting("StartPosition", "General"), Vector3(512.0f, 0.0f, 512.0f));
-	version = StringConverter::parseInt(m_terrain_config.getSetting("Version", "General"), 1);
-	gravity = StringConverter::parseReal(m_terrain_config.getSetting("Gravity", "General"), -9.81);
+	ambient_color  = m_terrain_config.GetColourValue("AmbientColor", "General", ColourValue::White);
+	category_id    = m_terrain_config.GetInt("CategoryID", "General", 129);
+	guid           = m_terrain_config.GetStringEx("GUID", "General");
+	start_position = StringConverter::parseVector3(m_terrain_config.GetStringEx("StartPosition", "General"), Vector3(512.0f, 0.0f, 512.0f));
+	version        = m_terrain_config.GetInt("Version", "General", 1);
+	gravity        = m_terrain_config.GetFloat("Gravity", "General", -9.81);
 
 	// parse author info
 	ConfigFile::SettingsIterator it = m_terrain_config.getSettingsIterator("Authors");
@@ -174,8 +176,8 @@ void TerrainManager::loadTerrainConfigBasics(Ogre::DataStreamPtr &ds)
 
 	while (it.hasMoreElements())
 	{
-		String type = it.peekNextKey();   // e.g. terrain
-		String name = it.peekNextValue(); // e.g. john doe
+		String type = RoR::Utils::SanitizeUtf8String(it.peekNextKey());   // e.g. terrain
+		String name = RoR::Utils::SanitizeUtf8String(it.peekNextValue()); // e.g. john doe
 
 		if (!name.empty())
 		{
@@ -356,13 +358,13 @@ void TerrainManager::initSkySubSystem()
 		gEnv->sky = sky_manager;
 
 		// try to load caelum config
-		String caelumConfig = m_terrain_config.getSetting("CaelumConfigFile", "General");
+		String caelumConfig = m_terrain_config.GetStringEx("CaelumConfigFile", "General");
 
 		if (!caelumConfig.empty() && ResourceGroupManager::getSingleton().resourceExistsInAnyGroup(caelumConfig))
 		{
 			// config provided and existing, use it :)
-			int caelumFogStart = StringConverter::parseInt(m_terrain_config.getSetting("CaelumFogStart", "General"),-1);
-			int caelumFogEnd = StringConverter::parseInt(m_terrain_config.getSetting("CaelumFogEnd", "General"),-1);
+			int caelumFogStart = m_terrain_config.GetInt("CaelumFogStart", "General",-1);
+			int caelumFogEnd   = m_terrain_config.GetInt("CaelumFogEnd",   "General",-1);
 			sky_manager->loadScript(caelumConfig, caelumFogStart, caelumFogEnd);
 		} else
 		{
@@ -373,7 +375,7 @@ void TerrainManager::initSkySubSystem()
 	} else
 #endif //USE_CAELUM
 	{
-		String sandStormConfig = m_terrain_config.getSetting("SandStormCubeMap", "General");
+		String sandStormConfig = m_terrain_config.GetStringEx("SandStormCubeMap", "General");
 
 		if (!sandStormConfig.empty())
 		{
@@ -587,12 +589,16 @@ void TerrainManager::initWater()
 	// disabled in global config
 	if (waterSettingsString == "None") return;
 	// disabled in map config
-	if (!StringConverter::parseBool(m_terrain_config.getSetting("Water", "General"))) return;
+    bool has_water = m_terrain_config.GetBool("Water", "General", false);
+    if (!has_water)
+    {
+        return;
+    }
 
 	if (waterSettingsString == "Hydrax")
 	{
 		// try to load hydrax config
-		String hydraxConfig = m_terrain_config.getSetting("HydraxConfigFile", "General");
+		String hydraxConfig = m_terrain_config.GetStringEx("HydraxConfigFile", "General");
 
 		if (!hydraxConfig.empty() && ResourceGroupManager::getSingleton().resourceExistsInAnyGroup(hydraxConfig))
 		{
@@ -675,7 +681,7 @@ void TerrainManager::initCollisions()
 
 void TerrainManager::initTerrainCollisions()
 {
-	String tractionMapConfig = m_terrain_config.getSetting("TractionMap", "General");
+	String tractionMapConfig = m_terrain_config.GetStringEx("TractionMap", "General");
 	if (!tractionMapConfig.empty())
 	{
 		gEnv->collisions->setupLandUse(tractionMapConfig.c_str());