diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index c7c839f7f4..ebf3570830 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -17,6 +17,7 @@ Rigs of Rods uses 3rd party libraries licensed under the following licenses: | boost | advanced C++ functions | Boost Software License 1.0 | | wxWidgets | cross-platform GUI system | wxWindows 3.1 | | CrashRpt | Crash reporting system | BSD New | +| UTFCpp | UTF-8 validation/conversion | Boost Software License 1.0 | diff --git a/source/main/audio/SoundScriptManager.cpp b/source/main/audio/SoundScriptManager.cpp index ee7d503f30..c6089398e6 100644 --- a/source/main/audio/SoundScriptManager.cpp +++ b/source/main/audio/SoundScriptManager.cpp @@ -25,6 +25,7 @@ along with Rigs of Rods. If not, see . #include "Settings.h" #include "Sound.h" #include "SoundManager.h" +#include "Utils.h" // some gcc fixes #if OGRE_PLATFORM == OGRE_PLATFORM_LINUX @@ -413,7 +414,7 @@ void SoundScriptManager::parseScript(DataStreamPtr& stream, const String& groupN while(!stream->eof()) { - line = stream->getLine(); + line = RoR::Utils::SanitizeUtf8String(stream->getLine()); // ignore comments & blanks if (!(line.length() == 0 || line.substr(0,2) == "//")) { diff --git a/source/main/gameplay/SkinManager.cpp b/source/main/gameplay/SkinManager.cpp index beb98f6e4d..792481beca 100644 --- a/source/main/gameplay/SkinManager.cpp +++ b/source/main/gameplay/SkinManager.cpp @@ -20,6 +20,7 @@ */ #include "SkinManager.h" +#include "Utils.h" using namespace Ogre; using namespace RoR; @@ -69,7 +70,7 @@ void SkinManager::parseScript(DataStreamPtr& stream, const String& groupName) while(!stream->eof()) { - line = stream->getLine(); + line = RoR::Utils::SanitizeUtf8String(stream->getLine()); // Ignore blanks & comments if (!line.length() || line.substr(0, 2) == "//") diff --git a/source/main/gameplay/TorqueCurve.cpp b/source/main/gameplay/TorqueCurve.cpp index 029a4d5328..f610e0a8a7 100644 --- a/source/main/gameplay/TorqueCurve.cpp +++ b/source/main/gameplay/TorqueCurve.cpp @@ -19,6 +19,7 @@ along with Rigs of Rods. If not, see . */ #include "TorqueCurve.h" +#include "Utils.h" #include @@ -69,7 +70,7 @@ int TorqueCurve::loadDefaultTorqueModels() while (!ds->eof()) { - line=ds->getLine(); + line = RoR::Utils::SanitizeUtf8String(ds->getLine()); StringUtil::trim(line); if (line.empty() || line[0]==';') diff --git a/source/main/gfx/Skidmark.cpp b/source/main/gfx/Skidmark.cpp index ee8549b8f8..e0f95eaff5 100644 --- a/source/main/gfx/Skidmark.cpp +++ b/source/main/gfx/Skidmark.cpp @@ -24,6 +24,7 @@ along with Rigs of Rods. If not, see . #include "BeamData.h" #include "IHeightFinder.h" #include "Settings.h" +#include "Utils.h" using namespace Ogre; @@ -65,7 +66,7 @@ int SkidmarkManager::loadDefaultModels() while (!ds->eof()) { - line=ds->getLine(); + line = RoR::Utils::SanitizeUtf8String(ds->getLine()); StringUtil::trim(line); if (line.empty() || line[0]==';') diff --git a/source/main/network/IRCWrapper.cpp b/source/main/network/IRCWrapper.cpp index d4479c371e..77ebca688c 100644 --- a/source/main/network/IRCWrapper.cpp +++ b/source/main/network/IRCWrapper.cpp @@ -306,29 +306,29 @@ int IRCWrapper::processAuthenticationResults(String &results) // fatal error? if (cfg.hasSetting("fatalError")) { - ErrorUtils::ShowOgreWebError(cfg.getSetting("fatalErrorTitle"), cfg.getSetting("fatalError"), cfg.getSetting("fatalErrorURL")); + ErrorUtils::ShowOgreWebError(cfg.GetString("fatalErrorTitle"), cfg.GetString("fatalError"), cfg.GetString("fatalErrorURL")); return 1; } // non-fatal error? if (cfg.hasSetting("error")) { - push(constructMessage(MT_ErrorAuth, 0, 0, cfg.getSetting("error").c_str())); + push(constructMessage(MT_ErrorAuth, 0, 0, cfg.GetString("error").c_str())); return 1; } if (!cfg.hasSetting("serverName") || !cfg.hasSetting("serverPort")) return 1; - serverName = cfg.getSetting("serverName"); - serverPort = cfg.getSettingInt("serverPort"); - serverPassword = cfg.getSetting("serverPassword"); - nick = cfg.getSetting("nick"); - userName = cfg.getSetting("userName"); - realName = cfg.getSetting("realName"); - channel = cfg.getSetting("channel"); - reJoin = cfg.getSettingBool("reJoin"); - reConnect = cfg.getSettingBool("reConnect"); + serverName = cfg.GetString("serverName"); + serverPort = cfg.GetInt ("serverPort"); + serverPassword = cfg.GetString("serverPassword"); + nick = cfg.GetString("nick"); + userName = cfg.GetString("userName"); + realName = cfg.GetString("realName"); + channel = cfg.GetString("channel"); + reJoin = cfg.GetBool ("reJoin"); + reConnect = cfg.GetBool ("reConnect"); // TODO: //userAuth = cfg.getSetting("userAuth"); diff --git a/source/main/physics/CmdKeyInertia.cpp b/source/main/physics/CmdKeyInertia.cpp index 8c934ffa2b..4d7f355ff1 100644 --- a/source/main/physics/CmdKeyInertia.cpp +++ b/source/main/physics/CmdKeyInertia.cpp @@ -18,6 +18,7 @@ You should have received a copy of the GNU General Public License along with Rigs of Rods. If not, see . */ #include "CmdKeyInertia.h" +#include "Utils.h" #include @@ -139,7 +140,7 @@ int CmdKeyInertia::loadDefaultInertiaModels() while (!ds->eof()) { - line=ds->getLine(); + line=RoR::Utils::SanitizeUtf8String(ds->getLine()); StringUtil::trim(line); if (line.empty() || line[0]==';') diff --git a/source/main/resources/CacheSystem.cpp b/source/main/resources/CacheSystem.cpp index d75bc8c051..fbb792a138 100644 --- a/source/main/resources/CacheSystem.cpp +++ b/source/main/resources/CacheSystem.cpp @@ -246,7 +246,6 @@ CacheSystem::CacheValidityState CacheSystem::IsCacheValid() { String cfgfilename = getCacheConfigFilename(false); ImprovedConfigFile cfg; - ConfigFile ff; if (!resourceExistsInAllGroups(cfgfilename)) { LOG("unable to load config file: "+cfgfilename); @@ -256,8 +255,8 @@ CacheSystem::CacheValidityState CacheSystem::IsCacheValid() String group = ResourceGroupManager::getSingleton().findGroupContainingResource(cfgfilename); DataStreamPtr stream=ResourceGroupManager::getSingleton().openResource(cfgfilename, group); cfg.load(stream, "\t:=", false); - String shaone = cfg.getSetting("shaone"); - String cacheformat = cfg.getSetting("cacheformat"); + String shaone = cfg.GetString("shaone"); + String cacheformat = cfg.GetString("cacheformat"); if (shaone == "" || shaone != currentSHA1) { diff --git a/source/main/scripting/LocalStorage.cpp b/source/main/scripting/LocalStorage.cpp index b0da4f4236..f2bf3cee74 100644 --- a/source/main/scripting/LocalStorage.cpp +++ b/source/main/scripting/LocalStorage.cpp @@ -122,7 +122,7 @@ std::string LocalStorage::get(std::string &key) { std::string sec; parseKey(key, sec); - return getSetting(key, sec); + return GetStringEx(key, sec); } void LocalStorage::set(std::string &key, const std::string &value) diff --git a/source/main/terrain/TerrainGeometryManager.cpp b/source/main/terrain/TerrainGeometryManager.cpp index 1902c6ae4b..64fecf4b5d 100644 --- a/source/main/terrain/TerrainGeometryManager.cpp +++ b/source/main/terrain/TerrainGeometryManager.cpp @@ -24,6 +24,7 @@ along with Rigs of Rods. If not, see . #include "TerrainManager.h" #include "ShadowManager.h" #include "OgreTerrainPSSMMaterialGenerator.h" +#include "Utils.h" using namespace Ogre; @@ -115,7 +116,7 @@ Ogre::String TerrainGeometryManager::getPageHeightmap(int x, int z) char buf[4096]; ds->readLine(buf, 4096); - return String(buf); + return RoR::Utils::SanitizeUtf8String(String(buf)); } void TerrainGeometryManager::initTerrain() @@ -312,11 +313,11 @@ void TerrainGeometryManager::loadLayers(int x, int z, Terrain *terrain) if (ds.isNull()) return; - char line[4096]; - ds->readLine(line, 4096); - String heightmapImage = String(line); - ds->readLine(line, 4096); - terrainLayers = PARSEINT(String(line)); + char line_buf[4096]; + ds->readLine(line_buf, 4096); + String heightmapImage = RoR::Utils::SanitizeUtf8String(String(line_buf)); + ds->readLine(line_buf, 4096); + terrainLayers = PARSEINT(RoR::Utils::SanitizeUtf8String(String(line_buf))); if (terrainLayers == 0) return; @@ -333,7 +334,8 @@ void TerrainGeometryManager::loadLayers(int x, int z, Terrain *terrain) while (!ds->eof()) { - size_t ll = ds->readLine(line, 4096); + size_t ll = ds->readLine(line_buf, 4096); + std::string line = RoR::Utils::SanitizeUtf8String(std::string(line_buf)); if (ll==0 || line[0]=='/' || line[0]==';') continue; StringVector args = StringUtil::split(String(line), ","); diff --git a/source/main/terrain/TerrainManager.cpp b/source/main/terrain/TerrainManager.cpp index 7e5724a6cb..1a72e3008d 100644 --- a/source/main/terrain/TerrainManager.cpp +++ b/source/main/terrain/TerrainManager.cpp @@ -1,25 +1,25 @@ /* -This source file is part of Rigs of Rods -Copyright 2005-2012 Pierre-Michel Ricordel -Copyright 2007-2012 Thomas Fischer + This source file is part of Rigs of Rods + Copyright 2005-2012 Pierre-Michel Ricordel + Copyright 2007-2012 Thomas Fischer + Copyright 2013-2016 Petr Ohlidal -For more information, see http://www.rigsofrods.com/ + For more information, see http://www.rigsofrods.com/ -Rigs of Rods is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License version 3, as -published by the Free Software Foundation. + Rigs of Rods is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. -Rigs of Rods is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. + Rigs of Rods is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. -You should have received a copy of the GNU General Public License -along with Rigs of Rods. If not, see . + You should have received a copy of the GNU General Public License + along with Rigs of Rods. If not, see . */ -#include "TerrainManager.h" -#include +#include "TerrainManager.h" #include "BeamData.h" #include "BeamFactory.h" @@ -43,9 +43,11 @@ along with Rigs of Rods. If not, see . #include "Utils.h" #include "Water.h" +#include + using namespace Ogre; -TerrainManager::TerrainManager() : +TerrainManager::TerrainManager() : m_terrain_config() , character(0) , collisions(0) @@ -145,14 +147,14 @@ void TerrainManager::loadTerrainConfigBasics(Ogre::DataStreamPtr &ds) m_terrain_config.load(ds, "\t:=", true); // read in the settings - terrain_name = m_terrain_config.getSetting("Name", "General"); + terrain_name = m_terrain_config.GetStringEx("Name", "General"); if (terrain_name.empty()) { ErrorUtils::ShowError(_L("Terrain loading error"), _L("the terrain name cannot be empty")); exit(125); } - ogre_terrain_config_filename = m_terrain_config.getSetting("GeometryConfig", "General"); + ogre_terrain_config_filename = m_terrain_config.GetStringEx("GeometryConfig", "General"); // otc = ogre terrain config if (ogre_terrain_config_filename.find(".otc") == String::npos) { @@ -160,12 +162,12 @@ void TerrainManager::loadTerrainConfigBasics(Ogre::DataStreamPtr &ds) exit(125); } - ambient_color = StringConverter::parseColourValue(m_terrain_config.getSetting("AmbientColor", "General"), ColourValue::White); - category_id = StringConverter::parseInt(m_terrain_config.getSetting("CategoryID", "General"), 129); - guid = m_terrain_config.getSetting("GUID", "General"); - start_position = StringConverter::parseVector3(m_terrain_config.getSetting("StartPosition", "General"), Vector3(512.0f, 0.0f, 512.0f)); - version = StringConverter::parseInt(m_terrain_config.getSetting("Version", "General"), 1); - gravity = StringConverter::parseReal(m_terrain_config.getSetting("Gravity", "General"), -9.81); + ambient_color = m_terrain_config.GetColourValue("AmbientColor", "General", ColourValue::White); + category_id = m_terrain_config.GetInt("CategoryID", "General", 129); + guid = m_terrain_config.GetStringEx("GUID", "General"); + start_position = StringConverter::parseVector3(m_terrain_config.GetStringEx("StartPosition", "General"), Vector3(512.0f, 0.0f, 512.0f)); + version = m_terrain_config.GetInt("Version", "General", 1); + gravity = m_terrain_config.GetFloat("Gravity", "General", -9.81); // parse author info ConfigFile::SettingsIterator it = m_terrain_config.getSettingsIterator("Authors"); @@ -174,8 +176,8 @@ void TerrainManager::loadTerrainConfigBasics(Ogre::DataStreamPtr &ds) while (it.hasMoreElements()) { - String type = it.peekNextKey(); // e.g. terrain - String name = it.peekNextValue(); // e.g. john doe + String type = RoR::Utils::SanitizeUtf8String(it.peekNextKey()); // e.g. terrain + String name = RoR::Utils::SanitizeUtf8String(it.peekNextValue()); // e.g. john doe if (!name.empty()) { @@ -356,13 +358,13 @@ void TerrainManager::initSkySubSystem() gEnv->sky = sky_manager; // try to load caelum config - String caelumConfig = m_terrain_config.getSetting("CaelumConfigFile", "General"); + String caelumConfig = m_terrain_config.GetStringEx("CaelumConfigFile", "General"); if (!caelumConfig.empty() && ResourceGroupManager::getSingleton().resourceExistsInAnyGroup(caelumConfig)) { // config provided and existing, use it :) - int caelumFogStart = StringConverter::parseInt(m_terrain_config.getSetting("CaelumFogStart", "General"),-1); - int caelumFogEnd = StringConverter::parseInt(m_terrain_config.getSetting("CaelumFogEnd", "General"),-1); + int caelumFogStart = m_terrain_config.GetInt("CaelumFogStart", "General",-1); + int caelumFogEnd = m_terrain_config.GetInt("CaelumFogEnd", "General",-1); sky_manager->loadScript(caelumConfig, caelumFogStart, caelumFogEnd); } else { @@ -373,7 +375,7 @@ void TerrainManager::initSkySubSystem() } else #endif //USE_CAELUM { - String sandStormConfig = m_terrain_config.getSetting("SandStormCubeMap", "General"); + String sandStormConfig = m_terrain_config.GetStringEx("SandStormCubeMap", "General"); if (!sandStormConfig.empty()) { @@ -587,12 +589,16 @@ void TerrainManager::initWater() // disabled in global config if (waterSettingsString == "None") return; // disabled in map config - if (!StringConverter::parseBool(m_terrain_config.getSetting("Water", "General"))) return; + bool has_water = m_terrain_config.GetBool("Water", "General", false); + if (!has_water) + { + return; + } if (waterSettingsString == "Hydrax") { // try to load hydrax config - String hydraxConfig = m_terrain_config.getSetting("HydraxConfigFile", "General"); + String hydraxConfig = m_terrain_config.GetStringEx("HydraxConfigFile", "General"); if (!hydraxConfig.empty() && ResourceGroupManager::getSingleton().resourceExistsInAnyGroup(hydraxConfig)) { @@ -675,7 +681,7 @@ void TerrainManager::initCollisions() void TerrainManager::initTerrainCollisions() { - String tractionMapConfig = m_terrain_config.getSetting("TractionMap", "General"); + String tractionMapConfig = m_terrain_config.GetStringEx("TractionMap", "General"); if (!tractionMapConfig.empty()) { gEnv->collisions->setupLandUse(tractionMapConfig.c_str()); diff --git a/source/main/terrain/TerrainObjectManager.cpp b/source/main/terrain/TerrainObjectManager.cpp index 265e723dd3..af82c86927 100644 --- a/source/main/terrain/TerrainObjectManager.cpp +++ b/source/main/terrain/TerrainObjectManager.cpp @@ -38,6 +38,7 @@ along with Rigs of Rods. If not, see . #include "SurveyMapManager.h" #include "TerrainGeometryManager.h" #include "TerrainManager.h" +#include "Utils.h" #include "WriteTextToTexture.h" #include @@ -711,15 +712,8 @@ void TerrainObjectManager::loadObject(const Ogre::String &name, const Ogre::Vect return; } - // nice idea, but too many random hits - //if (abs(rot.x+1) < 0.001) rot.x = Math::RangeRandom(0, 360); - //if (abs(rot.y+1) < 0.001) rot.y = Math::RangeRandom(0, 360); - //if (abs(rot.z+1) < 0.001) rot.z = Math::RangeRandom(0, 360); - if (name.empty()) return; - //FILE *fd; - //char oname[1024] = {}; char mesh[1024] = {}; char line[1024] = {}; char collmesh[1024] = {}; @@ -768,7 +762,8 @@ void TerrainObjectManager::loadObject(const Ogre::String &name, const Ogre::Vect //scale ds->readLine(line, 1023); sscanf(line, "%f, %f, %f",&sc.x, &sc.y, &sc.z); - String entityName = "object" + TOSTRING(objcounter) + "(" + name + ")"; + String entity_name = "object" + TOSTRING(objcounter) + "(" + name + ")"; + RoR::Utils::SanitizeUtf8String(entity_name); objcounter++; SceneNode *tenode = gEnv->sceneManager->getRootSceneNode()->createChildSceneNode(); @@ -777,7 +772,7 @@ void TerrainObjectManager::loadObject(const Ogre::String &name, const Ogre::Vect MeshObject *mo = NULL; if (String(mesh) != "none") { - mo = new MeshObject(mesh, entityName, tenode, NULL, background_loading); + mo = new MeshObject(mesh, entity_name, tenode, NULL, background_loading); } //mo->setQueryFlags(OBJECTS_MASK); @@ -803,15 +798,11 @@ void TerrainObjectManager::loadObject(const Ogre::String &name, const Ogre::Vect SubEntity *se = mo->getEntity()->getSubEntity(i); String matname = se->getMaterialName(); String newmatname = matname + "/" + instancename; - //LOG("subentity " + TOSTRING(i) + ": "+ matname + " -> " + newmatname); se->getMaterial()->clone(newmatname); se->setMaterialName(newmatname); } } - //String meshGroup = ResourceGroupManager::getSingleton().findGroupContainingResource(mesh); - //MeshPtr mainMesh = mo->getMesh(); - //collision box(es) bool virt=false; bool rotating=false; @@ -824,10 +815,11 @@ void TerrainObjectManager::loadObject(const Ogre::String &name, const Ogre::Vect size_t ll=ds->readLine(line, 1023); // little workaround to trim it - String lineStr = String(line); - Ogre::StringUtil::trim(lineStr); + String line_str = String(line); + Ogre::StringUtil::trim(line_str); + RoR::Utils::SanitizeUtf8String(line_str); - const char* ptline = lineStr.c_str(); + const char* ptline = line_str.c_str(); if (ll==0 || line[0]=='/' || line[0]==';') continue; if (!strcmp("end",ptline)) break; diff --git a/source/main/utils/ConfigFile.cpp b/source/main/utils/ConfigFile.cpp index 7993f4705b..34eeb9aa8a 100644 --- a/source/main/utils/ConfigFile.cpp +++ b/source/main/utils/ConfigFile.cpp @@ -1,63 +1,64 @@ /* - This source file is part of Rigs of Rods - Copyright 2005-2012 Pierre-Michel Ricordel - Copyright 2007-2012 Thomas Fischer - Copyright 2013-2014 Petr Ohlidal + This source file is part of Rigs of Rods + Copyright 2005-2012 Pierre-Michel Ricordel + Copyright 2007-2012 Thomas Fischer + Copyright 2013-2016 Petr Ohlidal - For more information, see http://www.rigsofrods.com/ + For more information, see http://www.rigsofrods.com/ - Rigs of Rods is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License version 3, as - published by the Free Software Foundation. + Rigs of Rods is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. - Rigs of Rods is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + Rigs of Rods is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with Rigs of Rods. If not, see . + You should have received a copy of the GNU General Public License + along with Rigs of Rods. If not, see . */ /** - @file ConfigFile.cpp - @date 06/2014 - @author Petr Ohlidal + @file ConfigFile.cpp + @date 06/2014 + @author Petr Ohlidal */ #include "ConfigFile.h" +#include "Utils.h" #include #include using namespace RoR; -float ConfigFile::GetFloat(Ogre::String const & key, float defaultValue) +float ConfigFile::GetFloat(Ogre::String const & key, Ogre::String const & section, float defaultValue) { - return Ogre::StringConverter::parseReal(getSetting(key), defaultValue); + return Ogre::StringConverter::parseReal(Ogre::ConfigFile::getSetting(key, section), defaultValue); } -Ogre::ColourValue ConfigFile::GetColourValue(Ogre::String const & key, Ogre::ColourValue const & defaultValue) +Ogre::ColourValue ConfigFile::GetColourValue(Ogre::String const & key, Ogre::String const & section, Ogre::ColourValue const & defaultValue) { - return Ogre::StringConverter::parseColourValue(getSetting(key), defaultValue); + return Ogre::StringConverter::parseColourValue(Ogre::ConfigFile::getSetting(key, section), defaultValue); } -int ConfigFile::GetInt(Ogre::String const & key, int defaultValue) +int ConfigFile::GetInt(Ogre::String const & key, Ogre::String const & section, int defaultValue) { - return Ogre::StringConverter::parseInt(getSetting(key), defaultValue); + return Ogre::StringConverter::parseInt(Ogre::ConfigFile::getSetting(key, section), defaultValue); } -bool ConfigFile::GetBool(Ogre::String const & key, bool defaultValue) +bool ConfigFile::GetBool(Ogre::String const & key, Ogre::String const & section, bool defaultValue) { - return Ogre::StringConverter::parseBool(getSetting(key), defaultValue); + return Ogre::StringConverter::parseBool(Ogre::ConfigFile::getSetting(key, section), defaultValue); } -Ogre::String ConfigFile::GetString(Ogre::String const & key, Ogre::String const & defaultValue) +Ogre::String ConfigFile::GetStringEx(Ogre::String const & key, Ogre::String const & section, Ogre::String const & defaultValue) { - auto setting = getSetting(key); - if (setting.empty()) - { - return defaultValue; - } - return setting; + auto setting = Ogre::ConfigFile::getSetting(key, section); + if (setting.empty()) + { + return defaultValue; + } + return RoR::Utils::SanitizeUtf8String(setting); } diff --git a/source/main/utils/ConfigFile.h b/source/main/utils/ConfigFile.h index 19529b18ac..bcbd3faa5c 100644 --- a/source/main/utils/ConfigFile.h +++ b/source/main/utils/ConfigFile.h @@ -1,28 +1,28 @@ /* - This source file is part of Rigs of Rods - Copyright 2005-2012 Pierre-Michel Ricordel - Copyright 2007-2012 Thomas Fischer - Copyright 2013-2014 Petr Ohlidal + This source file is part of Rigs of Rods + Copyright 2005-2012 Pierre-Michel Ricordel + Copyright 2007-2012 Thomas Fischer + Copyright 2013-2016 Petr Ohlidal - For more information, see http://www.rigsofrods.com/ + For more information, see http://www.rigsofrods.com/ - Rigs of Rods is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License version 3, as - published by the Free Software Foundation. + Rigs of Rods is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. - Rigs of Rods is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + Rigs of Rods is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with Rigs of Rods. If not, see . + You should have received a copy of the GNU General Public License + along with Rigs of Rods. If not, see . */ /** - @file ConfigFile.h - @date 06/2014 - @author Petr Ohlidal + @file ConfigFile.h + @date 06/2014 + @author Petr Ohlidal */ #pragma once @@ -38,17 +38,42 @@ namespace RoR */ class ConfigFile: public Ogre::ConfigFile { - public: +public: - Ogre::ColourValue GetColourValue(Ogre::String const & key, Ogre::ColourValue const & defaultValue = Ogre::ColourValue()); + Ogre::ColourValue GetColourValue(Ogre::String const & key, Ogre::ColourValue const & defaultValue = Ogre::ColourValue()) + { + return this->GetColourValue(key, Ogre::StringUtil::BLANK, defaultValue); + } + Ogre::ColourValue GetColourValue(Ogre::String const & key, Ogre::String const & section, Ogre::ColourValue const & defaultValue = Ogre::ColourValue()); - float GetFloat(Ogre::String const & key, float defaultValue = 0.f); + float GetFloat(Ogre::String const & key, float defaultValue = 0.f) + { + return this->GetFloat(key, Ogre::StringUtil::BLANK, defaultValue); + } + float GetFloat(Ogre::String const & key, Ogre::String const & section, float defaultValue = 0.f); - bool GetBool(Ogre::String const & key, bool defaultValue = false); + bool GetBool(Ogre::String const & key, bool defaultValue = false) + { + return this->GetBool(key, Ogre::StringUtil::BLANK, defaultValue); + } + bool GetBool(Ogre::String const & key, Ogre::String const & section, bool defaultValue = false); - int GetInt(Ogre::String const & key, int defaultValue = 0); + int GetInt(Ogre::String const & key, int defaultValue = 0) + { + return this->GetInt(key, Ogre::StringUtil::BLANK, defaultValue); + } + int GetInt(Ogre::String const & key, Ogre::String const & section, int defaultValue = 0); - Ogre::String GetString(Ogre::String const & key, Ogre::String const & defaultValue = ""); + Ogre::String GetString(Ogre::String const & key, Ogre::String const & defaultValue = "") + { + return this->GetStringEx(key, Ogre::StringUtil::BLANK, defaultValue); + } + Ogre::String GetStringEx(Ogre::String const & key, Ogre::String const & section, Ogre::String const & defaultValue = ""); + +private: + //Block access to Ogre::ConfigFile::getSetting() - not UTF8 safe! + Ogre::String getSetting(Ogre::String, Ogre::String); + Ogre::String getSetting(Ogre::String, Ogre::String, Ogre::String); }; } // namespace RoR diff --git a/source/main/utils/ImprovedConfigFile.h b/source/main/utils/ImprovedConfigFile.h index c0f6cb3d40..7dcc2859e4 100644 --- a/source/main/utils/ImprovedConfigFile.h +++ b/source/main/utils/ImprovedConfigFile.h @@ -23,12 +23,14 @@ along with Rigs of Rods. If not, see . #define __IMPROVEDCONFIGFILE_H_ #include "RoRPrerequisites.h" +#include "ConfigFile.h" + #include namespace Ogre { -class ImprovedConfigFile : public Ogre::ConfigFile +class ImprovedConfigFile : public RoR::ConfigFile { public: ImprovedConfigFile() : separators("="), filename() @@ -120,43 +122,43 @@ class ImprovedConfigFile : public Ogre::ConfigFile } // type specific implementations - Radian getSettingRadian(String key, String section = StringUtil::BLANK) { return StringConverter::parseAngle(getSetting(key, section)); } + Radian getSettingRadian(String key, String section = StringUtil::BLANK) { return StringConverter::parseAngle(GetStringEx(key, section)); } void setSetting(String key, Radian value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - bool getSettingBool(String key, String section = StringUtil::BLANK) { return StringConverter::parseBool(getSetting(key, section)); } + bool getSettingBool(String key, String section = StringUtil::BLANK) { return StringConverter::parseBool(GetStringEx(key, section)); } void setSetting(String key, bool value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - Real getSettingReal(String key, String section = StringUtil::BLANK) { return StringConverter::parseReal(getSetting(key, section)); } + Real getSettingReal(String key, String section = StringUtil::BLANK) { return StringConverter::parseReal(GetStringEx(key, section)); } void setSetting(String key, Real value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - int getSettingInt(String key, String section = StringUtil::BLANK) { return StringConverter::parseInt(getSetting(key, section)); } + int getSettingInt(String key, String section = StringUtil::BLANK) { return StringConverter::parseInt(GetStringEx(key, section)); } void setSetting(String key, int value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - unsigned int getSettingUnsignedInt(String key, String section = StringUtil::BLANK) { return StringConverter::parseUnsignedInt(getSetting(key, section)); } + unsigned int getSettingUnsignedInt(String key, String section = StringUtil::BLANK) { return StringConverter::parseUnsignedInt(GetStringEx(key, section)); } void setSetting(String key, unsigned int value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - long getSettingLong(String key, String section = StringUtil::BLANK) { return StringConverter::parseLong(getSetting(key, section)); } + long getSettingLong(String key, String section = StringUtil::BLANK) { return StringConverter::parseLong(GetStringEx(key, section)); } void setSetting(String key, long value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - unsigned long getSettingUnsignedLong(String key, String section = StringUtil::BLANK) { return StringConverter::parseUnsignedLong(getSetting(key, section)); } + unsigned long getSettingUnsignedLong(String key, String section = StringUtil::BLANK) { return StringConverter::parseUnsignedLong(GetStringEx(key, section)); } void setSetting(String key, unsigned long value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - Vector3 getSettingVector3(String key, String section = StringUtil::BLANK) { return StringConverter::parseVector3(getSetting(key, section)); } + Vector3 getSettingVector3(String key, String section = StringUtil::BLANK) { return StringConverter::parseVector3(GetStringEx(key, section)); } void setSetting(String key, Vector3 value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - Matrix3 getSettingMatrix3(String key, String section = StringUtil::BLANK) { return StringConverter::parseMatrix3(getSetting(key, section)); } + Matrix3 getSettingMatrix3(String key, String section = StringUtil::BLANK) { return StringConverter::parseMatrix3(GetStringEx(key, section)); } void setSetting(String key, Matrix3 value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - Matrix4 getSettingMatrix4(String key, String section = StringUtil::BLANK) { return StringConverter::parseMatrix4(getSetting(key, section)); } + Matrix4 getSettingMatrix4(String key, String section = StringUtil::BLANK) { return StringConverter::parseMatrix4(GetStringEx(key, section)); } void setSetting(String key, Matrix4 value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - Quaternion getSettingQuaternion(String key, String section = StringUtil::BLANK) { return StringConverter::parseQuaternion(getSetting(key, section)); } + Quaternion getSettingQuaternion(String key, String section = StringUtil::BLANK) { return StringConverter::parseQuaternion(GetStringEx(key, section)); } void setSetting(String key, Quaternion value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - ColourValue getSettingColorValue(String key, String section = StringUtil::BLANK) { return StringConverter::parseColourValue(getSetting(key, section)); } + ColourValue getSettingColorValue(String key, String section = StringUtil::BLANK) { return StringConverter::parseColourValue(GetStringEx(key, section)); } void setSetting(String key, ColourValue value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } - StringVector getSettingStringVector(String key, String section = StringUtil::BLANK) { return StringConverter::parseStringVector(getSetting(key, section)); } + StringVector getSettingStringVector(String key, String section = StringUtil::BLANK) { return StringConverter::parseStringVector(GetStringEx(key, section)); } void setSetting(String key, StringVector value, String section = StringUtil::BLANK) { setSetting(key, TOSTRING(value), section); } protected: diff --git a/source/main/utils/Settings.cpp b/source/main/utils/Settings.cpp index 89a9cc8188..b918aa518c 100644 --- a/source/main/utils/Settings.cpp +++ b/source/main/utils/Settings.cpp @@ -1,24 +1,32 @@ /* -This source file is part of Rigs of Rods -Copyright 2005-2012 Pierre-Michel Ricordel -Copyright 2007-2012 Thomas Fischer + This source file is part of Rigs of Rods + Copyright 2005-2012 Pierre-Michel Ricordel + Copyright 2007-2012 Thomas Fischer + Copyright 2013-2016 Petr Ohlidal -For more information, see http://www.rigsofrods.com/ + For more information, see http://www.rigsofrods.com/ -Rigs of Rods is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License version 3, as -published by the Free Software Foundation. + Rigs of Rods is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. -Rigs of Rods is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. + Rigs of Rods is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. -You should have received a copy of the GNU General Public License -along with Rigs of Rods. If not, see . + You should have received a copy of the GNU General Public License + along with Rigs of Rods. If not, see . */ -// created by thomas fischer, 4th of January 2009 + +/** + @file + @date 4th of January 2009 + @author Thomas Fischer +*/ + #include "Settings.h" +#include "Utils.h" #include @@ -225,26 +233,26 @@ void Settings::saveSettings(String configFile) void Settings::loadSettings(String configFile, bool overwrite) { - //printf("trying to load configfile: %s...\n", configFile.c_str()); ConfigFile cfg; cfg.load(configFile, "=:\t", false); // load all settings into a map! ConfigFile::SettingsIterator i = cfg.getSettingsIterator(); - String svalue, sname; + String s_value, s_name; while (i.hasMoreElements()) { - sname = i.peekNextKey(); - svalue = i.getNext(); - if (!overwrite && settings[sname] != "") continue; - settings[sname] = svalue; - //logfile->AddLine(conv("### ") + conv(sname) + conv(" : ") + conv(svalue));logfile->Write(); + s_name = RoR::Utils::SanitizeUtf8String(i.peekNextKey()); + s_value = RoR::Utils::SanitizeUtf8String(i.getNext()); + if (!overwrite && !settings[s_name].empty()) + { + continue; + } + settings[s_name] = s_value; } // add a GUID if not there checkGUID(); generateBinaryHash(); -#ifndef NOOGRE // generate hash of the token String usertoken = SSETTING("User Token", ""); char usertokensha1result[250]; @@ -258,7 +266,6 @@ void Settings::loadSettings(String configFile, bool overwrite) } setSetting("User Token Hash", String(usertokensha1result)); -#endif // NOOGRE } int Settings::generateBinaryHash() diff --git a/source/main/utils/Settings.h b/source/main/utils/Settings.h index 1d89c32a73..0ce3a85a48 100644 --- a/source/main/utils/Settings.h +++ b/source/main/utils/Settings.h @@ -1,23 +1,29 @@ /* -This source file is part of Rigs of Rods -Copyright 2005-2012 Pierre-Michel Ricordel -Copyright 2007-2012 Thomas Fischer + This source file is part of Rigs of Rods + Copyright 2005-2012 Pierre-Michel Ricordel + Copyright 2007-2012 Thomas Fischer + Copyright 2013-2016 Petr Ohlidal -For more information, see http://www.rigsofrods.com/ + For more information, see http://www.rigsofrods.com/ -Rigs of Rods is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License version 3, as -published by the Free Software Foundation. + Rigs of Rods is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. -Rigs of Rods is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. + Rigs of Rods is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. -You should have received a copy of the GNU General Public License -along with Rigs of Rods. If not, see . + You should have received a copy of the GNU General Public License + along with Rigs of Rods. If not, see . +*/ + +/** + @file + @date 4th of January 2009 + @author Thomas Fischer */ -// created by thomas fischer, 4th of January 2009 #pragma once #ifndef __Settings_H_ diff --git a/source/main/utils/Utils.cpp b/source/main/utils/Utils.cpp index 050bb74d5b..c3b16fe13c 100644 --- a/source/main/utils/Utils.cpp +++ b/source/main/utils/Utils.cpp @@ -1,30 +1,32 @@ /* -This source file is part of Rigs of Rods -Copyright 2005-2012 Pierre-Michel Ricordel -Copyright 2007-2012 Thomas Fischer + This source file is part of Rigs of Rods + Copyright 2005-2012 Pierre-Michel Ricordel + Copyright 2007-2012 Thomas Fischer + Copyright 2013-2016 Petr Ohlidal -For more information, see http://www.rigsofrods.com/ + For more information, see http://www.rigsofrods.com/ -Rigs of Rods is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License version 3, as -published by the Free Software Foundation. + Rigs of Rods is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. -Rigs of Rods is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. + Rigs of Rods is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. -You should have received a copy of the GNU General Public License -along with Rigs of Rods. If not, see . + You should have received a copy of the GNU General Public License + along with Rigs of Rods. If not, see . */ -#include "Utils.h" -#include +#include "Utils.h" #include "rornet.h" #include "RoRVersion.h" #include "SHA1.h" +#include + #ifndef _WIN32 #include #endif // _WIN32 @@ -89,18 +91,8 @@ String hexdump(void *pAddressIn, long lSize) UTFString tryConvertUTF(const char *buffer) { - try - { - UTFString s = UTFString(buffer); - if (s.empty()) - s = UTFString("(UTF conversion error 1)"); - return s; - - } catch(...) - { - return UTFString("(UTF conversion error 2)"); - } - //return UTFString("(UTF conversion error 3)"); + std::string str_in(buffer); + return UTFString(RoR::Utils::SanitizeUtf8String(str_in)); } UTFString formatBytes(double bytes) @@ -237,74 +229,12 @@ void fixRenderWindowIcon (RenderWindow *rw) UTFString ANSI_TO_UTF(const String source) { - return UTFString(ANSI_TO_WCHAR(source)); // UTF converts from wstring + return UTFString(RoR::Utils::SanitizeUtf8String(source)); } -// TODO: Make it bulletproof! e.g kills this std::wstring ANSI_TO_WCHAR(const String source) { -#ifdef _WIN32 - const char* srcPtr = source.c_str(); - int tmpSize = MultiByteToWideChar( CP_ACP, 0, srcPtr, -1, 0, 0 ); - WCHAR* tmpBuff = new WCHAR [ tmpSize + 1 ]; - MultiByteToWideChar( CP_ACP, 0, srcPtr, -1, tmpBuff, tmpSize ); - std::wstring ret = tmpBuff; - delete[] tmpBuff; - return ret; -#if 0 - // does not make much of a difference - int retval = MultiByteToWideChar(CP_ACP, MB_ERR_INVALID_CHARS, _source.c_str(), -1, NULL, 0); - if (!SUCCEEDED(retval)) - { - return std::wstring(L"ERR"); - } - WCHAR* wstr = new WCHAR [ retval + 1 ]; - if (wstr == NULL) - { - return std::wstring(L"ERR"); - } - std::fill_n(wstr, retval+1, '\0' ); - retval = MultiByteToWideChar(CP_ACP, MB_ERR_INVALID_CHARS, _source.c_str(), -1, wstr, retval); - if (!SUCCEEDED(retval)) - { - delete[] wstr; - return std::wstring(L"ERR"); - } - std::wstring ret = wstr; - delete[] wstr; - return ret; -#endif -#else - // TODO: GET THIS WORKING - /* - const char* srcPtr = source.c_str(); - iconv_t icv = iconv_open("ASCII", "UTF-8"); - if ( icv == (iconv_t) -1 ) - { - return std::wstring(L"ERR1"); - } - - char *inpbuf = const_cast(source.c_str()); - size_t inbytes = source.size(); - size_t outbytes = inbytes; - size_t nread = 0; - char *outbuf = (char *)calloc((outbytes*4+1)*sizeof(char), 1); - - size_t res = iconv(icv, &inpbuf, &inbytes, &outbuf, &outbytes); - if (res == (size_t) -1) - { - //free(outbuf); - return std::wstring(L"ERR2"); - } - iconv_close(icv); - //free(outbuf); - */ - - // hacky! - std::wstring str2(source.length(), L' '); // Make room for characters - std::copy(source.begin(), source.end(), str2.begin()); - return str2; -#endif // _WIN32 + return ANSI_TO_UTF(source).asWStr(); } void trimUTFString( UTFString &str, bool left, bool right) @@ -429,6 +359,14 @@ namespace Utils } return input.substr(substr_start, substr_count); } + + std::string SanitizeUtf8String(std::string const& str_in) + { + // Cloned from UTFCPP tutorial: http://utfcpp.sourceforge.net/#fixinvalid + std::string str_out; + utf8::replace_invalid(str_in.begin(), str_in.end(), std::back_inserter(str_out)); + return str_out; + } } } diff --git a/source/main/utils/Utils.h b/source/main/utils/Utils.h index 8e264ecc7c..2e71c16512 100644 --- a/source/main/utils/Utils.h +++ b/source/main/utils/Utils.h @@ -1,28 +1,28 @@ /* - This source file is part of Rigs of Rods - Copyright 2005-2012 Pierre-Michel Ricordel - Copyright 2007-2012 Thomas Fischer - Copyright 2013-2014 Petr Ohlidal + This source file is part of Rigs of Rods + Copyright 2005-2012 Pierre-Michel Ricordel + Copyright 2007-2012 Thomas Fischer + Copyright 2013-2016 Petr Ohlidal - For more information, see http://www.rigsofrods.com/ + For more information, see http://www.rigsofrods.com/ - Rigs of Rods is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License version 3, as - published by the Free Software Foundation. + Rigs of Rods is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3, as + published by the Free Software Foundation. - Rigs of Rods is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + Rigs of Rods is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with Rigs of Rods. If not, see . + You should have received a copy of the GNU General Public License + along with Rigs of Rods. If not, see . */ /** - @file - @author Thomas Fischer thomas{AT}thomasfischer{DOT}biz - @date 9th of August 2009 + @file + @author Thomas Fischer thomas{AT}thomasfischer{DOT}biz + @date 9th of August 2009 */ #pragma once @@ -30,6 +30,8 @@ #include "RoRPrerequisites.h" #include +#include "utf8/checked.h" +#include "utf8/unchecked.h" #ifdef USE_MYGUI #include @@ -143,7 +145,9 @@ namespace RoR namespace Utils { - std::string TrimBlanksAndLinebreaks(std::string const & input); + std::string TrimBlanksAndLinebreaks(std::string const & input); + + std::string SanitizeUtf8String(std::string const& str_in); } } diff --git a/source/main/utils/utf8/README.md b/source/main/utils/utf8/README.md new file mode 100644 index 0000000000..7ddfd40b98 --- /dev/null +++ b/source/main/utils/utf8/README.md @@ -0,0 +1,6 @@ +UTF-8 CPP library +================= + +Homepage: http://utfcpp.sourceforge.net/ +Repository: https://github.com/nemtrif/utfcpp +License: Boost Software License 1.0 \ No newline at end of file diff --git a/source/main/utils/utf8/checked.h b/source/main/utils/utf8/checked.h new file mode 100644 index 0000000000..1331155138 --- /dev/null +++ b/source/main/utils/utf8/checked.h @@ -0,0 +1,327 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" +#include + +namespace utf8 +{ + // Base for the exceptions that may be thrown from the library + class exception : public ::std::exception { + }; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public exception { + uint32_t cp; + public: + invalid_code_point(uint32_t cp) : cp(cp) {} + virtual const char* what() const throw() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + virtual const char* what() const throw() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + virtual const char* what() const throw() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public exception { + public: + virtual const char* what() const throw() { return "Not enough space"; } + }; + + /// The library API - functions intended to be called by the users + + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + throw not_enough_room(); + case internal::INVALID_LEAD: + out = utf8::append (replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::append (replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } + } + return out; + } + + template + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::replace_invalid(start, end, out, replacement_marker); + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + break; + case internal::NOT_ENOUGH_ROOM : + throw not_enough_room(); + case internal::INVALID_LEAD : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + throw invalid_utf8(*it); + case internal::INVALID_CODE_POINT : + throw invalid_code_point(cp); + } + return cp; + } + + template + uint32_t peek_next(octet_iterator it, octet_iterator end) + { + return utf8::next(it, end); + } + + template + uint32_t prior(octet_iterator& it, octet_iterator start) + { + // can't do much if it == start + if (it == start) + throw not_enough_room(); + + octet_iterator end = it; + // Go back until we hit either a lead octet or start + while (utf8::internal::is_trail(*(--it))) + if (it == start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + return utf8::peek_next(it, end); + } + + /// Deprecated in versions that include "prior" + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (utf8::internal::is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return utf8::next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + utf8::next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::next(first, last); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(cp)); + + } + // Lone trail surrogate + else if (utf8::internal::is_trail_surrogate(cp)) + throw invalid_utf16(static_cast(cp)); + + result = utf8::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = utf8::next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start != end) + (*result++) = utf8::next(start, end); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + octet_iterator range_start; + octet_iterator range_end; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it, + const octet_iterator& range_start, + const octet_iterator& range_end) : + it(octet_it), range_start(range_start), range_end(range_end) + { + if (it < range_start || it > range_end) + throw std::out_of_range("Invalid utf-8 iterator position"); + } + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::next(temp, range_end); + } + bool operator == (const iterator& rhs) const + { + if (range_start != rhs.range_start || range_end != rhs.range_end) + throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + utf8::next(it, range_end); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + utf8::next(it, range_end); + return temp; + } + iterator& operator -- () + { + utf8::prior(it, range_start); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::prior(it, range_start); + return temp; + } + }; // class iterator + +} // namespace utf8 + +#endif //header guard + + diff --git a/source/main/utils/utf8/core.h b/source/main/utils/utf8/core.h new file mode 100644 index 0000000000..693d388c07 --- /dev/null +++ b/source/main/utils/utf8/core.h @@ -0,0 +1,329 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + +// Helper code - not intended to be directly called by the library users. May be changed at any time +namespace internal +{ + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint16_t LEAD_SURROGATE_MIN = 0xd800u; + const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; + const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffffu; + + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((utf8::internal::mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_lead_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); + } + + template + inline bool is_trail_surrogate(u16 cp) + { + return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + template + inline bool is_code_point_valid(u32 cp) + { + return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); + } + + template + inline typename std::iterator_traits::difference_type + sequence_length(octet_iterator lead_it) + { + uint8_t lead = utf8::internal::mask8(*lead_it); + if (lead < 0x80) + return 1; + else if ((lead >> 5) == 0x6) + return 2; + else if ((lead >> 4) == 0xe) + return 3; + else if ((lead >> 3) == 0x1e) + return 4; + else + return 0; + } + + template + inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + { + if (cp < 0x80) { + if (length != 1) + return true; + } + else if (cp < 0x800) { + if (length != 2) + return true; + } + else if (cp < 0x10000) { + if (length != 3) + return true; + } + + return false; + } + + enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; + + /// Helper for get_sequence_x + template + utf_error increase_safely(octet_iterator& it, octet_iterator end) + { + if (++it == end) + return NOT_ENOUGH_ROOM; + + if (!utf8::internal::is_trail(*it)) + return INCOMPLETE_SEQUENCE; + + return UTF8_OK; + } + + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + + /// get_sequence_x functions decode utf-8 sequences of the length x + template + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + return UTF8_OK; + } + + template + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); + + return UTF8_OK; + } + + template + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + template + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + + code_point = utf8::internal::mask8(*it); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; + + UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) + + code_point += (*it) & 0x3f; + + return UTF8_OK; + } + + #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR + + template + utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + { + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + octet_iterator original_it = it; + + uint32_t cp = 0; + // Determine the sequence length based on the lead octet + typedef typename std::iterator_traits::difference_type octet_difference_type; + const octet_difference_type length = utf8::internal::sequence_length(it); + + // Get trail octets and calculate the code point + utf_error err = UTF8_OK; + switch (length) { + case 0: + return INVALID_LEAD; + case 1: + err = utf8::internal::get_sequence_1(it, end, cp); + break; + case 2: + err = utf8::internal::get_sequence_2(it, end, cp); + break; + case 3: + err = utf8::internal::get_sequence_3(it, end, cp); + break; + case 4: + err = utf8::internal::get_sequence_4(it, end, cp); + break; + } + + if (err == UTF8_OK) { + // Decoding succeeded. Now, security checks... + if (utf8::internal::is_code_point_valid(cp)) { + if (!utf8::internal::is_overlong_sequence(cp, length)){ + // Passed! Return here. + code_point = cp; + ++it; + return UTF8_OK; + } + else + err = OVERLONG_SEQUENCE; + } + else + err = INVALID_CODE_POINT; + } + + // Failure branch - restore the original value of the iterator + it = original_it; + return err; + } + + template + inline utf_error validate_next(octet_iterator& it, octet_iterator end) { + uint32_t ignored; + return utf8::internal::validate_next(it, end, ignored); + } + +} // namespace internal + + /// The library API - functions intended to be called by the users + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); + if (err_code != internal::UTF8_OK) + return result; + } + return result; + } + + template + inline bool is_valid(octet_iterator start, octet_iterator end) + { + return (utf8::find_invalid(start, end) == end); + } + + template + inline bool starts_with_bom (octet_iterator it, octet_iterator end) + { + return ( + ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && + ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && + ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) + ); + } + + //Deprecated in release 2.3 + template + inline bool is_bom (octet_iterator it) + { + return ( + (utf8::internal::mask8(*it++)) == bom[0] && + (utf8::internal::mask8(*it++)) == bom[1] && + (utf8::internal::mask8(*it)) == bom[2] + ); + } +} // namespace utf8 + +#endif // header guard + + diff --git a/source/main/utils/utf8/unchecked.h b/source/main/utils/utf8/unchecked.h new file mode 100644 index 0000000000..cb2427166b --- /dev/null +++ b/source/main/utils/utf8/unchecked.h @@ -0,0 +1,228 @@ +// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include "core.h" + +namespace utf8 +{ + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = utf8::internal::mask8(*it); + typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); + switch (length) { + case 1: + break; + case 2: + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + break; + case 3: + ++it; + cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + break; + case 4: + ++it; + cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); + ++it; + cp += (utf8::internal::mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + break; + } + ++it; + return cp; + } + + template + uint32_t peek_next(octet_iterator it) + { + return utf8::unchecked::next(it); + } + + template + uint32_t prior(octet_iterator& it) + { + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + + // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) + template + inline uint32_t previous(octet_iterator& it) + { + return utf8::unchecked::prior(it); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + utf8::unchecked::next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + utf8::unchecked::next(first); + return dist; + } + + template + octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first + if (utf8::internal::is_lead_surrogate(cp)) { + uint32_t trail_surrogate = utf8::internal::mask16(*start++); + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + } + result = utf8::unchecked::append(cp, result); + } + return result; + } + + template + u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start < end) { + uint32_t cp = utf8::unchecked::next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + return result; + } + + template + octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + result = utf8::unchecked::append(*(start++), result); + + return result; + } + + template + u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = utf8::unchecked::next(start); + + return result; + } + + // The iterator class + template + class iterator : public std::iterator { + octet_iterator it; + public: + iterator () {} + explicit iterator (const octet_iterator& octet_it): it(octet_it) {} + // the default "big three" are OK + octet_iterator base () const { return it; } + uint32_t operator * () const + { + octet_iterator temp = it; + return utf8::unchecked::next(temp); + } + bool operator == (const iterator& rhs) const + { + return (it == rhs.it); + } + bool operator != (const iterator& rhs) const + { + return !(operator == (rhs)); + } + iterator& operator ++ () + { + ::std::advance(it, utf8::internal::sequence_length(it)); + return *this; + } + iterator operator ++ (int) + { + iterator temp = *this; + ::std::advance(it, utf8::internal::sequence_length(it)); + return temp; + } + iterator& operator -- () + { + utf8::unchecked::prior(it); + return *this; + } + iterator operator -- (int) + { + iterator temp = *this; + utf8::unchecked::prior(it); + return temp; + } + }; // class iterator + + } // namespace utf8::unchecked +} // namespace utf8 + + +#endif // header guard + diff --git a/source/rig_file_input_output/RigDef_Parser.cpp b/source/rig_file_input_output/RigDef_Parser.cpp index d2d368c9af..60488314a7 100644 --- a/source/rig_file_input_output/RigDef_Parser.cpp +++ b/source/rig_file_input_output/RigDef_Parser.cpp @@ -31,6 +31,7 @@ #include "RigDef_Regexes.h" #include "BitFlags.h" #include "RoRPrerequisites.h" +#include "Utils.h" #include #include @@ -75,15 +76,17 @@ Parser::Parser(): Parser::~Parser() {} -void Parser::ParseLine(Ogre::String const & line) +void Parser::ParseLine(Ogre::String const & line_unchecked) { - unsigned int line_length = line.length(); + unsigned int line_length = line_unchecked.length(); if (line_length == 0) { m_current_line_number++; return; } + std::string line = RoR::Utils::SanitizeUtf8String(line_unchecked); + bool line_finished = false; bool scan_for_keyword = true;