From a1ecab2f95b8bb157b03f49bd067099c9dc8664c Mon Sep 17 00:00:00 2001 From: Dennis Luxen Date: Tue, 29 Apr 2014 15:33:13 +0200 Subject: [PATCH] refactor xml parser, uses faster string to double conversion --- Extractor/XMLParser.cpp | 526 ++++++++++++++++++++++------------------ Extractor/XMLParser.h | 19 +- Util/StringUtil.h | 34 +++ 3 files changed, 334 insertions(+), 245 deletions(-) diff --git a/Extractor/XMLParser.cpp b/Extractor/XMLParser.cpp index 06ace2f2d6..ad7da4fcdb 100644 --- a/Extractor/XMLParser.cpp +++ b/Extractor/XMLParser.cpp @@ -42,249 +42,307 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -XMLParser::XMLParser(const char * filename, ExtractorCallbacks* ec, ScriptingEnvironment& se) : BaseParser(ec, se) { - inputReader = inputReaderFactory(filename); +XMLParser::XMLParser(const char *filename, ExtractorCallbacks *ec, ScriptingEnvironment &se) + : BaseParser(ec, se) +{ + inputReader = inputReaderFactory(filename); } -bool XMLParser::ReadHeader() { - return (xmlTextReaderRead( inputReader ) == 1); -} -bool XMLParser::Parse() { - while ( xmlTextReaderRead( inputReader ) == 1 ) { - const int type = xmlTextReaderNodeType( inputReader ); - - //1 is Element - if ( type != 1 ) { - continue; - } - - xmlChar* currentName = xmlTextReaderName( inputReader ); - if ( currentName == NULL ) { - continue; - } - - if ( xmlStrEqual( currentName, ( const xmlChar* ) "node" ) == 1 ) { - ImportNode n = _ReadXMLNode(); - ParseNodeInLua( n, lua_state ); - extractor_callbacks->nodeFunction(n); -// if(!extractor_callbacks->nodeFunction(n)) -// std::cerr << "[XMLParser] dense node not parsed" << std::endl; - } - - if ( xmlStrEqual( currentName, ( const xmlChar* ) "way" ) == 1 ) { - ExtractionWay way = _ReadXMLWay( ); - ParseWayInLua( way, lua_state ); - extractor_callbacks->wayFunction(way); -// if(!extractor_callbacks->wayFunction(way)) -// std::cerr << "[PBFParser] way not parsed" << std::endl; - } - if( use_turn_restrictions ) { - if ( xmlStrEqual( currentName, ( const xmlChar* ) "relation" ) == 1 ) { - InputRestrictionContainer r = _ReadXMLRestriction(); - if(r.fromWay != UINT_MAX) { - if(!extractor_callbacks->restrictionFunction(r)) { - std::cerr << "[XMLParser] restriction not parsed" << std::endl; - } - } - } - } - xmlFree( currentName ); - } - return true; +bool XMLParser::ReadHeader() { return (xmlTextReaderRead(inputReader) == 1); } +bool XMLParser::Parse() +{ + while (xmlTextReaderRead(inputReader) == 1) + { + const int type = xmlTextReaderNodeType(inputReader); + + // 1 is Element + if (type != 1) + { + continue; + } + + xmlChar *currentName = xmlTextReaderName(inputReader); + if (currentName == NULL) + { + continue; + } + + if (xmlStrEqual(currentName, (const xmlChar *)"node") == 1) + { + ImportNode n = ReadXMLNode(); + ParseNodeInLua(n, lua_state); + extractor_callbacks->nodeFunction(n); + } + + if (xmlStrEqual(currentName, (const xmlChar *)"way") == 1) + { + ExtractionWay way = ReadXMLWay(); + ParseWayInLua(way, lua_state); + extractor_callbacks->wayFunction(way); + } + if (use_turn_restrictions && xmlStrEqual(currentName, (const xmlChar *)"relation") == 1) + { + InputRestrictionContainer r = ReadXMLRestriction(); + if ((UINT_MAX != r.fromWay) && !extractor_callbacks->restrictionFunction(r)) + { + std::cerr << "[XMLParser] restriction not parsed" << std::endl; + } + } + xmlFree(currentName); + } + return true; } -InputRestrictionContainer XMLParser::_ReadXMLRestriction() { +InputRestrictionContainer XMLParser::ReadXMLRestriction() +{ InputRestrictionContainer restriction; std::string except_tag_string; - if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) { - const int depth = xmlTextReaderDepth( inputReader );while ( xmlTextReaderRead( inputReader ) == 1 ) { - const int childType = xmlTextReaderNodeType( inputReader ); - if ( childType != 1 && childType != 15 ) { - continue; - } - const int childDepth = xmlTextReaderDepth( inputReader ); - xmlChar* childName = xmlTextReaderName( inputReader ); - if ( childName == NULL ) { - continue; - } - if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "relation" ) == 1 ) { - xmlFree( childName ); - break; - } - if ( childType != 1 ) { - xmlFree( childName ); - continue; - } - - if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) { - xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" ); - xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" ); - if ( k != NULL && value != NULL ) { - if(xmlStrEqual(k, ( const xmlChar* ) "restriction" )){ - if(0 == std::string((const char *) value).find("only_")) { - restriction.restriction.flags.isOnly = true; - } - } - if ( xmlStrEqual(k, (const xmlChar *) "except") ) { - except_tag_string = (const char*) value; - } - } - - if ( k != NULL ) { - xmlFree( k ); - } - if ( value != NULL ) { - xmlFree( value ); - } - } else if ( xmlStrEqual( childName, ( const xmlChar* ) "member" ) == 1 ) { - xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" ); - if ( ref != NULL ) { - xmlChar * role = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "role" ); - xmlChar * type = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "type" ); - - if(xmlStrEqual(role, (const xmlChar *) "to") && xmlStrEqual(type, (const xmlChar *) "way")) { - restriction.toWay = stringToUint((const char*) ref); - } - if(xmlStrEqual(role, (const xmlChar *) "from") && xmlStrEqual(type, (const xmlChar *) "way")) { - restriction.fromWay = stringToUint((const char*) ref); - } - if(xmlStrEqual(role, (const xmlChar *) "via") && xmlStrEqual(type, (const xmlChar *) "node")) { - restriction.restriction.viaNode = stringToUint((const char*) ref); - } - - if(NULL != type) { - xmlFree( type ); - } - if(NULL != role) { - xmlFree( role ); - } - if(NULL != ref) { - xmlFree( ref ); - } - } - } - xmlFree( childName ); - } - } - - if( ShouldIgnoreRestriction(except_tag_string) ) { - restriction.fromWay = UINT_MAX; //workaround to ignore the restriction - } - return restriction; + if (xmlTextReaderIsEmptyElement(inputReader) != 1) + { + const int depth = xmlTextReaderDepth(inputReader); + while (xmlTextReaderRead(inputReader) == 1) + { + const int childType = xmlTextReaderNodeType(inputReader); + if (childType != 1 && childType != 15) + { + continue; + } + const int childDepth = xmlTextReaderDepth(inputReader); + xmlChar *childName = xmlTextReaderName(inputReader); + if (childName == NULL) + { + continue; + } + if (depth == childDepth && childType == 15 && + xmlStrEqual(childName, (const xmlChar *)"relation") == 1) + { + xmlFree(childName); + break; + } + if (childType != 1) + { + xmlFree(childName); + continue; + } + + if (xmlStrEqual(childName, (const xmlChar *)"tag") == 1) + { + xmlChar *k = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"k"); + xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v"); + if (k != NULL && value != NULL) + { + if (xmlStrEqual(k, (const xmlChar *)"restriction") && + (0 == std::string((const char *)value).find("only_"))) + { + restriction.restriction.flags.isOnly = true; + } + if (xmlStrEqual(k, (const xmlChar *)"except")) + { + except_tag_string = (const char *)value; + } + } + + if (k != NULL) + { + xmlFree(k); + } + if (value != NULL) + { + xmlFree(value); + } + } + else if (xmlStrEqual(childName, (const xmlChar *)"member") == 1) + { + xmlChar *ref = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"ref"); + if (ref != NULL) + { + xmlChar *role = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"role"); + xmlChar *type = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"type"); + + if (xmlStrEqual(role, (const xmlChar *)"to") && + xmlStrEqual(type, (const xmlChar *)"way")) + { + restriction.toWay = stringToUint((const char *)ref); + } + if (xmlStrEqual(role, (const xmlChar *)"from") && + xmlStrEqual(type, (const xmlChar *)"way")) + { + restriction.fromWay = stringToUint((const char *)ref); + } + if (xmlStrEqual(role, (const xmlChar *)"via") && + xmlStrEqual(type, (const xmlChar *)"node")) + { + restriction.restriction.viaNode = stringToUint((const char *)ref); + } + + if (NULL != type) + { + xmlFree(type); + } + if (NULL != role) + { + xmlFree(role); + } + if (NULL != ref) + { + xmlFree(ref); + } + } + } + xmlFree(childName); + } + } + + if (ShouldIgnoreRestriction(except_tag_string)) + { + restriction.fromWay = UINT_MAX; // workaround to ignore the restriction + } + return restriction; } -ExtractionWay XMLParser::_ReadXMLWay() { - ExtractionWay way; - if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) { - const int depth = xmlTextReaderDepth( inputReader ); - while ( xmlTextReaderRead( inputReader ) == 1 ) { - const int childType = xmlTextReaderNodeType( inputReader ); - if ( childType != 1 && childType != 15 ) { - continue; - } - const int childDepth = xmlTextReaderDepth( inputReader ); - xmlChar* childName = xmlTextReaderName( inputReader ); - if ( childName == NULL ) { - continue; - } - - if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "way" ) == 1 ) { - xmlChar* id = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "id" ); - way.id = stringToUint((char*)id); - xmlFree(id); - xmlFree( childName ); - break; - } - if ( childType != 1 ) { - xmlFree( childName ); - continue; - } - - if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) { - xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" ); - xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" ); - // cout << "->k=" << k << ", v=" << value << endl; - if ( k != NULL && value != NULL ) { - way.keyVals.Add(std::string( (char *) k ), std::string( (char *) value)); - } - if ( k != NULL ) { - xmlFree( k ); - } - if ( value != NULL ) { - xmlFree( value ); - } - } else if ( xmlStrEqual( childName, ( const xmlChar* ) "nd" ) == 1 ) { - xmlChar* ref = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "ref" ); - if ( ref != NULL ) { - way.path.push_back( stringToUint(( const char* ) ref ) ); - xmlFree( ref ); - } - } - xmlFree( childName ); - } - } - return way; +ExtractionWay XMLParser::ReadXMLWay() +{ + ExtractionWay way; + if (xmlTextReaderIsEmptyElement(inputReader) != 1) + { + const int depth = xmlTextReaderDepth(inputReader); + while (xmlTextReaderRead(inputReader) == 1) + { + const int childType = xmlTextReaderNodeType(inputReader); + if (childType != 1 && childType != 15) + { + continue; + } + const int childDepth = xmlTextReaderDepth(inputReader); + xmlChar *childName = xmlTextReaderName(inputReader); + if (childName == NULL) + { + continue; + } + + if (depth == childDepth && childType == 15 && + xmlStrEqual(childName, (const xmlChar *)"way") == 1) + { + xmlChar *id = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"id"); + way.id = stringToUint((char *)id); + xmlFree(id); + xmlFree(childName); + break; + } + if (childType != 1) + { + xmlFree(childName); + continue; + } + + if (xmlStrEqual(childName, (const xmlChar *)"tag") == 1) + { + xmlChar *k = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"k"); + xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v"); + + if (k != NULL && value != NULL) + { + way.keyVals.Add(std::string((char *)k), std::string((char *)value)); + } + if (k != NULL) + { + xmlFree(k); + } + if (value != NULL) + { + xmlFree(value); + } + } + else if (xmlStrEqual(childName, (const xmlChar *)"nd") == 1) + { + xmlChar *ref = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"ref"); + if (ref != NULL) + { + way.path.push_back(stringToUint((const char *)ref)); + xmlFree(ref); + } + } + xmlFree(childName); + } + } + return way; } -ImportNode XMLParser::_ReadXMLNode() { - ImportNode node; - - xmlChar* attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "lat" ); - if ( attribute != NULL ) { - node.lat = static_cast(COORDINATE_PRECISION*atof(( const char* ) attribute ) ); - xmlFree( attribute ); - } - attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "lon" ); - if ( attribute != NULL ) { - node.lon = static_cast(COORDINATE_PRECISION*atof(( const char* ) attribute )); - xmlFree( attribute ); - } - attribute = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "id" ); - if ( attribute != NULL ) { - node.id = stringToUint(( const char* ) attribute ); - xmlFree( attribute ); - } - - if ( xmlTextReaderIsEmptyElement( inputReader ) != 1 ) { - const int depth = xmlTextReaderDepth( inputReader ); - while ( xmlTextReaderRead( inputReader ) == 1 ) { - const int childType = xmlTextReaderNodeType( inputReader ); - // 1 = Element, 15 = EndElement - if ( childType != 1 && childType != 15 ) { - continue; - } - const int childDepth = xmlTextReaderDepth( inputReader ); - xmlChar* childName = xmlTextReaderName( inputReader ); - if ( childName == NULL ) { - continue; - } - - if ( depth == childDepth && childType == 15 && xmlStrEqual( childName, ( const xmlChar* ) "node" ) == 1 ) { - xmlFree( childName ); - break; - } - if ( childType != 1 ) { - xmlFree( childName ); - continue; - } - - if ( xmlStrEqual( childName, ( const xmlChar* ) "tag" ) == 1 ) { - xmlChar* k = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "k" ); - xmlChar* value = xmlTextReaderGetAttribute( inputReader, ( const xmlChar* ) "v" ); - if ( k != NULL && value != NULL ) { - node.keyVals.emplace(std::string(reinterpret_cast(k)), std::string(reinterpret_cast(value))); - } - if ( k != NULL ) { - xmlFree( k ); - } - if ( value != NULL ) { - xmlFree( value ); - } - } - - xmlFree( childName ); - } - } - return node; +ImportNode XMLParser::ReadXMLNode() +{ + ImportNode node; + + xmlChar *attribute = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"lat"); + if (attribute != NULL) + { + node.lat = COORDINATE_PRECISION * StringToDouble((const char *)attribute); + xmlFree(attribute); + } + attribute = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"lon"); + if (attribute != NULL) + { + node.lon = COORDINATE_PRECISION * StringToDouble((const char *)attribute); + xmlFree(attribute); + } + attribute = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"id"); + if (attribute != NULL) + { + node.id = stringToUint((const char *)attribute); + xmlFree(attribute); + } + + if (xmlTextReaderIsEmptyElement(inputReader) != 1) + { + const int depth = xmlTextReaderDepth(inputReader); + while (xmlTextReaderRead(inputReader) == 1) + { + const int childType = xmlTextReaderNodeType(inputReader); + // 1 = Element, 15 = EndElement + if (childType != 1 && childType != 15) + { + continue; + } + const int childDepth = xmlTextReaderDepth(inputReader); + xmlChar *childName = xmlTextReaderName(inputReader); + if (childName == NULL) + { + continue; + } + + if (depth == childDepth && childType == 15 && + xmlStrEqual(childName, (const xmlChar *)"node") == 1) + { + xmlFree(childName); + break; + } + if (childType != 1) + { + xmlFree(childName); + continue; + } + + if (xmlStrEqual(childName, (const xmlChar *)"tag") == 1) + { + xmlChar *k = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"k"); + xmlChar *value = xmlTextReaderGetAttribute(inputReader, (const xmlChar *)"v"); + if (k != NULL && value != NULL) + { + node.keyVals.emplace(std::string((char *)(k)), + std::string((char *)(value))); + } + if (k != NULL) + { + xmlFree(k); + } + if (value != NULL) + { + xmlFree(value); + } + } + + xmlFree(childName); + } + } + return node; } diff --git a/Extractor/XMLParser.h b/Extractor/XMLParser.h index 7c88d48a7c..26b6717d97 100644 --- a/Extractor/XMLParser.h +++ b/Extractor/XMLParser.h @@ -33,20 +33,17 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -class XMLParser : public BaseParser { -public: - XMLParser( - const char* filename, - ExtractorCallbacks * ec, - ScriptingEnvironment & se - ); +class XMLParser : public BaseParser +{ + public: + XMLParser(const char *filename, ExtractorCallbacks *ec, ScriptingEnvironment &se); bool ReadHeader(); bool Parse(); -private: - InputRestrictionContainer _ReadXMLRestriction(); - ExtractionWay _ReadXMLWay(); - ImportNode _ReadXMLNode(); + private: + InputRestrictionContainer ReadXMLRestriction(); + ExtractionWay ReadXMLWay(); + ImportNode ReadXMLNode(); xmlTextReaderPtr inputReader; }; diff --git a/Util/StringUtil.h b/Util/StringUtil.h index 158928820a..633feea55a 100644 --- a/Util/StringUtil.h +++ b/Util/StringUtil.h @@ -126,6 +126,40 @@ static inline uint64_t stringToInt64(const std::string& input) { return value; } +// source: http://tinodidriksen.com/2011/05/28/cpp-convert-string-to-double-speed/ +static inline double StringToDouble(const char *p) +{ + double r = 0.0; + bool neg = false; + if (*p == '-') + { + neg = true; + ++p; + } + while (*p >= '0' && *p <= '9') + { + r = (r*10.0) + (*p - '0'); + ++p; + } + if (*p == '.') + { + double f = 0.0; + int n = 0; + ++p; + while (*p >= '0' && *p <= '9') + { + f = (f*10.0) + (*p - '0'); + ++p; + ++n; + } + r += f / std::pow(10.0, n); + } + if (neg) + { + r = -r; + } + return r; +} static inline void doubleToString(const double value, std::string & output){ output.clear();