From 3c7abf95d7857a37d5acc4f81a3261a26c87ba9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaakko=20Kera=CC=88nen?= Date: Thu, 17 Nov 2016 11:01:39 +0200 Subject: [PATCH] Resources|UDMF: Parsing the UDMF TEXTMAP lump contents When a UDMF map is recognized, the importudmf plugin is used to read its contents. UDMFLex is based on de::Lex for tokenizing the UDMF source, and UDMFParser makes a callback for each encountered global assignment and block. --- .../plugins/importudmf/include/importudmf.h | 2 +- .../apps/plugins/importudmf/include/udmflex.h | 62 +++++++ .../plugins/importudmf/include/udmfparser.h | 65 +++++++ .../plugins/importudmf/src/importudmf.cpp | 63 ++++--- .../apps/plugins/importudmf/src/udmflex.cpp | 146 +++++++++++++++ .../plugins/importudmf/src/udmfparser.cpp | 166 ++++++++++++++++++ 6 files changed, 474 insertions(+), 30 deletions(-) create mode 100644 doomsday/apps/plugins/importudmf/include/udmflex.h create mode 100644 doomsday/apps/plugins/importudmf/include/udmfparser.h create mode 100644 doomsday/apps/plugins/importudmf/src/udmflex.cpp create mode 100644 doomsday/apps/plugins/importudmf/src/udmfparser.cpp diff --git a/doomsday/apps/plugins/importudmf/include/importudmf.h b/doomsday/apps/plugins/importudmf/include/importudmf.h index 3b776451dd..242b49adfb 100644 --- a/doomsday/apps/plugins/importudmf/include/importudmf.h +++ b/doomsday/apps/plugins/importudmf/include/importudmf.h @@ -23,7 +23,7 @@ #include // Doomsday's Public API //DENG_USING_API(Base); -//DENG_USING_API(F); +DENG_USING_API(F); DENG_USING_API(Map); DENG_USING_API(Material); DENG_USING_API(MPE); diff --git a/doomsday/apps/plugins/importudmf/include/udmflex.h b/doomsday/apps/plugins/importudmf/include/udmflex.h new file mode 100644 index 0000000000..37ef727797 --- /dev/null +++ b/doomsday/apps/plugins/importudmf/include/udmflex.h @@ -0,0 +1,62 @@ +/** @file udmflex.h UDMF lexical analyzer. + * + * @authors Copyright (c) 2016 Jaakko Keränen + * + * @par License + * GPL: http://www.gnu.org/licenses/gpl.html + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. This program is distributed in the hope that it + * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. You should have received a copy of the GNU + * General Public License along with this program; if not, see: + * http://www.gnu.org/licenses + */ + +#ifndef IMPORTUDMF_UDMFLEX_H +#define IMPORTUDMF_UDMFLEX_H + +#include +#include + +class UDMFLex : public de::Lex +{ +public: + // Keywords. + static de::String const NAMESPACE; + static de::String const LINEDEF; + static de::String const SIDEDEF; + static de::String const VERTEX; + static de::String const SECTOR; + static de::String const THING; + static de::String const T_TRUE; + static de::String const T_FALSE; + + // Operators. + static de::String const ASSIGN; + + // Literals. + static de::String const BRACKET_OPEN; + static de::String const BRACKET_CLOSE; + static de::String const SEMICOLON; + +public: + UDMFLex(de::String const &input = ""); + + de::dsize getExpression(de::TokenBuffer &output); + + /** + * Parse a string. + * + * @param output Output token buffer. + */ + void parseString(de::TokenBuffer &output); + + /// Determines whether a token is a keyword. + static bool isKeyword(de::Token const &token); +}; + +#endif // IMPORTUDMF_UDMFLEX_H diff --git a/doomsday/apps/plugins/importudmf/include/udmfparser.h b/doomsday/apps/plugins/importudmf/include/udmfparser.h new file mode 100644 index 0000000000..5b2686efe2 --- /dev/null +++ b/doomsday/apps/plugins/importudmf/include/udmfparser.h @@ -0,0 +1,65 @@ +/** @file udmfparser.h UDMF parser. + * + * @authors Copyright (c) 2016 Jaakko Keränen + * + * @par License + * GPL: http://www.gnu.org/licenses/gpl.html + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. This program is distributed in the hope that it + * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. You should have received a copy of the GNU + * General Public License along with this program; if not, see: + * http://www.gnu.org/licenses + */ + +#ifndef IMPORTUDMF_UDMFPARSER_H +#define IMPORTUDMF_UDMFPARSER_H + +#include "udmflex.h" +#include +#include +#include + +/** + * UMDF parser. + * + * Reads input text and makes callbacks for each parsed block. + */ +class UDMFParser +{ +public: + typedef QHash Block; + typedef std::function AssignmentFunc; + typedef std::function BlockFunc; + + DENG2_ERROR(SyntaxError); + +public: + UDMFParser(); + + void setGlobalAssignmentHandler(AssignmentFunc func); + void setBlockHandler(BlockFunc func); + + Block const &globals() const; + + void parse(de::String const &input); + +protected: + de::dsize nextExpression(); + void parseBlock(Block &block); + void parseAssignment(Block &block); + +private: + AssignmentFunc _assignmentHandler; + BlockFunc _blockHandler; + UDMFLex _analyzer; + de::TokenBuffer _tokens; + de::TokenRange _range; + Block _globals; +}; + +#endif // IMPORTUDMF_UDMFPARSER_H diff --git a/doomsday/apps/plugins/importudmf/src/importudmf.cpp b/doomsday/apps/plugins/importudmf/src/importudmf.cpp index 07a2caeb1c..43d4b086cf 100644 --- a/doomsday/apps/plugins/importudmf/src/importudmf.cpp +++ b/doomsday/apps/plugins/importudmf/src/importudmf.cpp @@ -18,6 +18,7 @@ */ #include "importudmf.h" +#include "udmfparser.h" #include #include @@ -32,36 +33,42 @@ using namespace de; * Our job is to read in the map data structures then use the Doomsday map editing * interface to recreate the map in native format. */ -static int convertMapHook(int /*hookType*/, int /*parm*/, void *context) +static int importMapHook(int /*hookType*/, int /*parm*/, void *context) { - Id1MapRecognizer const *recognizer = reinterpret_cast(context); - if (!recognizer) return false; - - if (recognizer->format() == Id1MapRecognizer::UniversalFormat) + if (Id1MapRecognizer const *recognizer = reinterpret_cast(context)) { - LOG_AS("UDMFConverter"); - - // Attempt a conversion... - try + if (recognizer->format() == Id1MapRecognizer::UniversalFormat) { -#if 0 - std::unique_ptr map(new MapImporter(recognizer)); + LOG_AS("importudmf"); + try + { + // Read the contents of the TEXTMAP lump. + auto *src = recognizer->lumps()[Id1MapRecognizer::UDMFTextmapData]; + Block bytes(src->size()); + src->read(bytes.data(), false); - // The archived map data was read successfully. - // Transfer to the engine via the runtime map editing interface. - /// @todo Build it using native components directly... - LOG_AS("importidtech1"); - map->transfer(); -#endif - return true; // success - } - catch (Error const &er) - { - LOG_MAP_ERROR("Erroring while loading UDMF: ") << er.asText(); + String const source = String::fromUtf8(bytes); + + // Parse the UDMF source and use the MPE API to create the map elements. + UDMFParser parser; + parser.setGlobalAssignmentHandler([] (String const &ident, QVariant const &value) + { + qDebug() << "Global:" << ident << value; + }); + parser.setBlockHandler([] (String const &type, UDMFParser::Block const &block) + { + qDebug() << "Block" << type << block; + }); + parser.parse(source); + return true; + } + catch (Error const &er) + { + LOG_MAP_ERROR("Erroring while loading UDMF: ") << er.asText(); + } } } - - return false; // failure :( + return false; } /** @@ -70,7 +77,7 @@ static int convertMapHook(int /*hookType*/, int /*parm*/, void *context) */ extern "C" void DP_Initialize() { - Plug_AddHook(HOOK_MAP_CONVERT, convertMapHook); + Plug_AddHook(HOOK_MAP_CONVERT, importMapHook); } /** @@ -83,17 +90,15 @@ extern "C" char const *deng_LibraryType() } //DENG_DECLARE_API(Base); -//DENG_DECLARE_API(F); +DENG_DECLARE_API(F); DENG_DECLARE_API(Map); DENG_DECLARE_API(Material); DENG_DECLARE_API(MPE); -//DENG_DECLARE_API(Uri); DENG_API_EXCHANGE( //DENG_GET_API(DE_API_BASE, Base); - //DENG_GET_API(DE_API_FILE_SYSTEM, F); + DENG_GET_API(DE_API_FILE_SYSTEM, F); DENG_GET_API(DE_API_MAP, Map); DENG_GET_API(DE_API_MATERIALS, Material); DENG_GET_API(DE_API_MAP_EDIT, MPE); - //DENG_GET_API(DE_API_URI, Uri); ) diff --git a/doomsday/apps/plugins/importudmf/src/udmflex.cpp b/doomsday/apps/plugins/importudmf/src/udmflex.cpp new file mode 100644 index 0000000000..266a3abdb6 --- /dev/null +++ b/doomsday/apps/plugins/importudmf/src/udmflex.cpp @@ -0,0 +1,146 @@ +/** @file udmflex.cpp UDMF lexical analyzer. + * + * @authors Copyright (c) 2016 Jaakko Keränen + * + * @par License + * GPL: http://www.gnu.org/licenses/gpl.html + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. This program is distributed in the hope that it + * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. You should have received a copy of the GNU + * General Public License along with this program; if not, see: + * http://www.gnu.org/licenses + */ + +#include "udmflex.h" + +using namespace de; + +String const UDMFLex::NAMESPACE("namespace"); +String const UDMFLex::LINEDEF("linedef"); +String const UDMFLex::SIDEDEF("sidedef"); +String const UDMFLex::VERTEX("vertex"); +String const UDMFLex::SECTOR("sector"); +String const UDMFLex::THING("thing"); +String const UDMFLex::T_TRUE("true"); +String const UDMFLex::T_FALSE("false"); +String const UDMFLex::ASSIGN("="); +String const UDMFLex::BRACKET_OPEN("{"); +String const UDMFLex::BRACKET_CLOSE("}"); +String const UDMFLex::SEMICOLON(";"); + +UDMFLex::UDMFLex(String const &input) + : Lex(input, QChar('/'), DoubleCharComment) +{} + +dsize UDMFLex::getExpression(TokenBuffer &output) +{ + output.clear(); + + while (!atEnd()) + { + skipWhite(); + + if (atEnd() || (output.size() && peek() == '}')) break; + + // First character of the token. + QChar c = get(); + + output.newToken(lineNumber()); + output.appendChar(c); + + // Single-character tokens. + if (c == '{' || c == '}' || c == '=' || c == ';') + { + output.setType(c == '='? Token::OPERATOR : Token::LITERAL); + output.endToken(); + + if (output.latest().type() != Token::OPERATOR) break; + continue; + } + + if (c == '"') + { + // Parse the string into one token. + output.setType(Token::LITERAL_STRING_QUOTED); + parseString(output); + output.endToken(); + continue; + } + + // Number literal? + if (parseLiteralNumber(c, output)) + { + continue; + } + + // Alphanumeric characters are joined into a token. + if (c == '_' || c.isLetter()) + { + output.setType(Token::IDENTIFIER); + + while (isAlphaNumeric((c = peek()))) + { + output.appendChar(get()); + } + + // It might be that this is a keyword. + if (isKeyword(output.latest())) + { + output.setType(Token::KEYWORD); + } + + output.endToken(); + continue; + } + } + + return output.size(); +} + +void UDMFLex::parseString(TokenBuffer &output) +{ + ModeSpan readingMode(*this, SkipComments); + + // The token already contains the first quote char. + // This will throw an exception if the string is unterminated. + forever + { + QChar c = get(); + output.appendChar(c); + if (c == '"') + { + return; + } + if (c == '\\') // Escape. + { + output.appendChar(get()); + } + } +} + +bool UDMFLex::isKeyword(Token const &token) +{ + static QVector const keywordStr + { + NAMESPACE, + LINEDEF, + SIDEDEF, + VERTEX, + SECTOR, + THING, + T_TRUE, + T_FALSE, + }; + foreach (auto const &kw, keywordStr) + { + if (!kw.compareWithoutCase(token.str())) + return true; + } + return false; +} + diff --git a/doomsday/apps/plugins/importudmf/src/udmfparser.cpp b/doomsday/apps/plugins/importudmf/src/udmfparser.cpp new file mode 100644 index 0000000000..843baa9526 --- /dev/null +++ b/doomsday/apps/plugins/importudmf/src/udmfparser.cpp @@ -0,0 +1,166 @@ +/** @file udmfparser.cpp UDMF parser. + * + * @authors Copyright (c) 2016 Jaakko Keränen + * + * @par License + * GPL: http://www.gnu.org/licenses/gpl.html + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. This program is distributed in the hope that it + * will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. You should have received a copy of the GNU + * General Public License along with this program; if not, see: + * http://www.gnu.org/licenses + */ + +#include "udmfparser.h" + +using namespace de; + +UDMFParser::UDMFParser() +{} + +void UDMFParser::setGlobalAssignmentHandler(UDMFParser::AssignmentFunc func) +{ + _assignmentHandler = func; +} + +void UDMFParser::setBlockHandler(UDMFParser::BlockFunc func) +{ + _blockHandler = func; +} + +UDMFParser::Block const &UDMFParser::globals() const +{ + return _globals; +} + +void UDMFParser::parse(String const &input) +{ + // Lexical analyzer for Haw scripts. + _analyzer = UDMFLex(input); + + while (nextExpression() > 0) + { + if (_range.lastToken().equals(UDMFLex::BRACKET_OPEN)) + { + String const blockType = _range.firstToken().str(); + Block block; + parseBlock(block); + + if (_blockHandler) + { + _blockHandler(blockType, block); + } + } + else + { + parseAssignment(_globals); + } + } + + // We're done, free the remaining tokens. + _tokens.clear(); +} + +dsize UDMFParser::nextExpression() +{ + _analyzer.getExpression(_tokens); + + // Begin with the whole thing. + _range = TokenRange(_tokens); + + return _tokens.size(); +} + +void UDMFParser::parseBlock(Block &block) +{ + // Read all the assignments in the block. + while (nextExpression() > 0) + { + if (_range.firstToken().equals(UDMFLex::BRACKET_CLOSE)) + break; + + parseAssignment(block); + } +} + +void UDMFParser::parseAssignment(Block &block) +{ + if (_range.isEmpty()) + return; // Nothing here? + + if (!_range.lastToken().equals(UDMFLex::SEMICOLON)) + { + throw SyntaxError("UDMFParser::parseAssignment", + "Expected expression to end in a semicolon at " + + _range.lastToken().asText()); + } + if (_range.size() == 1) + return; // Just a semicolon? + + if (!_range.token(1).equals(UDMFLex::ASSIGN)) + { + throw SyntaxError("UDMFParser::parseAssignment", + "Expected expression to have an assignment operator at " + + _range.token(1).asText()); + } + + String const identifier = _range.firstToken().str(); + Token const &valueToken = _range.token(2); + + // Store the assigned value into a variant. + QVariant value; + switch (valueToken.type()) + { + case Token::KEYWORD: + if (valueToken.equals(UDMFLex::T_TRUE)) + { + value.setValue(true); + } + else if (valueToken.equals(UDMFLex::T_FALSE)) + { + value.setValue(false); + } + else + { + throw SyntaxError("UDMFParser::parseAssignment", + "Unexpected value for assignment at " + valueToken.asText()); + } + break; + + case Token::LITERAL_NUMBER: + if (valueToken.isInteger()) + { + value.setValue(valueToken.toInteger()); + } + else + { + value.setValue(valueToken.toDouble()); + } + break; + + case Token::LITERAL_STRING_QUOTED: + value.setValue(QString(valueToken.unescapeStringLiteral())); + break; + + case Token::IDENTIFIER: + value.setValue(QString(valueToken.str())); + break; + + default: + DENG2_ASSERT(0!="[UMDFParser::parseAssignment] Invalid token type"); + break; + } + + block.insert(identifier, value); + + // Handler global assignments. + if (_assignmentHandler && (&block == &_globals)) + { + _assignmentHandler(identifier, value); + } +}