From a9896bcb5d7d7bed312925ce57811d85e1713cca Mon Sep 17 00:00:00 2001 From: codereader Date: Fri, 24 Jun 2022 19:55:08 +0200 Subject: [PATCH] #5977: Some restructuring to make the parsing logic reusable outside the ThreadedDeclParser. --- radiantcore/CMakeLists.txt | 3 +- radiantcore/decl/DeclarationFile.h | 35 ++++++++++ ...onParser.cpp => DeclarationFileParser.cpp} | 67 ++++++++++++------- ...rationParser.h => DeclarationFileParser.h} | 23 +++---- radiantcore/decl/DeclarationFolderParser.cpp | 28 ++++++++ radiantcore/decl/DeclarationFolderParser.h | 35 ++++++++++ radiantcore/decl/DeclarationManager.cpp | 47 +++++++++++-- radiantcore/decl/DeclarationManager.h | 15 +++-- tools/msvc/DarkRadiantCore.vcxproj | 8 ++- tools/msvc/DarkRadiantCore.vcxproj.filters | 16 ++++- 10 files changed, 223 insertions(+), 54 deletions(-) create mode 100644 radiantcore/decl/DeclarationFile.h rename radiantcore/decl/{DeclarationParser.cpp => DeclarationFileParser.cpp} (50%) rename radiantcore/decl/{DeclarationParser.h => DeclarationFileParser.h} (53%) create mode 100644 radiantcore/decl/DeclarationFolderParser.cpp create mode 100644 radiantcore/decl/DeclarationFolderParser.h diff --git a/radiantcore/CMakeLists.txt b/radiantcore/CMakeLists.txt index f9d6bab70b..6eb6b0e668 100644 --- a/radiantcore/CMakeLists.txt +++ b/radiantcore/CMakeLists.txt @@ -23,7 +23,8 @@ add_library(radiantcore MODULE clipper/ClipPoint.cpp clipper/SplitAlgorithm.cpp commandsystem/CommandSystem.cpp - decl/DeclarationParser.cpp + decl/DeclarationFileParser.cpp + decl/DeclarationFolderParser.cpp decl/DeclarationManager.cpp decl/FavouritesManager.cpp eclass/EntityClass.cpp diff --git a/radiantcore/decl/DeclarationFile.h b/radiantcore/decl/DeclarationFile.h new file mode 100644 index 0000000000..81b7d5c1ee --- /dev/null +++ b/radiantcore/decl/DeclarationFile.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include "idecltypes.h" + +namespace decl +{ + +class DeclarationFile +{ +public: + // mod-relative path + std::string fullPath; + + // The default type of this file, used for all + // declarations without explicitly declared type + decl::Type defaultDeclType; + + bool operator< (const DeclarationFile& other) const + { + if (defaultDeclType < other.defaultDeclType) + { + return true; + } + + if (defaultDeclType == other.defaultDeclType) + { + return fullPath < other.fullPath; + } + + return false; + } +}; + +} diff --git a/radiantcore/decl/DeclarationParser.cpp b/radiantcore/decl/DeclarationFileParser.cpp similarity index 50% rename from radiantcore/decl/DeclarationParser.cpp rename to radiantcore/decl/DeclarationFileParser.cpp index 13c5f0cc9e..4d46141f34 100644 --- a/radiantcore/decl/DeclarationParser.cpp +++ b/radiantcore/decl/DeclarationFileParser.cpp @@ -1,41 +1,62 @@ -#include "DeclarationParser.h" +#include "DeclarationFileParser.h" #include "DeclarationManager.h" +#include "parser/DefBlockTokeniser.h" namespace decl { -inline DeclarationBlockSyntax createBlock(const parser::BlockTokeniser::Block& block, - const vfs::FileInfo& fileInfo, const std::string& modName) +namespace { - auto spacePos = block.name.find(' '); + inline DeclarationBlockSyntax createBlock(const parser::BlockTokeniser::Block& block, + const vfs::FileInfo& fileInfo, const std::string& modName) + { + auto spacePos = block.name.find(' '); - DeclarationBlockSyntax syntax; + DeclarationBlockSyntax syntax; - syntax.typeName = spacePos != std::string::npos ? block.name.substr(0, spacePos) : std::string(); - syntax.name = spacePos != std::string::npos ? block.name.substr(spacePos + 1) : block.name; - syntax.contents = block.contents; - syntax.modName = modName; - syntax.fileInfo = fileInfo; + syntax.typeName = spacePos != std::string::npos ? block.name.substr(0, spacePos) : std::string(); + syntax.name = spacePos != std::string::npos ? block.name.substr(spacePos + 1) : block.name; + syntax.contents = block.contents; + syntax.modName = modName; + syntax.fileInfo = fileInfo; - return syntax; + return syntax; + } } -DeclarationParser::DeclarationParser(DeclarationManager& owner, Type declType, - const std::string& baseDir, const std::string& extension, +DeclarationFileParser::DeclarationFileParser(Type declType, const std::map& parsersByTypename) : - ThreadedDeclParser(declType, baseDir, extension, 1), - _owner(owner), _defaultDeclType(declType), _parsersByTypename(parsersByTypename) { _defaultTypeParser = getParserByType(declType); - if (!_defaultTypeParser) throw std::invalid_argument("No parser has been associated to the default type " + getTypeName(declType)); + if (!_defaultTypeParser) + { + throw std::invalid_argument("No parser has been associated to the default type " + getTypeName(declType)); + } +} + +std::map& DeclarationFileParser::getParsedDecls() +{ + return _parsedDecls; +} + +const std::set& DeclarationFileParser::getParsedFiles() const +{ + return _parsedFiles; +} + +const std::vector& DeclarationFileParser::getUnrecognisedBlocks() const +{ + return _unrecognisedBlocks; } -void DeclarationParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir) +void DeclarationFileParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir) { + _parsedFiles.emplace(DeclarationFile{ fileInfo.fullPath(), _defaultDeclType }); + // Cut the incoming stream into declaration blocks parser::BasicDefBlockTokeniser tokeniser(stream); @@ -69,15 +90,9 @@ void DeclarationParser::parse(std::istream& stream, const vfs::FileInfo& fileInf } } -void DeclarationParser::onFinishParsing() -{ - // Submit all parsed declarations to the decl manager - _owner.onParserFinished(_defaultDeclType, std::move(_parsedDecls), std::move(_unrecognisedBlocks)); -} - -void DeclarationParser::parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& block) +void DeclarationFileParser::parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& block) { - auto declaration = parser.parseFromBlock(block); + auto declaration = parser.parseFromBlock(block); auto& declMap = _parsedDecls.try_emplace(parser.getDeclType(), NamedDeclarations()).first->second; @@ -85,7 +100,7 @@ void DeclarationParser::parseBlock(IDeclarationParser& parser, const Declaration DeclarationManager::InsertDeclaration(declMap, std::move(declaration)); } -IDeclarationParser::Ptr DeclarationParser::getParserByType(Type declType) const +IDeclarationParser::Ptr DeclarationFileParser::getParserByType(Type declType) const { // Get the default type parser for (const auto& pair : _parsersByTypename) diff --git a/radiantcore/decl/DeclarationParser.h b/radiantcore/decl/DeclarationFileParser.h similarity index 53% rename from radiantcore/decl/DeclarationParser.h rename to radiantcore/decl/DeclarationFileParser.h index 859f924b14..02ef3d60e3 100644 --- a/radiantcore/decl/DeclarationParser.h +++ b/radiantcore/decl/DeclarationFileParser.h @@ -1,41 +1,38 @@ #pragma once #include "ideclmanager.h" - -#include "parser/DefBlockTokeniser.h" -#include "parser/ThreadedDeclParser.h" +#include "DeclarationFile.h" namespace decl { class DeclarationManager; -class DeclarationParser : - public parser::ThreadedDeclParser +class DeclarationFileParser { private: - DeclarationManager& _owner; Type _defaultDeclType; std::map _parsersByTypename; IDeclarationParser::Ptr _defaultTypeParser; + std::set _parsedFiles; std::map _parsedDecls; std::vector _unrecognisedBlocks; public: - DeclarationParser(DeclarationManager& owner, Type declType, - const std::string& baseDir, const std::string& extension, - const std::map& parsersByTypename); + DeclarationFileParser(Type declType, const std::map& parsersByTypename); + + void parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir); -protected: - void parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir) override; - void onFinishParsing() override; + std::map& getParsedDecls(); + const std::set& getParsedFiles() const; + const std::vector& getUnrecognisedBlocks() const; private: + void parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& block); IDeclarationParser::Ptr getParserByType(Type declType) const; - void parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& blockSyntax); }; } diff --git a/radiantcore/decl/DeclarationFolderParser.cpp b/radiantcore/decl/DeclarationFolderParser.cpp new file mode 100644 index 0000000000..b76c697ca9 --- /dev/null +++ b/radiantcore/decl/DeclarationFolderParser.cpp @@ -0,0 +1,28 @@ +#include "DeclarationFolderParser.h" + +#include "DeclarationManager.h" + +namespace decl +{ + +DeclarationFolderParser::DeclarationFolderParser(DeclarationManager& owner, Type declType, + const std::string& baseDir, const std::string& extension, + const std::map& parsersByTypename) : + ThreadedDeclParser(declType, baseDir, extension, 1), + _owner(owner), + _fileParser(declType, parsersByTypename), + _defaultDeclType(declType) +{} + +void DeclarationFolderParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir) +{ + _fileParser.parse(stream, fileInfo, modDir); +} + +void DeclarationFolderParser::onFinishParsing() +{ + // Submit all parsed declarations to the decl manager + _owner.onParserFinished(_defaultDeclType, _fileParser.getParsedDecls(), _fileParser.getUnrecognisedBlocks(), _fileParser.getParsedFiles()); +} + +} diff --git a/radiantcore/decl/DeclarationFolderParser.h b/radiantcore/decl/DeclarationFolderParser.h new file mode 100644 index 0000000000..9efeae3013 --- /dev/null +++ b/radiantcore/decl/DeclarationFolderParser.h @@ -0,0 +1,35 @@ +#pragma once + +#include "DeclarationFile.h" +#include "DeclarationFileParser.h" +#include "ideclmanager.h" + +#include "parser/ThreadedDeclParser.h" + +namespace decl +{ + +class DeclarationManager; + +// Threaded parser processing all files in the configured decl folder +// Submits all parsed declarations to the IDeclarationManager when finished +class DeclarationFolderParser : + public parser::ThreadedDeclParser +{ +private: + DeclarationManager& _owner; + + DeclarationFileParser _fileParser; + Type _defaultDeclType; + +public: + DeclarationFolderParser(DeclarationManager& owner, Type declType, + const std::string& baseDir, const std::string& extension, + const std::map& parsersByTypename); + +protected: + void parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir) override; + void onFinishParsing() override; +}; + +} diff --git a/radiantcore/decl/DeclarationManager.cpp b/radiantcore/decl/DeclarationManager.cpp index bc0616574c..807e80202e 100644 --- a/radiantcore/decl/DeclarationManager.cpp +++ b/radiantcore/decl/DeclarationManager.cpp @@ -2,16 +2,18 @@ #include "DeclarationManager.h" +#include "DeclarationFolderParser.h" #include "ifilesystem.h" #include "module/StaticModule.h" #include "string/trim.h" -#include "DeclarationParser.h" namespace decl { void DeclarationManager::registerDeclType(const std::string& typeName, const IDeclarationParser::Ptr& parser) { + std::lock_guard parserLock(_parserLock); + if (_parsersByTypename.count(typeName) > 0) { throw std::logic_error("Type name " + typeName + " has already been registered"); @@ -25,6 +27,8 @@ void DeclarationManager::registerDeclType(const std::string& typeName, const IDe void DeclarationManager::unregisterDeclType(const std::string& typeName) { + std::lock_guard parserLock(_parserLock); + auto existing = _parsersByTypename.find(typeName); if (existing == _parsersByTypename.end()) @@ -47,7 +51,7 @@ void DeclarationManager::registerDeclFolder(Type defaultType, const std::string& auto& decls = _declarationsByType.try_emplace(defaultType, Declarations()).first->second; // Start the parser thread - decls.parser = std::make_unique(*this, defaultType, vfsPath, extension, _parsersByTypename); + decls.parser = std::make_unique(*this, defaultType, vfsPath, extension, _parsersByTypename); decls.parser->start(); } @@ -105,7 +109,25 @@ void DeclarationManager::doWithDeclarations(Type type, const std::function fileLock(_parsedFileLock); + std::lock_guard parserLock(_parserLock); + + for (const auto& [type, files] : _parsedFilesByDefaultType) + { + DeclarationFileParser parser(type, _parsersByTypename); + + for (const auto& file : files) + { + auto vfsFile = GlobalFileSystem().openTextFile(file.fullPath); + + if (!vfsFile) continue; + + // TODO + } + + // Submit the changes + // TODO + } } sigc::signal& DeclarationManager::signal_DeclsReloaded(Type type) @@ -113,8 +135,8 @@ sigc::signal& DeclarationManager::signal_DeclsReloaded(Type type) return _declsReloadedSignals.try_emplace(type).first->second; } -void DeclarationManager::onParserFinished(Type parserType, std::map&& parsedDecls, - std::vector&& unrecognisedBlocks) +void DeclarationManager::onParserFinished(Type parserType, std::map& parsedDecls, + const std::vector& unrecognisedBlocks, const std::set& parsedFiles) { { std::lock_guard declLock(_declarationLock); @@ -139,6 +161,18 @@ void DeclarationManager::onParserFinished(Type parserType, std::map lock(_parsedFileLock); + + // Merge all parsed files into the main set + for (const auto& parsedFile : parsedFiles) + { + auto& fileSet = _parsedFilesByDefaultType.try_emplace(parsedFile.defaultDeclType, std::set()).first->second; + + fileSet.emplace(parsedFile); + } + } + // We might have received a parser in the meantime handleUnrecognisedBlocks(); @@ -210,7 +244,7 @@ void DeclarationManager::initialiseModule(const IApplicationContext& ctx) void DeclarationManager::shutdownModule() { auto declLock = std::make_unique>(_declarationLock); - std::vector> parsersToFinish; + std::vector> parsersToFinish; for (auto& [_, decl] : _declarationsByType) { @@ -226,6 +260,7 @@ void DeclarationManager::shutdownModule() // All parsers have finished, clear the structure, no need to lock anything _registeredFolders.clear(); + _parsedFilesByDefaultType.clear(); _unrecognisedBlocks.clear(); _declarationsByType.clear(); _parsersByTypename.clear(); diff --git a/radiantcore/decl/DeclarationManager.h b/radiantcore/decl/DeclarationManager.h index 4c499d6c7b..481e9159a0 100644 --- a/radiantcore/decl/DeclarationManager.h +++ b/radiantcore/decl/DeclarationManager.h @@ -2,19 +2,23 @@ #include "ideclmanager.h" #include +#include #include #include +#include "DeclarationFile.h" + namespace decl { -class DeclarationParser; +class DeclarationFolderParser; class DeclarationManager : public IDeclarationManager { private: std::map _parsersByTypename; + std::mutex _parserLock; struct RegisteredFolder { @@ -25,13 +29,16 @@ class DeclarationManager : std::vector _registeredFolders; + std::map> _parsedFilesByDefaultType; + std::mutex _parsedFileLock; + struct Declarations { // The decl library NamedDeclarations decls; // If not empty, holds the running parser - std::unique_ptr parser; + std::unique_ptr parser; }; // One entry for each decl @@ -58,8 +65,8 @@ class DeclarationManager : void shutdownModule() override; // Invoked once a parser thread has finished. It will move its data over to here. - void onParserFinished(Type parserType, std::map&& parsedDecls, - std::vector&& unrecognisedBlocks); + void onParserFinished(Type parserType, std::map& parsedDecls, + const std::vector& unrecognisedBlocks, const std::set& parsedFiles); static void InsertDeclaration(NamedDeclarations& map, IDeclaration::Ptr&& declaration); diff --git a/tools/msvc/DarkRadiantCore.vcxproj b/tools/msvc/DarkRadiantCore.vcxproj index 287ea79893..778701c769 100644 --- a/tools/msvc/DarkRadiantCore.vcxproj +++ b/tools/msvc/DarkRadiantCore.vcxproj @@ -38,8 +38,9 @@ + + - @@ -760,8 +761,11 @@ + + + - + diff --git a/tools/msvc/DarkRadiantCore.vcxproj.filters b/tools/msvc/DarkRadiantCore.vcxproj.filters index 3d485c5edb..04f3035b7f 100644 --- a/tools/msvc/DarkRadiantCore.vcxproj.filters +++ b/tools/msvc/DarkRadiantCore.vcxproj.filters @@ -1153,7 +1153,10 @@ src\decl - + + src\decl + + src\decl @@ -2379,7 +2382,16 @@ src\decl - + + src\decl + + + src\decl + + + src\decl + + src\decl