#5977: Some restructuring to make the parsing logic reusable outside …

…the ThreadedDeclParser.

radiantcore/CMakeLists.txt

@@ -23,7 +23,8 @@ add_library(radiantcore MODULE
             clipper/ClipPoint.cpp
             clipper/SplitAlgorithm.cpp
             commandsystem/CommandSystem.cpp
-            decl/DeclarationParser.cpp
+            decl/DeclarationFileParser.cpp
+            decl/DeclarationFolderParser.cpp
             decl/DeclarationManager.cpp
             decl/FavouritesManager.cpp
             eclass/EntityClass.cpp

radiantcore/decl/DeclarationFile.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <string>
+#include "idecltypes.h"
+
+namespace decl
+{
+
+class DeclarationFile
+{
+public:
+    // mod-relative path
+    std::string fullPath;
+
+    // The default type of this file, used for all
+    // declarations without explicitly declared type
+    decl::Type defaultDeclType;
+
+    bool operator< (const DeclarationFile& other) const
+    {
+        if (defaultDeclType < other.defaultDeclType)
+        {
+            return true;
+        }
+
+        if (defaultDeclType == other.defaultDeclType)
+        {
+            return fullPath < other.fullPath;
+        }
+
+        return false;
+    }
+};
+
+}

radiantcore/decl/DeclarationParser.cpp → radiantcore/decl/DeclarationFileParser.cpp

@@ -1,41 +1,62 @@
-#include "DeclarationParser.h"
+#include "DeclarationFileParser.h"
 
 #include "DeclarationManager.h"
+#include "parser/DefBlockTokeniser.h"
 
 namespace decl
 {
 
-inline DeclarationBlockSyntax createBlock(const parser::BlockTokeniser::Block& block, 
-    const vfs::FileInfo& fileInfo, const std::string& modName)
+namespace
 {
-    auto spacePos = block.name.find(' ');
+    inline DeclarationBlockSyntax createBlock(const parser::BlockTokeniser::Block& block,
+        const vfs::FileInfo& fileInfo, const std::string& modName)
+    {
+        auto spacePos = block.name.find(' ');
 
-    DeclarationBlockSyntax syntax;
+        DeclarationBlockSyntax syntax;
 
-    syntax.typeName = spacePos != std::string::npos ? block.name.substr(0, spacePos) : std::string();
-    syntax.name = spacePos != std::string::npos ? block.name.substr(spacePos + 1) : block.name;
-    syntax.contents = block.contents;
-    syntax.modName = modName;
-    syntax.fileInfo = fileInfo;
+        syntax.typeName = spacePos != std::string::npos ? block.name.substr(0, spacePos) : std::string();
+        syntax.name = spacePos != std::string::npos ? block.name.substr(spacePos + 1) : block.name;
+        syntax.contents = block.contents;
+        syntax.modName = modName;
+        syntax.fileInfo = fileInfo;
 
-    return syntax;
+        return syntax;
+    }
 }
 
-DeclarationParser::DeclarationParser(DeclarationManager& owner, Type declType, 
-    const std::string& baseDir, const std::string& extension,
+DeclarationFileParser::DeclarationFileParser(Type declType,
     const std::map<std::string, IDeclarationParser::Ptr>& parsersByTypename) :
-    ThreadedDeclParser<void>(declType, baseDir, extension, 1),
-    _owner(owner),
     _defaultDeclType(declType),
     _parsersByTypename(parsersByTypename)
 {
     _defaultTypeParser = getParserByType(declType);
 
-    if (!_defaultTypeParser) throw std::invalid_argument("No parser has been associated to the default type " + getTypeName(declType));
+    if (!_defaultTypeParser)
+    {
+        throw std::invalid_argument("No parser has been associated to the default type " + getTypeName(declType));
+    }
+}
+
+std::map<Type, NamedDeclarations>& DeclarationFileParser::getParsedDecls()
+{
+    return _parsedDecls;
+}
+
+const std::set<DeclarationFile>& DeclarationFileParser::getParsedFiles() const
+{
+    return _parsedFiles;
+}
+
+const std::vector<DeclarationBlockSyntax>& DeclarationFileParser::getUnrecognisedBlocks() const
+{
+    return _unrecognisedBlocks;
 }
 
-void DeclarationParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir)
+void DeclarationFileParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir)
 {
+    _parsedFiles.emplace(DeclarationFile{ fileInfo.fullPath(), _defaultDeclType });
+
     // Cut the incoming stream into declaration blocks
     parser::BasicDefBlockTokeniser<std::istream> tokeniser(stream);
 
@@ -69,23 +90,17 @@ void DeclarationParser::parse(std::istream& stream, const vfs::FileInfo& fileInf
     }
 }
 
-void DeclarationParser::onFinishParsing()
-{
-    // Submit all parsed declarations to the decl manager
-    _owner.onParserFinished(_defaultDeclType, std::move(_parsedDecls), std::move(_unrecognisedBlocks));
-}
-
-void DeclarationParser::parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& block)
+void DeclarationFileParser::parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& block)
 {
-    auto declaration =  parser.parseFromBlock(block);
+    auto declaration = parser.parseFromBlock(block);
 
     auto& declMap = _parsedDecls.try_emplace(parser.getDeclType(), NamedDeclarations()).first->second;
 
     // Insert into map, emit a warning on duplicates
     DeclarationManager::InsertDeclaration(declMap, std::move(declaration));
 }
 
-IDeclarationParser::Ptr DeclarationParser::getParserByType(Type declType) const
+IDeclarationParser::Ptr DeclarationFileParser::getParserByType(Type declType) const
 {
     // Get the default type parser
     for (const auto& pair : _parsersByTypename)

radiantcore/decl/DeclarationParser.h → radiantcore/decl/DeclarationFileParser.h

@@ -1,41 +1,38 @@
 #pragma once
 
 #include "ideclmanager.h"
-
-#include "parser/DefBlockTokeniser.h"
-#include "parser/ThreadedDeclParser.h"
+#include "DeclarationFile.h"
 
 namespace decl
 {
 
 class DeclarationManager;
 
-class DeclarationParser :
-    public parser::ThreadedDeclParser<void>
+class DeclarationFileParser
 {
 private:
-    DeclarationManager& _owner;
     Type _defaultDeclType;
 
     std::map<std::string, IDeclarationParser::Ptr> _parsersByTypename;
     IDeclarationParser::Ptr _defaultTypeParser;
 
+    std::set<DeclarationFile> _parsedFiles;
     std::map<Type, NamedDeclarations> _parsedDecls;
 
     std::vector<DeclarationBlockSyntax> _unrecognisedBlocks;
 
 public:
-    DeclarationParser(DeclarationManager& owner, Type declType, 
-        const std::string& baseDir, const std::string& extension,
-        const std::map<std::string, IDeclarationParser::Ptr>& parsersByTypename);
+    DeclarationFileParser(Type declType, const std::map<std::string, IDeclarationParser::Ptr>& parsersByTypename);
+
+    void parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir);
 
-protected:
-    void parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir) override;
-    void onFinishParsing() override;
+    std::map<Type, NamedDeclarations>& getParsedDecls();
+    const std::set<DeclarationFile>& getParsedFiles() const;
+    const std::vector<DeclarationBlockSyntax>& getUnrecognisedBlocks() const;
 
 private:
+    void parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& block);
     IDeclarationParser::Ptr getParserByType(Type declType) const;
-    void parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& blockSyntax);
 };
 
 }

radiantcore/decl/DeclarationFolderParser.cpp

@@ -0,0 +1,28 @@
+#include "DeclarationFolderParser.h"
+
+#include "DeclarationManager.h"
+
+namespace decl
+{
+
+DeclarationFolderParser::DeclarationFolderParser(DeclarationManager& owner, Type declType, 
+    const std::string& baseDir, const std::string& extension,
+    const std::map<std::string, IDeclarationParser::Ptr>& parsersByTypename) :
+    ThreadedDeclParser<void>(declType, baseDir, extension, 1),
+    _owner(owner),
+    _fileParser(declType, parsersByTypename),
+    _defaultDeclType(declType)
+{}
+
+void DeclarationFolderParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir)
+{
+    _fileParser.parse(stream, fileInfo, modDir);
+}
+
+void DeclarationFolderParser::onFinishParsing()
+{
+    // Submit all parsed declarations to the decl manager
+    _owner.onParserFinished(_defaultDeclType, _fileParser.getParsedDecls(), _fileParser.getUnrecognisedBlocks(), _fileParser.getParsedFiles());
+}
+
+}

radiantcore/decl/DeclarationFolderParser.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "DeclarationFile.h"
+#include "DeclarationFileParser.h"
+#include "ideclmanager.h"
+
+#include "parser/ThreadedDeclParser.h"
+
+namespace decl
+{
+
+class DeclarationManager;
+
+// Threaded parser processing all files in the configured decl folder
+// Submits all parsed declarations to the IDeclarationManager when finished
+class DeclarationFolderParser :
+    public parser::ThreadedDeclParser<void>
+{
+private:
+    DeclarationManager& _owner;
+
+    DeclarationFileParser _fileParser;
+    Type _defaultDeclType;
+
+public:
+    DeclarationFolderParser(DeclarationManager& owner, Type declType,
+        const std::string& baseDir, const std::string& extension,
+        const std::map<std::string, IDeclarationParser::Ptr>& parsersByTypename);
+
+protected:
+    void parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir) override;
+    void onFinishParsing() override;
+};
+
+}

radiantcore/decl/DeclarationManager.cpp

@@ -2,16 +2,18 @@
 
 #include "DeclarationManager.h"
 
+#include "DeclarationFolderParser.h"
 #include "ifilesystem.h"
 #include "module/StaticModule.h"
 #include "string/trim.h"
-#include "DeclarationParser.h"
 
 namespace decl
 {
 
 void DeclarationManager::registerDeclType(const std::string& typeName, const IDeclarationParser::Ptr& parser)
 {
+    std::lock_guard<std::mutex> parserLock(_parserLock);
+
     if (_parsersByTypename.count(typeName) > 0)
     {
         throw std::logic_error("Type name " + typeName + " has already been registered");
@@ -25,6 +27,8 @@ void DeclarationManager::registerDeclType(const std::string& typeName, const IDe
 
 void DeclarationManager::unregisterDeclType(const std::string& typeName)
 {
+    std::lock_guard<std::mutex> parserLock(_parserLock);
+
     auto existing = _parsersByTypename.find(typeName);
 
     if (existing == _parsersByTypename.end())
@@ -47,7 +51,7 @@ void DeclarationManager::registerDeclFolder(Type defaultType, const std::string&
     auto& decls = _declarationsByType.try_emplace(defaultType, Declarations()).first->second;
 
     // Start the parser thread
-    decls.parser = std::make_unique<DeclarationParser>(*this, defaultType, vfsPath, extension, _parsersByTypename);
+    decls.parser = std::make_unique<DeclarationFolderParser>(*this, defaultType, vfsPath, extension, _parsersByTypename);
     decls.parser->start();
 }
 
@@ -105,16 +109,34 @@ void DeclarationManager::doWithDeclarations(Type type, const std::function<void(
 
 void DeclarationManager::reloadDecarations()
 {
-    // TODO
+    std::lock_guard<std::mutex> fileLock(_parsedFileLock);
+    std::lock_guard<std::mutex> parserLock(_parserLock);
+
+    for (const auto& [type, files] : _parsedFilesByDefaultType)
+    {
+        DeclarationFileParser parser(type, _parsersByTypename);
+
+        for (const auto& file : files)
+        {
+            auto vfsFile = GlobalFileSystem().openTextFile(file.fullPath);
+
+            if (!vfsFile) continue;
+
+            // TODO
+        }
+
+        // Submit the changes
+        // TODO
+    }
 }
 
 sigc::signal<void>& DeclarationManager::signal_DeclsReloaded(Type type)
 {
     return _declsReloadedSignals.try_emplace(type).first->second;
 }
 
-void DeclarationManager::onParserFinished(Type parserType, std::map<Type, NamedDeclarations>&& parsedDecls,
-    std::vector<DeclarationBlockSyntax>&& unrecognisedBlocks)
+void DeclarationManager::onParserFinished(Type parserType, std::map<Type, NamedDeclarations>& parsedDecls,
+    const std::vector<DeclarationBlockSyntax>& unrecognisedBlocks, const std::set<DeclarationFile>& parsedFiles)
 {
     {
         std::lock_guard<std::mutex> declLock(_declarationLock);
@@ -139,6 +161,18 @@ void DeclarationManager::onParserFinished(Type parserType, std::map<Type, NamedD
             std::make_move_iterator(unrecognisedBlocks.begin()), std::make_move_iterator(unrecognisedBlocks.end()));
     }
 
+    {
+        std::lock_guard<std::mutex> lock(_parsedFileLock);
+
+        // Merge all parsed files into the main set
+        for (const auto& parsedFile : parsedFiles)
+        {
+            auto& fileSet = _parsedFilesByDefaultType.try_emplace(parsedFile.defaultDeclType, std::set<DeclarationFile>()).first->second;
+
+            fileSet.emplace(parsedFile);
+        }
+    }
+
     // We might have received a parser in the meantime
     handleUnrecognisedBlocks();
 
@@ -210,7 +244,7 @@ void DeclarationManager::initialiseModule(const IApplicationContext& ctx)
 void DeclarationManager::shutdownModule()
 {
     auto declLock = std::make_unique<std::lock_guard<std::mutex>>(_declarationLock);
-    std::vector<std::unique_ptr<DeclarationParser>> parsersToFinish;
+    std::vector<std::unique_ptr<DeclarationFolderParser>> parsersToFinish;
 
     for (auto& [_, decl] : _declarationsByType)
     {
@@ -226,6 +260,7 @@ void DeclarationManager::shutdownModule()
 
     // All parsers have finished, clear the structure, no need to lock anything
     _registeredFolders.clear();
+    _parsedFilesByDefaultType.clear();
     _unrecognisedBlocks.clear();
     _declarationsByType.clear();
     _parsersByTypename.clear();