#5977: First iteration supporting parsing of known declarations.

include/ideclmanager.h

@@ -37,6 +37,8 @@ class IDeclaration
     virtual Type getType() const = 0;
 };
 
+using NamedDeclarations = std::map<std::string, IDeclaration::Ptr>;
+
 // Interface of a parser that can handle a single declaration type
 class IDeclarationParser
 {
@@ -71,13 +73,17 @@ class IDeclarationManager :
     // Unregisters the given typename and the associated parser
     virtual void unregisterDeclType(const std::string& typeName) = 0;
 
-    // Associates the given VFS folder path to a certain declaration type
-    // all files matching the given file extension will be searched and parsed.
+    // Associates the given VFS folder (with trailing slash) to a certain declaration type
+    // all files matching the given file extension (without dot) will be searched and parsed.
     // The folder will not be recursively searched for files, only immediate children will be parsed.
     // Any untyped declaration blocks found in the files will be assumed to be of the given defaultType.
     // All explicitly typed resources will be processed using the parser that has been previously
     // associated in registerDeclType()
+    // Registering a folder will immediately trigger parsing of all contained files matching the criteria.
     virtual void registerDeclFolder(Type defaultType, const std::string& vfsFolder, const std::string& extension) = 0;
+
+    // Iterate over all known declarations, using the given visitor
+    virtual void foreachDeclaration(Type type, const std::function<void(const IDeclaration&)>& functor) = 0;
 };
 
 }

include/idecltypes.h

@@ -10,13 +10,14 @@ namespace decl
 // Enumeration of declaration types supported by DarkRadiant
 enum class Type
 {
-    None,
-    Material,
+    None = -1,
+    Material = 0,
     EntityDef,
     SoundShader,
     Model,
     Particle,
     Skin,
+    NumDeclarationTypes,
 };
 
 inline std::string getTypeName(Type type)

radiantcore/decl/DeclarationManager.cpp

@@ -1,63 +1,123 @@
-#include "ideclmanager.h"
+#include <future>
 
-#include <map>
+#include "DeclarationManager.h"
+
+#include "ifilesystem.h"
 #include "module/StaticModule.h"
+#include "string/trim.h"
+#include "DeclarationParser.h"
 
 namespace decl
 {
 
-class DeclarationManager :
-    public IDeclarationManager
+void DeclarationManager::registerDeclType(const std::string& typeName, const IDeclarationParser::Ptr& parser)
 {
-private:
-    std::map<std::string, IDeclarationParser::Ptr> _parsersByTypename;
-
-public:
-    void registerDeclType(const std::string& typeName, const IDeclarationParser::Ptr& parser) override
+    if (_parsersByTypename.count(typeName) > 0)
     {
-        if (_parsersByTypename.count(typeName) > 0)
-        {
-            throw std::logic_error("Type name " + typeName + " has already been registered");
-        }
-
-        _parsersByTypename.emplace(typeName, parser);
+        throw std::logic_error("Type name " + typeName + " has already been registered");
     }
 
-    void unregisterDeclType(const std::string& typeName) override
-    {
-        auto existing = _parsersByTypename.find(typeName);
+    _parsersByTypename.emplace(typeName, parser);
+}
 
-        if (existing == _parsersByTypename.end())
-        {
-            throw std::logic_error("Type name " + typeName + " has not been registered");
-        }
+void DeclarationManager::unregisterDeclType(const std::string& typeName)
+{
+    auto existing = _parsersByTypename.find(typeName);
 
-        _parsersByTypename.erase(existing);
+    if (existing == _parsersByTypename.end())
+    {
+        throw std::logic_error("Type name " + typeName + " has not been registered");
     }
 
-    void registerDeclFolder(Type defaultType, const std::string& vfsFolder, const std::string& extension) override
+    _parsersByTypename.erase(existing);
+}
+
+void DeclarationManager::registerDeclFolder(Type defaultType, const std::string& inputFolder, const std::string& inputExtension)
+{
+    // Sanitise input strings
+    auto vfsPath = os::standardPathWithSlash(inputFolder);
+    auto extension = string::trim_left_copy(inputExtension, ".");
+
+    _registeredFolders.emplace_back(RegisteredFolder{ vfsPath, extension, defaultType });
+
+    std::lock_guard<std::mutex> declLock(_declarationLock);
+    auto& decls = _declarationsByType.try_emplace(defaultType, Declarations()).first->second;
+
+    // Start the parser thread
+    decls.parser = std::make_unique<DeclarationParser>(*this, defaultType, vfsPath, extension, _parsersByTypename);
+    decls.parser->start();
+}
+
+void DeclarationManager::foreachDeclaration(Type type, const std::function<void(const IDeclaration&)>& functor)
+{
+    auto declLock = std::make_unique<std::lock_guard<std::mutex>>(_declarationLock);
+
+    auto decls = _declarationsByType.find(type);
+
+    if (decls == _declarationsByType.end()) return;
+
+    // Ensure the parser is done
+    if (decls->second.parser)
     {
+        // Release the lock to give the thread a chance to finish
+        declLock.reset();
+
+        decls->second.parser->ensureFinished(); // blocks
+        decls->second.parser.reset();
 
+        declLock = std::make_unique<std::lock_guard<std::mutex>>(_declarationLock);
     }
 
-    const std::string& getName() const override
+    for (const auto& [_, decl] : decls->second.decls)
     {
-        static std::string _name(MODULE_DECLMANAGER);
-        return _name;
+        functor(*decl);
     }
+}
 
-    const StringSet& getDependencies() const override
+void DeclarationManager::onParserFinished(std::map<Type, NamedDeclarations>&& parsedDecls,
+    std::vector<DeclarationBlockSyntax>&& unrecognisedBlocks)
+{
     {
-        static StringSet _dependencies{};
+        std::lock_guard<std::mutex> declLock(_declarationLock);
 
-        return _dependencies;
+        // Coming back from a parser thread, sort the parsed decls into the main dictionary
+        for (auto& pair : parsedDecls)
+        {
+            auto& map = _declarationsByType.try_emplace(pair.first, Declarations()).first->second.decls;
+
+            map.merge(pair.second);
+        }
     }
 
-    void initialiseModule(const IApplicationContext& ctx) override
     {
-        rMessage() << getName() << "::initialiseModule called." << std::endl;
+        std::lock_guard<std::mutex> lock(_unrecognisedBlockLock);
+
+        // Move all blocks into this one
+        _unrecognisedBlocks.insert(_unrecognisedBlocks.end(),
+            std::make_move_iterator(unrecognisedBlocks.begin()), std::make_move_iterator(unrecognisedBlocks.end()));
     }
-};
+}
+
+const std::string& DeclarationManager::getName() const
+{
+    static std::string _name(MODULE_DECLMANAGER);
+    return _name;
+}
+
+const StringSet& DeclarationManager::getDependencies() const
+{
+    static StringSet _dependencies
+    {
+        MODULE_VIRTUALFILESYSTEM
+    };
+
+    return _dependencies;
+}
+
+void DeclarationManager::initialiseModule(const IApplicationContext& ctx)
+{
+    rMessage() << getName() << "::initialiseModule called." << std::endl;
+}
 
 module::StaticModuleRegistration<DeclarationManager> _declManagerModule;
 

radiantcore/decl/DeclarationManager.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "ideclmanager.h"
+#include <map>
+#include <vector>
+#include <memory>
+#include <thread>
+
+namespace decl
+{
+
+class DeclarationParser;
+
+class DeclarationManager :
+    public IDeclarationManager
+{
+private:
+    std::map<std::string, IDeclarationParser::Ptr> _parsersByTypename;
+
+    struct RegisteredFolder
+    {
+        std::string folder;
+        std::string extension;
+        Type defaultType;
+    };
+
+    std::vector<RegisteredFolder> _registeredFolders;
+
+    struct Declarations
+    {
+        // The decl library
+        NamedDeclarations decls;
+
+        // If not empty, holds the running parser
+        std::unique_ptr<DeclarationParser> parser;
+    };
+
+    // One entry for each decl
+    std::map<Type, Declarations> _declarationsByType;
+    std::mutex _declarationLock;
+
+    std::vector<DeclarationBlockSyntax> _unrecognisedBlocks;
+    std::mutex _unrecognisedBlockLock;
+
+public:
+    void registerDeclType(const std::string& typeName, const IDeclarationParser::Ptr& parser) override;
+    void unregisterDeclType(const std::string& typeName) override;
+    void registerDeclFolder(Type defaultType, const std::string& inputFolder, const std::string& inputExtension) override;
+    void foreachDeclaration(Type type, const std::function<void(const IDeclaration&)>& functor) override;
+
+    const std::string& getName() const override;
+    const StringSet& getDependencies() const override;
+    void initialiseModule(const IApplicationContext& ctx) override;
+
+    // Invoked once a parser thread has finished. It will move its data over to here.
+    void onParserFinished(std::map<Type, NamedDeclarations>&& parsedDecls, 
+        std::vector<DeclarationBlockSyntax>&& unrecognisedBlocks);
+};
+
+}

radiantcore/decl/DeclarationParser.cpp

@@ -0,0 +1,103 @@
+#include "DeclarationParser.h"
+
+#include "DeclarationManager.h"
+
+namespace decl
+{
+
+inline DeclarationBlockSyntax createBlock(const parser::BlockTokeniser::Block& block)
+{
+    auto spacePos = block.name.find(' ');
+
+    return DeclarationBlockSyntax
+    {
+        spacePos != std::string::npos ? block.name.substr(0, spacePos) : std::string(), // type
+        spacePos != std::string::npos ? block.name.substr(spacePos + 1) : block.name, // name
+        block.contents
+    };
+}
+
+DeclarationParser::DeclarationParser(DeclarationManager& owner, Type declType, 
+    const std::string& baseDir, const std::string& extension,
+    const std::map<std::string, IDeclarationParser::Ptr>& parsersByTypename) :
+    ThreadedDeclParser<void>(declType, baseDir, extension, 1),
+    _owner(owner),
+    _parsersByTypename(parsersByTypename)
+{
+    _defaultTypeParser = getParserByType(declType);
+
+    if (!_defaultTypeParser) throw std::invalid_argument("No parser has been associated to the default type " + getTypeName(declType));
+}
+
+void DeclarationParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir)
+{
+    // Cut the incoming stream into declaration blocks
+    parser::BasicDefBlockTokeniser<std::istream> tokeniser(stream);
+
+    while (tokeniser.hasMoreBlocks())
+    {
+        auto block = tokeniser.nextBlock();
+
+        // Convert the incoming block to a DeclarationBlockSyntax
+        auto blockSyntax = createBlock(block);
+
+        auto spacePos = block.name.find(' ');
+
+        if (spacePos == std::string::npos)
+        {
+            // No type specified, use the default type parser
+            parseBlock(*_defaultTypeParser, blockSyntax);
+            continue;
+        }
+
+        // Locate a parser capable of handling that block
+        auto parser = _parsersByTypename.find(block.name.substr(0, spacePos));
+
+        if (parser != _parsersByTypename.end())
+        {
+            parseBlock(*parser->second, blockSyntax);
+            continue;
+        }
+
+        // Unknown block type, move to buffer
+        _unrecognisedBlocks.emplace_back(std::move(blockSyntax));
+    }
+}
+
+void DeclarationParser::onFinishParsing()
+{
+    // Submit all parsed declarations to the decl manager
+    _owner.onParserFinished(std::move(_parsedDecls), std::move(_unrecognisedBlocks));
+}
+
+void DeclarationParser::parseBlock(IDeclarationParser& parser, const DeclarationBlockSyntax& block)
+{
+    auto declaration =  parser.parseFromBlock(block);
+
+    auto& declMap = _parsedDecls.try_emplace(parser.getDeclType(), NamedDeclarations()).first->second;
+
+    // Insert into map, emit a warning on duplicates
+    auto result = declMap.try_emplace(declaration->getName(), std::move(declaration));
+
+    if (!result.second)
+    {
+        rWarning() << "[DeclParser]: " << getTypeName(parser.getDeclType()) << " " << 
+            result.first->second->getName() << " has already been declared" << std::endl;
+    }
+}
+
+IDeclarationParser::Ptr DeclarationParser::getParserByType(Type declType) const
+{
+    // Get the default type parser
+    for (const auto& pair : _parsersByTypename)
+    {
+        if (pair.second->getDeclType() == declType)
+        {
+            return pair.second;
+        }
+    }
+
+    return {};
+}
+
+}