Skip to content

Commit

Permalink
#5977: First working version of reloadDecls
Browse files Browse the repository at this point in the history
  • Loading branch information
@codereader
codereader committed Jun 26, 2022
1 parent 73d4171 commit 9eff828
Show file tree
Hide file tree
Showing 7 changed files with 131 additions and 35 deletions.
3 changes: 3 additions & 0 deletions radiantcore/decl/DeclarationFile.h
View file
Expand Up @@ -16,6 +16,9 @@ class DeclarationFile
// declarations without explicitly declared type
decl::Type defaultDeclType;

// All declarations (type+name) declared in this file
std::vector<std::pair<Type, std::string>> declarations;

bool operator< (const DeclarationFile& other) const
{
if (defaultDeclType < other.defaultDeclType)
Expand Down
14 changes: 7 additions & 7 deletions radiantcore/decl/DeclarationFileParser.cpp
View file
Expand Up @@ -45,15 +45,11 @@ std::map<Type, std::vector<DeclarationBlockSyntax>>& DeclarationFileParser::getP
return _parsedBlocks;
}

const std::set<DeclarationFile>& DeclarationFileParser::getParsedFiles() const
{
return _parsedFiles;
}

void DeclarationFileParser::parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir)
{
_parsedFiles.emplace(DeclarationFile{ fileInfo.fullPath(), _defaultDeclType });

#if 0
auto& declFile = _parsedFiles.emplace_back(DeclarationFile{ fileInfo.fullPath(), _defaultDeclType });
#endif
// Cut the incoming stream into declaration blocks
parser::BasicDefBlockTokeniser<std::istream> tokeniser(stream);

Expand All @@ -68,6 +64,10 @@ void DeclarationFileParser::parse(std::istream& stream, const vfs::FileInfo& fil
auto declType = determineBlockType(blockSyntax);
auto& blockList = _parsedBlocks.try_emplace(declType).first->second;
blockList.emplace_back(std::move(blockSyntax));
#if 0
// Add this declaration to the parsed file
//declFile.declarations.emplace_back(declType, block.name);
#endif
}
}

Expand Down
8 changes: 6 additions & 2 deletions radiantcore/decl/DeclarationFileParser.h
View file
Expand Up @@ -15,7 +15,9 @@ class DeclarationFileParser

std::map<std::string, Type> _typeMapping;

std::set<DeclarationFile> _parsedFiles;
#if 0
std::map<std::string, DeclarationFile> _parsedFiles;
#endif
std::map<Type, std::vector<DeclarationBlockSyntax>> _parsedBlocks;

public:
Expand All @@ -24,7 +26,9 @@ class DeclarationFileParser
void parse(std::istream& stream, const vfs::FileInfo& fileInfo, const std::string& modDir);

std::map<Type, std::vector<DeclarationBlockSyntax>>& getParsedBlocks();
const std::set<DeclarationFile>& getParsedFiles() const;
#if 0
const std::map<std::string, DeclarationFile>& getParsedFiles() const;
#endif

private:
Type determineBlockType(const DeclarationBlockSyntax& block);
Expand Down
2 changes: 1 addition & 1 deletion radiantcore/decl/DeclarationFolderParser.cpp
View file
Expand Up @@ -22,7 +22,7 @@ void DeclarationFolderParser::parse(std::istream& stream, const vfs::FileInfo& f
void DeclarationFolderParser::onFinishParsing()
{
// Submit all parsed declarations to the decl manager
_owner.onParserFinished(_defaultDeclType, _fileParser.getParsedBlocks(), _fileParser.getParsedFiles());
_owner.onParserFinished(_defaultDeclType, _fileParser.getParsedBlocks());
}

}
125 changes: 104 additions & 21 deletions radiantcore/decl/DeclarationManager.cpp
View file
Expand Up @@ -6,14 +6,15 @@
#include "ifilesystem.h"
#include "module/StaticModule.h"
#include "string/trim.h"
#include "util/ScopedBoolLock.h"

namespace decl
{

void DeclarationManager::registerDeclType(const std::string& typeName, const IDeclarationCreator::Ptr& creator)
{
{
std::lock_guard<std::recursive_mutex> creatorLock(_creatorLock);
std::lock_guard creatorLock(_creatorLock);

if (_creatorsByTypename.count(typeName) > 0 || _creatorsByType.count(creator->getDeclType()) > 0)
{
Expand All @@ -31,7 +32,7 @@ void DeclarationManager::registerDeclType(const std::string& typeName, const IDe

void DeclarationManager::unregisterDeclType(const std::string& typeName)
{
std::lock_guard<std::recursive_mutex> parserLock(_creatorLock);
std::lock_guard parserLock(_creatorLock);

auto existing = _creatorsByTypename.find(typeName);

Expand All @@ -49,9 +50,10 @@ void DeclarationManager::registerDeclFolder(Type defaultType, const std::string&
auto vfsPath = os::standardPathWithSlash(inputFolder);
auto extension = string::trim_left_copy(inputExtension, ".");

std::lock_guard folderLock(_registeredFoldersLock);
_registeredFolders.emplace_back(RegisteredFolder{ vfsPath, extension, defaultType });

std::lock_guard<std::recursive_mutex> declLock(_declarationLock);
std::lock_guard declLock(_declarationLock);
auto& decls = _declarationsByType.try_emplace(defaultType, Declarations()).first->second;

// Start the parser thread
Expand All @@ -63,7 +65,7 @@ std::map<std::string, Type> DeclarationManager::getTypenameMapping()
{
std::map<std::string, Type> result;

std::lock_guard<std::recursive_mutex> creatorLock(_creatorLock);
std::lock_guard creatorLock(_creatorLock);

for (const auto& [name, creator] : _creatorsByTypename)
{
Expand Down Expand Up @@ -127,14 +129,52 @@ void DeclarationManager::doWithDeclarations(Type type, const std::function<void(

void DeclarationManager::reloadDecarations()
{
std::lock_guard<std::recursive_mutex> fileLock(_parsedFileLock);
std::lock_guard<std::recursive_mutex> parserLock(_creatorLock);
if (_reparseInProgress) return;

for (const auto& [type, files] : _parsedFilesByDefaultType)
util::ScopedBoolLock reparseLock(_reparseInProgress);

_parseStamp++;

// Remove all unrecognised blocks from previous runs
{
std::lock_guard lock(_unrecognisedBlockLock);
_unrecognisedBlocks.clear();
}

runParsersForAllFolders();

// Empty all declarations that haven't been touched during this reparse run
{
std::lock_guard declLock(_declarationLock);

for (const auto& [_, namedDecls] : _declarationsByType)
{
for (const auto& [name, decl] : namedDecls.decls)
{
if (decl->getParseStamp() < _parseStamp)
{
rMessage() << "Declaration no longer present after reloadDecls: " << name << std::endl;

auto syntax = decl->getBlockSyntax();

// Clear name and file info
syntax.contents.clear();
syntax.fileInfo = vfs::FileInfo();

decl->setBlockSyntax(syntax);
}
}
}
}
#if 0
std::lock_guard fileLock(_parsedFileLock);
std::lock_guard parserLock(_creatorLock);

for (auto& [type, files] : _parsedFilesByDefaultType)
{
DeclarationFileParser parser(type, getTypenameMapping());

for (const auto& file : files)
for (auto& file : files)
{
auto vfsFile = GlobalFileSystem().openTextFile(file.fullPath);

Expand All @@ -153,6 +193,11 @@ void DeclarationManager::reloadDecarations()
rError() << "[DeclParser] Failed to parse " << fileInfo.fullPath()
<< " (" << e.what() << ")" << std::endl;
}

const auto& newlyParsedFile = parser.getParsedFiles().find(file.fullPath);

// Invalidate all declarations that have been removed from the file
handleMissingDecls(file, newlyParsedFile);
}

// Submit the changes
Expand All @@ -161,11 +206,37 @@ void DeclarationManager::reloadDecarations()

// Process the list of unrecognised blocks (from this and any previous run)
handleUnrecognisedBlocks();
#endif

std::lock_guard folderLock(_registeredFoldersLock);

// Invoke the declsReloaded signal for all types
for (const auto& [type, files] : _parsedFilesByDefaultType)
for (const auto& folder : _registeredFolders)
{
signal_DeclsReloaded(folder.defaultType).emit();
}
}

void DeclarationManager::runParsersForAllFolders()
{
std::vector<std::unique_ptr<DeclarationFolderParser>> parsers;

std::lock_guard folderLock(_registeredFoldersLock);

// Start a parser for each known folder
for (const auto& folder : _registeredFolders)
{
auto& parser = parsers.emplace_back(
std::make_unique<DeclarationFolderParser>(*this, folder.defaultType, folder.folder, folder.extension, getTypenameMapping())
);
parser->start();
}

// Wait for all parsers to complete
while (!parsers.empty())
{
signal_DeclsReloaded(type).emit();
parsers.back()->ensureFinished();
parsers.pop_back();
}
}

Expand All @@ -175,21 +246,23 @@ sigc::signal<void>& DeclarationManager::signal_DeclsReloaded(Type type)
}

void DeclarationManager::onParserFinished(Type parserType,
const std::map<Type, std::vector<DeclarationBlockSyntax>>& parsedBlocks,
const std::set<DeclarationFile>& parsedFiles)
const std::map<Type, std::vector<DeclarationBlockSyntax>>& parsedBlocks)
{
#if 0
// Sort all parsed files into the large list
{
std::lock_guard<std::recursive_mutex> lock(_parsedFileLock);
std::lock_guard lock(_parsedFileLock);

// Merge all parsed files into the main set
for (const auto& parsedFile : parsedFiles)
for (const auto& [_, parsedFile] : parsedFiles)
{
auto& fileSet = _parsedFilesByDefaultType.try_emplace(parsedFile.defaultDeclType, std::set<DeclarationFile>()).first->second;
auto& fileSet = _parsedFilesByDefaultType.try_emplace(
parsedFile.defaultDeclType, std::vector<DeclarationFile>()).first->second;

fileSet.emplace(parsedFile);
}
}
#endif

// Sort all parsed blocks into our main dictionary
// unrecognised blocks will be pushed to _unrecognisedBlocks
Expand All @@ -204,8 +277,8 @@ void DeclarationManager::onParserFinished(Type parserType,

void DeclarationManager::processParsedBlocks(const std::map<Type, std::vector<DeclarationBlockSyntax>>& parsedBlocks)
{
std::lock_guard<std::recursive_mutex> declLock(_declarationLock);
std::lock_guard<std::recursive_mutex> creatorLock(_creatorLock);
std::lock_guard declLock(_declarationLock);
std::lock_guard creatorLock(_creatorLock);

// Coming back from a parser thread, sort the parsed decls into the main dictionary
for (auto& pair : parsedBlocks)
Expand All @@ -217,7 +290,7 @@ void DeclarationManager::processParsedBlocks(const std::map<Type, std::vector<De
if (type == Type::Undetermined)
{
// Block type unknown, put it on the pile, it will be processed later
std::lock_guard<std::recursive_mutex> lock(_unrecognisedBlockLock);
std::lock_guard lock(_unrecognisedBlockLock);
_unrecognisedBlocks.emplace_back(std::move(block));
continue;
}
Expand Down Expand Up @@ -262,18 +335,22 @@ void DeclarationManager::createOrUpdateDeclaration(Type type, const DeclarationB
auto creator = _creatorsByType.at(type);
existing = map.emplace(block.name, creator->createDeclaration(block.name)).first;
}
else
else if (existing->second->getParseStamp() == _parseStamp)
{
// TODO: Compare parse stamps to see if this decl has duplicates
rWarning() << "[DeclParser]: " << getTypeName(type) << " " <<
existing->second->getDeclName() << " has already been declared" << std::endl;
}

// Assign the block to the declaration instance
existing->second->setBlockSyntax(block);

// Update the parse stamp for this instance
existing->second->setParseStamp(_parseStamp);
}

void DeclarationManager::handleUnrecognisedBlocks()
{
std::lock_guard<std::recursive_mutex> lock(_unrecognisedBlockLock);
std::lock_guard lock(_unrecognisedBlockLock);

if (_unrecognisedBlocks.empty()) return;

Expand Down Expand Up @@ -323,6 +400,10 @@ const StringSet& DeclarationManager::getDependencies() const
void DeclarationManager::initialiseModule(const IApplicationContext& ctx)
{
rMessage() << getName() << "::initialiseModule called." << std::endl;

// After the initial parsing, all decls will have a parseStamp of 0
_parseStamp = 0;
_reparseInProgress = false;
}

void DeclarationManager::shutdownModule()
Expand All @@ -344,7 +425,9 @@ void DeclarationManager::shutdownModule()

// All parsers have finished, clear the structure, no need to lock anything
_registeredFolders.clear();
#if 0
_parsedFilesByDefaultType.clear();
#endif
_unrecognisedBlocks.clear();
_declarationsByType.clear();
_creatorsByTypename.clear();
Expand Down
12 changes: 9 additions & 3 deletions radiantcore/decl/DeclarationManager.h
View file
Expand Up @@ -29,9 +29,12 @@ class DeclarationManager :
};

std::vector<RegisteredFolder> _registeredFolders;
std::recursive_mutex _registeredFoldersLock;

std::map<decl::Type, std::set<DeclarationFile>> _parsedFilesByDefaultType;
#if 0
std::map<Type, std::set<DeclarationFile>> _parsedFilesByDefaultType;
std::recursive_mutex _parsedFileLock;
#endif

struct Declarations
{
Expand All @@ -51,6 +54,9 @@ class DeclarationManager :

std::map<Type, sigc::signal<void>> _declsReloadedSignals;

std::size_t _parseStamp;
bool _reparseInProgress;

public:
void registerDeclType(const std::string& typeName, const IDeclarationCreator::Ptr& parser) override;
void unregisterDeclType(const std::string& typeName) override;
Expand All @@ -67,12 +73,12 @@ class DeclarationManager :

// Invoked once a parser thread has finished. It will move its data over to here.
void onParserFinished(Type parserType,
const std::map<Type, std::vector<DeclarationBlockSyntax>>& parsedBlocks,
const std::set<DeclarationFile>& parsedFiles);
const std::map<Type, std::vector<DeclarationBlockSyntax>>& parsedBlocks);

static void InsertDeclaration(NamedDeclarations& map, IDeclaration::Ptr&& declaration);

private:
void runParsersForAllFolders();
// Attempts to resolve the block type of the given block, returns true on success, false otherwise.
// Stores the determined type in the given reference.
std::map<std::string, Type> getTypenameMapping();
Expand Down
2 changes: 1 addition & 1 deletion test/DeclManager.cpp
View file
Expand Up @@ -451,9 +451,9 @@ testdecl decl/temporary/13 { diffusemap textures/temporary/13 }
GlobalDeclarationManager().registerDeclType("testdecl", std::make_shared<TestDeclarationCreator>());
GlobalDeclarationManager().registerDeclFolder(decl::Type::Material, "testdecls", ".decl");

expectMaterialIsPresent(decl::Type::Material, "decl/temporary/11");
expectMaterialIsPresent(decl::Type::Material, "decl/temporary/12");
expectMaterialIsPresent(decl::Type::Material, "decl/temporary/13");
expectMaterialIsPresent(decl::Type::Material, "decl/temporary/14");

// Move a decl from the first file to the second
tempFile.setContents(R"(
Expand Down

0 comments on commit 9eff828

Please sign in to comment.