Skip to content

Multi thread processing #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Neko-Box-Coder merged 6 commits into from
Feb 8, 2025
Merged

Multi thread processing #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f26cdf8
Adding multi-thread support for compiling
Neko-Box-Coder Feb 5, 2025
d951cfe
Adding dependencies setup/build/cleanup to be multithreaded
Neko-Box-Coder Feb 7, 2025
8dbd2df
Exposing thread option from command line
Neko-Box-Coder Feb 7, 2025
256ff49
Fixing log level not being set for new threads
Neko-Box-Coder Feb 7, 2025
83c1f9f
Using fixed length deadline and adding timeout error message
Neko-Box-Coder Feb 8, 2025
04b67ba
Updating documentations
Neko-Box-Coder Feb 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ target_link_libraries(runcpp2 PRIVATE ssLogger System2 CppOverride dylib)
target_link_libraries(runcpp2 PUBLIC ghc_filesystem ryml::ryml mpark_variant)

if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# TODO: Try to change to /Wall
set(STANDARD_COMPILE_FLAGS "/utf-8;/W1;/DGHC_WIN_DISABLE_WSTRING_STORAGE_TYPE=1")
else()
set(STANDARD_COMPILE_FLAGS "-Wall"
Expand Down
6 changes: 4 additions & 2 deletions Include/runcpp2/CompilingLinking.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ namespace runcpp2
const Data::ScriptInfo& scriptInfo,
const std::vector<Data::DependencyInfo*>& availableDependencies,
const Data::Profile& profile,
bool buildExecutable);
bool buildExecutable,
const int maxThreads);

//TODO: Convert string paths to filesystem paths
bool CompileAndLinkScript( const ghc::filesystem::path& buildDir,
Expand All @@ -35,7 +36,8 @@ namespace runcpp2
const std::vector<Data::DependencyInfo*>& availableDependencies,
const Data::Profile& profile,
const std::vector<std::string>& compiledObjectsPaths,
bool buildExecutable);
bool buildExecutable,
const int maxThreads);
}

#endif
1 change: 1 addition & 0 deletions Include/runcpp2/Data/CmdOptions.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ namespace runcpp2
CONFIG_FILE,
CLEANUP,
BUILD_SOURCE_ONLY,
THREADS,
COUNT
};
}
Expand Down
6 changes: 4 additions & 2 deletions Include/runcpp2/DependenciesHelper.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,14 @@ namespace runcpp2
const Data::ScriptInfo& scriptInfo,
std::vector<Data::DependencyInfo*>& availableDependencies,
const std::vector<std::string>& dependenciesLocalCopiesPaths,
const std::vector<std::string>& dependenciesSourcePaths);
const std::vector<std::string>& dependenciesSourcePaths,
const int maxThreads);

bool BuildDependencies( const runcpp2::Data::Profile& profile,
const Data::ScriptInfo& scriptInfo,
const std::vector<Data::DependencyInfo*>& availableDependencies,
const std::vector<std::string>& dependenciesLocalCopiesPaths);
const std::vector<std::string>& dependenciesLocalCopiesPaths,
const int maxThreads);

bool GatherDependenciesBinaries(const std::vector<Data::DependencyInfo*>& availableDependencies,
const std::vector<std::string>& dependenciesCopiesPaths,
Expand Down
2 changes: 2 additions & 0 deletions Include/runcpp2/PipelineSteps.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ namespace runcpp2
const Data::Profile& profile,
const ghc::filesystem::path& absoluteScriptPath,
const Data::ScriptInfo* lastScriptInfo,
const int maxThreads,
bool& outRecompileNeeded,
bool& outRelinkNeeded,
std::vector<std::string>& outChangedDependencies);
Expand All @@ -77,6 +78,7 @@ namespace runcpp2
const ghc::filesystem::path& buildDir,
const std::unordered_map<CmdOptions, std::string>& currentOptions,
const std::vector<std::string>& changedDependencies,
const int maxThreads,
std::vector<Data::DependencyInfo*>& outAvailableDependencies,
std::vector<std::string>& outGatheredBinariesPaths);

Expand Down
2 changes: 1 addition & 1 deletion Include/runcpp2/runcpp2.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ namespace runcpp2
const std::unordered_map<CmdOptions, std::string> currentOptions,
const std::vector<std::string>& runArgs,
const Data::ScriptInfo* lastScriptInfo,
Data::ScriptInfo& outScriptInfo,
const std::string& buildOutputDir,
Data::ScriptInfo& outScriptInfo,
int& returnStatus);

std::string PipelineResultToString(PipelineResult result);
Expand Down
310 changes: 201 additions & 109 deletions Src/runcpp2/CompilingLinking.cpp
View file
Open in desktop

Large diffs are not rendered by default.

152 changes: 131 additions & 21 deletions Src/runcpp2/DependenciesHelper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include "ssLogger/ssLog.hpp"

#include <unordered_set>
#include <future>
#include <chrono>

namespace
{
Expand Down Expand Up @@ -489,7 +491,8 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
const Data::ScriptInfo& scriptInfo,
std::vector<Data::DependencyInfo*>& availableDependencies,
const std::vector<std::string>& dependenciesLocalCopiesPaths,
const std::vector<std::string>& dependenciesSourcePaths)
const std::vector<std::string>& dependenciesSourcePaths,
const int maxThreads)
{
ssLOG_FUNC_INFO();

Expand All @@ -510,9 +513,13 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
}

if(!PopulateAbsoluteIncludePaths(availableDependencies, dependenciesLocalCopiesPaths))
{
return false;
}

std::vector<std::future<bool>> actions;

//Cache logs for worker threads
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
int logLevel = ssLOG_GET_CURRENT_THREAD_TARGET_LEVEL();

//Run setup steps
for(int i = 0; i < availableDependencies.size(); ++i)
Expand All @@ -524,46 +531,150 @@ bool runcpp2::SetupDependenciesIfNeeded(const runcpp2::Data::Profile& profile,
continue;
}

ssLOG_INFO("Running setup commands for " << availableDependencies.at(i)->Name);
if(!RunDependenciesSteps( profile,
availableDependencies.at(i)->Setup,
dependenciesLocalCopiesPaths.at(i),
true))
actions.emplace_back
(
std::async
(
std::launch::async,
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths, logLevel]()
{
ssLOG_SET_CURRENT_THREAD_TARGET_LEVEL(logLevel);

ssLOG_INFO("Running setup commands for " << availableDependencies.at(i)->Name);
if(!RunDependenciesSteps( profile,
availableDependencies.at(i)->Setup,
dependenciesLocalCopiesPaths.at(i),
true))
{
ssLOG_ERROR("Failed to setup dependency " <<
availableDependencies.at(i)->Name);
return false;
}
return true;
}
)
);

//Evaluate the setup results for each batch
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
{
ssLOG_ERROR("Failed to setup dependency " << availableDependencies.at(i)->Name);
return false;
std::chrono::system_clock::time_point deadline =
std::chrono::system_clock::now() + std::chrono::seconds(60);
for(int j = 0; j < actions.size(); ++j)
{
if(!actions.at(j).valid())
{
ssLOG_ERROR("Failed to construct actions for setup");
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return false;
}

std::future_status actionStatus = actions.at(j).wait_until(deadline);
if(actionStatus == std::future_status::ready)
{
if(!actions.at(j).get())
{
ssLOG_ERROR("Setup failed for dependencies");
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return false;
}
}
else
{
ssLOG_ERROR("Dependencies setup timeout");
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return false;
}
}
actions.clear();
}
}


ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return true;
}

bool runcpp2::BuildDependencies(const runcpp2::Data::Profile& profile,
const Data::ScriptInfo& scriptInfo,
const std::vector<Data::DependencyInfo*>& availableDependencies,
const std::vector<std::string>& dependenciesLocalCopiesPaths)
const std::vector<std::string>& dependenciesLocalCopiesPaths,
const int maxThreads)
{
ssLOG_FUNC_INFO();

//If the script info is not populated (i.e. empty script info), don't do anything
if(!scriptInfo.Populated)
return true;


std::vector<std::future<bool>> actions;

//Cache logs for worker threads
ssLOG_ENABLE_CACHE_OUTPUT_FOR_NEW_THREADS();
int logLevel = ssLOG_GET_CURRENT_THREAD_TARGET_LEVEL();

//Run build steps
for(int i = 0; i < availableDependencies.size(); ++i)
{
ssLOG_INFO("Running build commands for " << availableDependencies.at(i)->Name);

if(!RunDependenciesSteps( profile,
availableDependencies.at(i)->Build,
dependenciesLocalCopiesPaths.at(i),
true))
actions.emplace_back
(
std::async
(
std::launch::async,
[i, &profile, &availableDependencies, &dependenciesLocalCopiesPaths, logLevel]()
{
ssLOG_SET_CURRENT_THREAD_TARGET_LEVEL(logLevel);

if(!RunDependenciesSteps( profile,
availableDependencies.at(i)->Build,
dependenciesLocalCopiesPaths.at(i),
true))
{
ssLOG_ERROR("Failed to build dependency " << availableDependencies.at(i)->Name);
return false;
}
return true;
}
)
);

//Evaluate the setup results for each batch
if(actions.size() >= maxThreads || i == availableDependencies.size() - 1)
{
ssLOG_ERROR("Failed to build dependency " << availableDependencies.at(i)->Name);
return false;
std::chrono::system_clock::time_point deadline =
std::chrono::system_clock::now() + std::chrono::seconds(60);
for(int j = 0; j < actions.size(); ++j)
{
if(!actions.at(j).valid())
{
ssLOG_ERROR("Failed to construct actions for building dependencies");
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return false;
}

std::future_status actionStatus = actions.at(j).wait_until(deadline);
if(actionStatus == std::future_status::ready)
{
if(!actions.at(j).get())
{
ssLOG_ERROR("Build failed for dependencies");
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return false;
}
}
else
{
ssLOG_ERROR("Dependencies build timeout");
ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return false;
}
}
actions.clear();
}
}

ssLOG_OUTPUT_ALL_CACHE_GROUPED();
return true;
}

Expand Down Expand Up @@ -875,7 +986,6 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
INTERNAL_RUNCPP2_SAFE_START();

//For each dependency, check if import path exists
//for(Data::DependencyInfo& dependency : scriptInfo.Dependencies)
for(int i = 0; i < scriptInfo.Dependencies.size(); ++i)
{
Data::DependencyInfo& dependency = scriptInfo.Dependencies.at(i);
Expand All @@ -884,7 +994,7 @@ bool runcpp2::ResolveImports( Data::ScriptInfo& scriptInfo,
Data::DependencySource& source = dependency.Source;
if(source.ImportPath.empty())
continue;

if(!source.ImportPath.is_relative())
{
ssLOG_ERROR("Import path is not relative: " << source.ImportPath.string());
Expand Down
8 changes: 6 additions & 2 deletions Src/runcpp2/PipelineSteps.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,7 @@ runcpp2::CheckScriptInfoChanges(const ghc::filesystem::path& buildDir,
const Data::Profile& profile,
const ghc::filesystem::path& absoluteScriptPath,
const Data::ScriptInfo* lastScriptInfo,
const int maxThreads,
bool& outRecompileNeeded,
bool& outRelinkNeeded,
std::vector<std::string>& outChangedDependencies)
Expand Down Expand Up @@ -731,6 +732,7 @@ runcpp2::ProcessDependencies( Data::ScriptInfo& scriptInfo,
const ghc::filesystem::path& buildDir,
const std::unordered_map<CmdOptions, std::string>& currentOptions,
const std::vector<std::string>& changedDependencies,
const int maxThreads,
std::vector<Data::DependencyInfo*>& outAvailableDependencies,
std::vector<std::string>& outGatheredBinariesPaths)
{
Expand Down Expand Up @@ -796,7 +798,8 @@ runcpp2::ProcessDependencies( Data::ScriptInfo& scriptInfo,
scriptInfo,
outAvailableDependencies,
dependenciesLocalCopiesPaths,
dependenciesSourcePaths))
dependenciesSourcePaths,
maxThreads))
{
ssLOG_ERROR("Failed to setup script dependencies");
return PipelineResult::DEPENDENCIES_FAILED;
Expand All @@ -816,7 +819,8 @@ runcpp2::ProcessDependencies( Data::ScriptInfo& scriptInfo,
if(!BuildDependencies( profile,
scriptInfo,
outAvailableDependencies,
dependenciesLocalCopiesPaths))
dependenciesLocalCopiesPaths,
maxThreads))
{
ssLOG_ERROR("Failed to build script dependencies");
return PipelineResult::DEPENDENCIES_FAILED;
Expand Down
Loading